Mailman 3 June 2023 - Python-checkins

GH-104584: Allow optimizers to opt out of optimizing. (GH-105244)
by markshannon June 5, 2023

June 5, 2023

https://github.com/python/cpython/commit/e8ecb9ee6bec03d0c4490f3e7f1524e56e… commit: e8ecb9ee6bec03d0c4490f3e7f1524e56e2f6a0f branch: main author: Mark Shannon <mark(a)hotpy.org> committer: markshannon <mark(a)hotpy.org> date: 2023-06-05T09:44:23+01:00 summary: GH-104584: Allow optimizers to opt out of optimizing. (GH-105244) files: M Include/cpython/optimizer.h M Python/optimizer.c diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index eb257d73f934..b2d173fb913e 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -21,7 +21,8 @@ typedef struct _PyExecutorObject { typedef struct _PyOptimizerObject _PyOptimizerObject; -typedef _PyExecutorObject *(*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr); +/* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ +typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **); typedef struct _PyOptimizerObject { PyObject_HEAD diff --git a/Python/optimizer.c b/Python/optimizer.c index f29e072410e5..d721bfa2b2de 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -93,14 +93,15 @@ PyUnstable_Replace_Executor(PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutor return 0; } -static _PyExecutorObject * +static int error_optimize( _PyOptimizerObject* self, PyCodeObject *code, - _Py_CODEUNIT *instr) + _Py_CODEUNIT *instr, + _PyExecutorObject **exec) { PyErr_Format(PyExc_SystemError, "Should never call error_optimize"); - return NULL; + return -1; } static PyTypeObject DefaultOptimizer_Type = { @@ -154,15 +155,19 @@ _PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNI return frame; } _PyOptimizerObject *opt = interp->optimizer; - _PyExecutorObject *executor = opt->optimize(opt, frame->f_code, dest); - if (executor == NULL) { - return NULL; + _PyExecutorObject *executor; + int err = opt->optimize(opt, frame->f_code, dest, &executor); + if (err <= 0) { + if (err < 0) { + return NULL; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + return frame; } insert_executor(frame->f_code, src, index, executor); return executor->execute(executor, frame, stack_pointer); } - /** Test support **/ @@ -202,21 +207,23 @@ counter_execute(_PyExecutorObject *self, _PyInterpreterFrame *frame, PyObject ** return frame; } -static _PyExecutorObject * +static int counter_optimize( _PyOptimizerObject* self, PyCodeObject *code, - _Py_CODEUNIT *instr) + _Py_CODEUNIT *instr, + _PyExecutorObject **exec_ptr) { _PyCounterExecutorObject *executor = (_PyCounterExecutorObject *)_PyObject_New(&CounterExecutor_Type); if (executor == NULL) { - return NULL; + return -1; } executor->executor.execute = counter_execute; Py_INCREF(self); executor->optimizer = (_PyCounterOptimizerObject *)self; executor->next_instr = instr; - return (_PyExecutorObject *)executor; + *exec_ptr = (_PyExecutorObject *)executor; + return 1; } static PyObject *

1 0

gh-103558: Add coverage tests for argparse (#103570)
by hauntsaninja June 5, 2023

June 5, 2023

https://github.com/python/cpython/commit/9efaff5fd31a55e3beaa1fa430058de36a… commit: 9efaff5fd31a55e3beaa1fa430058de36a145566 branch: main author: Tian Gao <gaogaotiantian(a)hotmail.com> committer: hauntsaninja <12621235+hauntsaninja(a)users.noreply.github.com> date: 2023-06-05T00:14:00-07:00 summary: gh-103558: Add coverage tests for argparse (#103570) Co-authored-by: Shantanu <12621235+hauntsaninja(a)users.noreply.github.com> Co-authored-by: hauntsaninja <hauntsaninja(a)gmail.com> files: A Misc/NEWS.d/next/Library/2023-04-15-23-26-16.gh-issue-103558.w9OzK4.rst M Lib/argparse.py M Lib/test/test_argparse.py diff --git a/Lib/argparse.py b/Lib/argparse.py index 543d9944f9ed..dfc98695f64e 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1528,6 +1528,8 @@ def _add_container_actions(self, container): title_group_map = {} for group in self._action_groups: if group.title in title_group_map: + # This branch could happen if a derived class added + # groups with duplicated titles in __init__ msg = _('cannot merge actions - two groups are named %r') raise ValueError(msg % (group.title)) title_group_map[group.title] = group @@ -1811,13 +1813,11 @@ def identity(string): # add parent arguments and defaults for parent in parents: + if not isinstance(parent, ArgumentParser): + raise TypeError('parents must be a list of ArgumentParser') self._add_container_actions(parent) - try: - defaults = parent._defaults - except AttributeError: - pass - else: - self._defaults.update(defaults) + defaults = parent._defaults + self._defaults.update(defaults) # ======================= # Pretty __repr__ methods diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 3a62a16cee31..7c1f5d36999a 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -15,7 +15,7 @@ import argparse import warnings -from test.support import os_helper +from test.support import os_helper, captured_stderr from unittest import mock @@ -1382,6 +1382,19 @@ class TestPositionalsActionAppend(ParserTestCase): ('a b c', NS(spam=['a', ['b', 'c']])), ] + +class TestPositionalsActionExtend(ParserTestCase): + """Test the 'extend' action""" + + argument_signatures = [ + Sig('spam', action='extend'), + Sig('spam', action='extend', nargs=2), + ] + failures = ['', '--foo', 'a', 'a b', 'a b c d'] + successes = [ + ('a b c', NS(spam=['a', 'b', 'c'])), + ] + # ======================================== # Combined optionals and positionals tests # ======================================== @@ -1419,6 +1432,32 @@ class TestOptionalsAlmostNumericAndPositionals(ParserTestCase): ] +class TestOptionalsAndPositionalsAppend(ParserTestCase): + argument_signatures = [ + Sig('foo', nargs='*', action='append'), + Sig('--bar'), + ] + failures = ['-foo'] + successes = [ + ('a b', NS(foo=[['a', 'b']], bar=None)), + ('--bar a b', NS(foo=[['b']], bar='a')), + ('a b --bar c', NS(foo=[['a', 'b']], bar='c')), + ] + + +class TestOptionalsAndPositionalsExtend(ParserTestCase): + argument_signatures = [ + Sig('foo', nargs='*', action='extend'), + Sig('--bar'), + ] + failures = ['-foo'] + successes = [ + ('a b', NS(foo=['a', 'b'], bar=None)), + ('--bar a b', NS(foo=['b'], bar='a')), + ('a b --bar c', NS(foo=['a', 'b'], bar='c')), + ] + + class TestEmptyAndSpaceContainingArguments(ParserTestCase): argument_signatures = [ @@ -1899,6 +1938,10 @@ def test_open_args(self): type('foo') m.assert_called_with('foo', *args) + def test_invalid_file_type(self): + with self.assertRaises(ValueError): + argparse.FileType('b')('-test') + class TestFileTypeMissingInitialization(TestCase): """ @@ -2092,6 +2135,27 @@ class TestActionExtend(ParserTestCase): ('--foo f1 --foo f2 f3 f4', NS(foo=['f1', 'f2', 'f3', 'f4'])), ] + +class TestInvalidAction(TestCase): + """Test invalid user defined Action""" + + class ActionWithoutCall(argparse.Action): + pass + + def test_invalid_type(self): + parser = argparse.ArgumentParser() + + parser.add_argument('--foo', action=self.ActionWithoutCall) + self.assertRaises(NotImplementedError, parser.parse_args, ['--foo', 'bar']) + + def test_modified_invalid_action(self): + parser = ErrorRaisingArgumentParser() + action = parser.add_argument('--foo') + # Someone got crazy and did this + action.type = 1 + self.assertRaises(ArgumentParserError, parser.parse_args, ['--foo', 'bar']) + + # ================ # Subparsers tests # ================ @@ -2727,6 +2791,9 @@ def test_groups_parents(self): -x X '''.format(progname, ' ' if progname else '' ))) + def test_wrong_type_parents(self): + self.assertRaises(TypeError, ErrorRaisingArgumentParser, parents=[1]) + # ============================== # Mutually exclusive group tests # ============================== @@ -4743,6 +4810,9 @@ def test_invalid_option_strings(self): self.assertValueError('--') self.assertValueError('---') + def test_invalid_prefix(self): + self.assertValueError('--foo', '+foo') + def test_invalid_type(self): self.assertValueError('--foo', type='int') self.assertValueError('--foo', type=(int, float)) @@ -4807,6 +4877,9 @@ def test_parsers_action_missing_params(self): self.assertTypeError('command', action='parsers', parser_class=argparse.ArgumentParser) + def test_version_missing_params(self): + self.assertTypeError('command', action='version') + def test_required_positional(self): self.assertTypeError('foo', required=True) @@ -5400,6 +5473,17 @@ def test_exclusive_incompatible(self): self.assertRaises(TypeError, parser.parse_intermixed_args, []) self.assertEqual(group.required, True) + def test_invalid_args(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + self.assertRaises(ArgumentParserError, parser.parse_intermixed_args, ['a']) + + parser = ErrorRaisingArgumentParser(prog='PROG') + parser.add_argument('--foo', nargs="*") + parser.add_argument('foo') + with captured_stderr() as stderr: + parser.parse_intermixed_args(['hello', '--foo']) + self.assertIn("UserWarning", stderr.getvalue()) + class TestIntermixedMessageContentError(TestCase): # case where Intermixed gives different error message # error is raised by 1st parsing step diff --git a/Misc/NEWS.d/next/Library/2023-04-15-23-26-16.gh-issue-103558.w9OzK4.rst b/Misc/NEWS.d/next/Library/2023-04-15-23-26-16.gh-issue-103558.w9OzK4.rst new file mode 100644 index 000000000000..e62af647fcc9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-15-23-26-16.gh-issue-103558.w9OzK4.rst @@ -0,0 +1 @@ +Fixed ``parent`` argument validation mechanism of :mod:`argparse`. Improved test coverage.

1 0

[3.10] Update GitHub CI workflow for macOS. (GH-105304)
by ned-deily June 5, 2023

June 5, 2023

https://github.com/python/cpython/commit/6cae23662e091fa528ebddeac89a88f3a8… commit: 6cae23662e091fa528ebddeac89a88f3a8ed34c7 branch: 3.10 author: Ned Deily <nad(a)python.org> committer: ned-deily <nad(a)python.org> date: 2023-06-05T02:42:02-04:00 summary: [3.10] Update GitHub CI workflow for macOS. (GH-105304) files: M .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ac0dd2b84bb44..591df445bf166 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -184,9 +184,8 @@ jobs: run: brew install pkg-config openssl(a)1.1 xz gdbm tcl-tk - name: Configure CPython run: | - CFLAGS="-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" \ - LDFLAGS="-L$(brew --prefix gdbm)/lib -I$(brew --prefix xz)/lib" \ - PKG_CONFIG_PATH="$(brew --prefix tcl-tk)/lib/pkgconfig" \ + CPPFLAGS="-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" \ + LDFLAGS="-L$(brew --prefix gdbm)/lib -L$(brew --prefix xz)/lib" \ ./configure \ --with-pydebug \ --prefix=/opt/python-dev \

1 0

[3.9] Update GitHub CI workflow for macOS. (GH-105303)
by ned-deily June 5, 2023

June 5, 2023

https://github.com/python/cpython/commit/c9bf00b1ad440077b28143693025bcba0d… commit: c9bf00b1ad440077b28143693025bcba0d90f26e branch: 3.9 author: Ned Deily <nad(a)python.org> committer: ned-deily <nad(a)python.org> date: 2023-06-05T02:23:32-04:00 summary: [3.9] Update GitHub CI workflow for macOS. (GH-105303) files: M .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f21cc35b48005..c55eeaab4809f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -147,11 +147,23 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: + HOMEBREW_NO_ANALYTICS: 1 + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v3 - name: Configure CPython - run: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-dev --with-tcltk-includes=-I$(brew --prefix tcl-tk)/include --with-tcltk-libs="-L$(brew --prefix tcl-tk)/lib -ltcl8.6 -ltk8.6" + run: | + brew install pkg-config openssl(a)1.1 xz gdbm tcl-tk + CC=clang \ + CPPFLAGS="-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" \ + LDFLAGS="-L$(brew --prefix gdbm)/lib -L$(brew --prefix xz)/lib" \ + ./configure --prefix=/opt/python-dev \ + --with-pydebug \ + --with-openssl="$(brew --prefix openssl(a)1.1)" \ + --with-tcltk-libs="$(pkg-config --libs tk)" \ + --with-tcltk-includes="$(pkg-config --cflags tk)" - name: Build CPython run: make -j4 - name: Display build info

1 0

[3.8] Update GitHub CI workflow for macOS. (GH-105302)
by ned-deily June 5, 2023

June 5, 2023

https://github.com/python/cpython/commit/b28acfa396efc73c11aaf6387bed807684… commit: b28acfa396efc73c11aaf6387bed807684d446f7 branch: 3.8 author: Ned Deily <nad(a)python.org> committer: ned-deily <nad(a)python.org> date: 2023-06-05T02:11:31-04:00 summary: [3.8] Update GitHub CI workflow for macOS. (GH-105302) files: M .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f34dc5d793447..83b952e6e4591 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -126,10 +126,24 @@ jobs: runs-on: macos-latest needs: check_source if: needs.check_source.outputs.run_tests == 'true' + env: + HOMEBREW_NO_ANALYTICS: 1 + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 steps: - uses: actions/checkout@v2 - name: Configure CPython - run: SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX12.sdk ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-dev + run: | + brew install pkg-config openssl(a)1.1 xz gdbm tcl-tk + SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX12.sdk \ + CC=clang \ + CPPFLAGS="-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" \ + LDFLAGS="-L$(brew --prefix gdbm)/lib -L$(brew --prefix xz)/lib" \ + ./configure --prefix=/opt/python-dev \ + --with-pydebug \ + --with-openssl="$(brew --prefix openssl(a)1.1)" \ + --with-tcltk-libs="$(pkg-config --libs tk)" \ + --with-tcltk-includes="$(pkg-config --cflags tk)" - name: Build CPython run: make -j4 - name: Display build info

1 0

[3.7] gh-105184: document that marshal functions can fail and need to be checked with PyErr_Occurred (GH-105223)
by ned-deily June 5, 2023

June 5, 2023

https://github.com/python/cpython/commit/32590d5fbd29c258316793ac0992ed8d45… commit: 32590d5fbd29c258316793ac0992ed8d45bfb1e8 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: ned-deily <nad(a)python.org> date: 2023-06-05T00:08:36-04:00 summary: [3.7] gh-105184: document that marshal functions can fail and need to be checked with PyErr_Occurred (GH-105223) (cherry picked from commit ee26ca13a129da8cf549409d0a1b2e892ff2b4ec) Co-authored-by: Irit Katriel <1055913+iritkatriel(a)users.noreply.github.com> files: M Doc/c-api/marshal.rst M Python/marshal.c diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index bf5fb4ecbb7dc..8c444223b9d28 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -25,12 +25,16 @@ unmarshalling. Version 2 uses a binary format for floating point numbers. the least-significant 32 bits of *value*; regardless of the size of the native :c:type:`long` type. *version* indicates the file format. + This function can fail, in which case it sets the error indicator. + Use :c:func:`PyErr_Occurred` to check for that. .. c:function:: void PyMarshal_WriteObjectToFile(PyObject *value, FILE *file, int version) Marshal a Python object, *value*, to *file*. *version* indicates the file format. + This function can fail, in which case it sets the error indicator. + Use :c:func:`PyErr_Occurred` to check for that. .. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) diff --git a/Python/marshal.c b/Python/marshal.c index 2e911b7be2759..a4ca71388e6b9 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -619,6 +619,10 @@ w_clear_refs(WFILE *wf) } /* version currently has no effect for writing ints. */ +/* Note that while the documentation states that this function + * can error, currently it never does. Setting an exception in + * this function should be regarded as an API-breaking change. + */ void PyMarshal_WriteLongToFile(long x, FILE *fp, int version) {

1 0

[3.7] gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-104896)
by ned-deily June 5, 2023

June 5, 2023

https://github.com/python/cpython/commit/d28bafa2d3e424b6fdcfd7ae7cde8e71d7… commit: d28bafa2d3e424b6fdcfd7ae7cde8e71d7177369 branch: 3.7 author: stratakis <cstratak(a)redhat.com> committer: ned-deily <nad(a)python.org> date: 2023-06-05T00:02:03-04:00 summary: [3.7] gh-102153: Start stripping C0 control and space chars in `urlsplit` (GH-104896) `urllib.parse.urlsplit` has already been respecting the WHATWG spec a bit GH-25595. This adds more sanitizing to respect the "Remove any leading C0 control or space from input" [rule](https://url.spec.whatwg.org/GH-url-parsing:~:text=Remove%20any%20lead… in response to [CVE-2023-24329](https://nvd.nist.gov/vuln/detail/CVE-2023-24329). (cherry picked from commit d7f8a5fe07b0ff3a419ccec434cc405b21a5a304) (cherry picked from commit 2f630e1ce18ad2e07428296532a68b11dc66ad10) (cherry picked from commit 610cc0ab1b760b2abaac92bd256b96191c46b941) (cherry picked from commit f48a96a28012d28ae37a2f4587a780a5eb779946) Co-authored-by: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> Co-authored-by: Illia Volochii <illia.volochii(a)gmail.com> Co-authored-by: Gregory P. Smith [Google] <greg(a)krypto.org> files: A Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst M Doc/library/urllib.parse.rst M Lib/test/test_urlparse.py M Lib/urllib/parse.py diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index aea6505904be4..7060ee2851523 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -147,6 +147,10 @@ or on combining URL components into a URL string. ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') + .. warning:: + + :func:`urlparse` does not perform validation. See :ref:`URL parsing + security <url-parsing-security>` for details. .. versionchanged:: 3.2 Added IPv6 URL parsing capabilities. @@ -311,8 +315,14 @@ or on combining URL components into a URL string. ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is decomposed before parsing, no error will be raised. - Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline - ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL. + Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0 + control and space characters are stripped from the URL. ``\n``, + ``\r`` and tab ``\t`` characters are removed from the URL at any position. + + .. warning:: + + :func:`urlsplit` does not perform validation. See :ref:`URL parsing + security <url-parsing-security>` for details. .. versionchanged:: 3.6 Out-of-range port numbers now raise :exc:`ValueError`, instead of @@ -325,6 +335,9 @@ or on combining URL components into a URL string. .. versionchanged:: 3.7.11 ASCII newline and tab characters are stripped from the URL. + .. versionchanged:: 3.7.17 + Leading WHATWG C0 control and space characters are stripped from the URL. + .. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser .. function:: urlunsplit(parts) @@ -394,6 +407,27 @@ or on combining URL components into a URL string. .. versionchanged:: 3.2 Result is a structured object rather than a simple 2-tuple. +.. _url-parsing-security: + +URL parsing security +-------------------- + +The :func:`urlsplit` and :func:`urlparse` APIs do not perform **validation** of +inputs. They may not raise errors on inputs that other applications consider +invalid. They may also succeed on some inputs that might not be considered +URLs elsewhere. Their purpose is for practical functionality rather than +purity. + +Instead of raising an exception on unusual input, they may instead return some +component parts as empty strings. Or components may contain more than perhaps +they should. + +We recommend that users of these APIs where the values may be used anywhere +with security implications code defensively. Do some verification within your +code before trusting a returned component part. Does that ``scheme`` make +sense? Is that a sensible ``path``? Is there anything strange about that +``hostname``? etc. + .. _parsing-ascii-encoded-bytes: Parsing ASCII Encoded Bytes diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 3509278a01694..7fd61ffea9f52 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -660,6 +660,65 @@ def test_urlsplit_remove_unsafe_bytes(self): self.assertEqual(p.scheme, "https") self.assertEqual(p.geturl(), "https://www.python.org/javascript:alert('msg')/?query=something#fragment") + def test_urlsplit_strip_url(self): + noise = bytes(range(0, 0x20 + 1)) + base_url = "http://User:Pass@www.python.org:080/doc/?query=yes#frag" + + url = noise.decode("utf-8") + base_url + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, "http") + self.assertEqual(p.netloc, "User:Pass@www.python.org:080") + self.assertEqual(p.path, "/doc/") + self.assertEqual(p.query, "query=yes") + self.assertEqual(p.fragment, "frag") + self.assertEqual(p.username, "User") + self.assertEqual(p.password, "Pass") + self.assertEqual(p.hostname, "www.python.org") + self.assertEqual(p.port, 80) + self.assertEqual(p.geturl(), base_url) + + url = noise + base_url.encode("utf-8") + p = urllib.parse.urlsplit(url) + self.assertEqual(p.scheme, b"http") + self.assertEqual(p.netloc, b"User:Pass@www.python.org:080") + self.assertEqual(p.path, b"/doc/") + self.assertEqual(p.query, b"query=yes") + self.assertEqual(p.fragment, b"frag") + self.assertEqual(p.username, b"User") + self.assertEqual(p.password, b"Pass") + self.assertEqual(p.hostname, b"www.python.org") + self.assertEqual(p.port, 80) + self.assertEqual(p.geturl(), base_url.encode("utf-8")) + + # Test that trailing space is preserved as some applications rely on + # this within query strings. + query_spaces_url = "https://www.python.org:88/doc/?query= " + p = urllib.parse.urlsplit(noise.decode("utf-8") + query_spaces_url) + self.assertEqual(p.scheme, "https") + self.assertEqual(p.netloc, "www.python.org:88") + self.assertEqual(p.path, "/doc/") + self.assertEqual(p.query, "query= ") + self.assertEqual(p.port, 88) + self.assertEqual(p.geturl(), query_spaces_url) + + p = urllib.parse.urlsplit("www.pypi.org ") + # That "hostname" gets considered a "path" due to the + # trailing space and our existing logic... YUCK... + # and re-assembles via geturl aka unurlsplit into the original. + # django.core.validators.URLValidator (at least through v3.2) relies on + # this, for better or worse, to catch it in a ValidationError via its + # regular expressions. + # Here we test the basic round trip concept of such a trailing space. + self.assertEqual(urllib.parse.urlunsplit(p), "www.pypi.org ") + + # with scheme as cache-key + url = "//www.python.org/" + scheme = noise.decode("utf-8") + "https" + noise.decode("utf-8") + for _ in range(2): + p = urllib.parse.urlsplit(url, scheme=scheme) + self.assertEqual(p.scheme, "https") + self.assertEqual(p.geturl(), "https://www.python.org/") + def test_attributes_bad_port(self): """Check handling of invalid ports.""" for bytes in (False, True): @@ -667,7 +726,7 @@ def test_attributes_bad_port(self): for port in ("foo", "1.5", "-1", "0x10"): with self.subTest(bytes=bytes, parse=parse, port=port): netloc = "www.example.net:" + port - url = "http://" + netloc + url = "http://" + netloc + "/" if bytes: netloc = netloc.encode("ascii") url = url.encode("ascii") diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 4f21ce784eee9..8d3556e6d09f9 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -25,6 +25,10 @@ scenarios for parsing, and for backward compatibility purposes, some parsing quirks from older RFCs are retained. The testcases in test_urlparse.py provides a good indicator of parsing behavior. + +The WHATWG URL Parser spec should also be considered. We are not compliant with +it either due to existing user code API behavior expectations (Hyrum's Law). +It serves as a useful guide when making changes. """ import re @@ -76,6 +80,10 @@ '0123456789' '+-.') +# Leading and trailing C0 control and space to be stripped per WHATWG spec. +# == "".join([chr(i) for i in range(0, 0x20 + 1)]) +_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f ' + # Unsafe bytes to be removed per WHATWG spec _UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n'] @@ -426,6 +434,10 @@ def urlsplit(url, scheme='', allow_fragments=True): url, scheme, _coerce_result = _coerce_args(url, scheme) url = _remove_unsafe_bytes_from_url(url) scheme = _remove_unsafe_bytes_from_url(scheme) + # Only lstrip url as some applications rely on preserving trailing space. + # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) + url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) + scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) allow_fragments = bool(allow_fragments) key = url, scheme, allow_fragments, type(url), type(scheme) cached = _parse_cache.get(key, None) diff --git a/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst new file mode 100644 index 0000000000000..e57ac4ed3ac5d --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-03-07-20-59-17.gh-issue-102153.14CLSZ.rst @@ -0,0 +1,3 @@ +:func:`urllib.parse.urlsplit` now strips leading C0 control and space +characters following the specification for URLs defined by WHATWG in +response to CVE-2023-24329. Patch by Illia Volochii.

1 0

[3.9] gh-68966: fix versionchanged in docs (GH-105298)
by ned-deily June 5, 2023

June 5, 2023

https://github.com/python/cpython/commit/89507d5378a4fdf805bd22860c16a4ce02… commit: 89507d5378a4fdf805bd22860c16a4ce02c970b3 branch: 3.9 author: Ned Deily <nad(a)python.org> committer: ned-deily <nad(a)python.org> date: 2023-06-04T23:56:15-04:00 summary: [3.9] gh-68966: fix versionchanged in docs (GH-105298) files: M Doc/library/mailcap.rst diff --git a/Doc/library/mailcap.rst b/Doc/library/mailcap.rst index 7aa3380fecae5..34d5d5ed86b8b 100644 --- a/Doc/library/mailcap.rst +++ b/Doc/library/mailcap.rst @@ -60,7 +60,7 @@ standard. However, mailcap files are supported on most Unix systems. use) to determine whether or not the mailcap line applies. :func:`findmatch` will automatically check such conditions and skip the entry if the check fails. - .. versionchanged:: 3.11 + .. versionchanged:: 3.9.16 To prevent security issues with shell metacharacters (symbols that have special effects in a shell command line), ``findmatch`` will refuse

1 0

[3.8] gh-68966: fix versionchanged in docs (GH-105299)
by ned-deily June 5, 2023

June 5, 2023

https://github.com/python/cpython/commit/d958960466184844bfb36699fa0f2cf910… commit: d958960466184844bfb36699fa0f2cf9109eca7b branch: 3.8 author: Ned Deily <nad(a)python.org> committer: ned-deily <nad(a)python.org> date: 2023-06-04T23:52:32-04:00 summary: [3.8] gh-68966: fix versionchanged in docs (GH-105299) files: M Doc/library/mailcap.rst diff --git a/Doc/library/mailcap.rst b/Doc/library/mailcap.rst index a75857be623ee..2da98bda57af7 100644 --- a/Doc/library/mailcap.rst +++ b/Doc/library/mailcap.rst @@ -54,7 +54,7 @@ standard. However, mailcap files are supported on most Unix systems. use) to determine whether or not the mailcap line applies. :func:`findmatch` will automatically check such conditions and skip the entry if the check fails. - .. versionchanged:: 3.11 + .. versionchanged:: 3.8.16 To prevent security issues with shell metacharacters (symbols that have special effects in a shell command line), ``findmatch`` will refuse

1 0