Mailman 3 August 2024 - Python-checkins

[3.12] gh-121285: Remove backtracking when parsing tarfile headers (GH-121286) (GH-123543)
by gpshead 31 Aug '24

31 Aug '24

https://github.com/python/cpython/commit/4eaf4891c12589e3c7bdad5f5b076e4c83… commit: 4eaf4891c12589e3c7bdad5f5b076e4c8392dd06 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: gpshead <greg(a)krypto.org> date: 2024-08-31T22:35:24Z summary: [3.12] gh-121285: Remove backtracking when parsing tarfile headers (GH-121286) (GH-123543) gh-121285: Remove backtracking when parsing tarfile headers (GH-121286) * Remove backtracking when parsing tarfile headers * Rewrite PAX header parsing to be stricter * Optimize parsing of GNU extended sparse headers v0.0 (cherry picked from commit 34ddb64d088dd7ccc321f6103d23153256caa5d4) Co-authored-by: Seth Michael Larson <seth(a)python.org> Co-authored-by: Kirill Podoprigora <kirill.bast9(a)mail.ru> Co-authored-by: Gregory P. Smith <greg(a)krypto.org> files: A Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Lib/tarfile.py b/Lib/tarfile.py index e1487e3864d44b..0a0f31eca06c04 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -843,6 +843,9 @@ def data_filter(member, dest_path): # Sentinel for replace() defaults, meaning "don't change the attribute" _KEEP = object() +# Header length is digits followed by a space. +_header_length_prefix_re = re.compile(br"([0-9]{1,20}) ") + class TarInfo(object): """Informational class which holds the details about an archive member given by a tar header block. @@ -1412,37 +1415,59 @@ def _proc_pax(self, tarfile): else: pax_headers = tarfile.pax_headers.copy() - # Check if the pax header contains a hdrcharset field. This tells us - # the encoding of the path, linkpath, uname and gname fields. Normally, - # these fields are UTF-8 encoded but since POSIX.1-2008 tar - # implementations are allowed to store them as raw binary strings if - # the translation to UTF-8 fails. - match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) - if match is not None: - pax_headers["hdrcharset"] = match.group(1).decode("utf-8") - - # For the time being, we don't care about anything other than "BINARY". - # The only other value that is currently allowed by the standard is - # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. - hdrcharset = pax_headers.get("hdrcharset") - if hdrcharset == "BINARY": - encoding = tarfile.encoding - else: - encoding = "utf-8" - # Parse pax header information. A record looks like that: # "%d %s=%s\n" % (length, keyword, value). length is the size # of the complete record including the length field itself and - # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(br"(\d+) ([^=]+)=") + # the newline. pos = 0 - while match := regex.match(buf, pos): - length, keyword = match.groups() - length = int(length) - if length == 0: + encoding = None + raw_headers = [] + while len(buf) > pos and buf[pos] != 0x00: + if not (match := _header_length_prefix_re.match(buf, pos)): + raise InvalidHeaderError("invalid header") + try: + length = int(match.group(1)) + except ValueError: + raise InvalidHeaderError("invalid header") + # Headers must be at least 5 bytes, shortest being '5 x=\n'. + # Value is allowed to be empty. + if length < 5: + raise InvalidHeaderError("invalid header") + if pos + length > len(buf): + raise InvalidHeaderError("invalid header") + + header_value_end_offset = match.start(1) + length - 1 # Last byte of the header + keyword_and_value = buf[match.end(1) + 1:header_value_end_offset] + raw_keyword, equals, raw_value = keyword_and_value.partition(b"=") + + # Check the framing of the header. The last character must be '\n' (0x0A) + if not raw_keyword or equals != b"=" or buf[header_value_end_offset] != 0x0A: raise InvalidHeaderError("invalid header") - value = buf[match.end(2) + 1:match.start(1) + length - 1] + raw_headers.append((length, raw_keyword, raw_value)) + + # Check if the pax header contains a hdrcharset field. This tells us + # the encoding of the path, linkpath, uname and gname fields. Normally, + # these fields are UTF-8 encoded but since POSIX.1-2008 tar + # implementations are allowed to store them as raw binary strings if + # the translation to UTF-8 fails. For the time being, we don't care about + # anything other than "BINARY". The only other value that is currently + # allowed by the standard is "ISO-IR 10646 2000 UTF-8" in other words UTF-8. + # Note that we only follow the initial 'hdrcharset' setting to preserve + # the initial behavior of the 'tarfile' module. + if raw_keyword == b"hdrcharset" and encoding is None: + if raw_value == b"BINARY": + encoding = tarfile.encoding + else: # This branch ensures only the first 'hdrcharset' header is used. + encoding = "utf-8" + pos += length + + # If no explicit hdrcharset is set, we use UTF-8 as a default. + if encoding is None: + encoding = "utf-8" + + # After parsing the raw headers we can decode them to text. + for length, raw_keyword, raw_value in raw_headers: # Normally, we could just use "utf-8" as the encoding and "strict" # as the error handler, but we better not take the risk. For # example, GNU tar <= 1.23 is known to store filenames it cannot @@ -1450,17 +1475,16 @@ def _proc_pax(self, tarfile): # hdrcharset=BINARY header). # We first try the strict standard encoding, and if that fails we # fall back on the user's encoding and error handler. - keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", + keyword = self._decode_pax_field(raw_keyword, "utf-8", "utf-8", tarfile.errors) if keyword in PAX_NAME_FIELDS: - value = self._decode_pax_field(value, encoding, tarfile.encoding, + value = self._decode_pax_field(raw_value, encoding, tarfile.encoding, tarfile.errors) else: - value = self._decode_pax_field(value, "utf-8", "utf-8", + value = self._decode_pax_field(raw_value, "utf-8", "utf-8", tarfile.errors) pax_headers[keyword] = value - pos += length # Fetch the next header. try: @@ -1475,7 +1499,7 @@ def _proc_pax(self, tarfile): elif "GNU.sparse.size" in pax_headers: # GNU extended sparse format version 0.0. - self._proc_gnusparse_00(next, pax_headers, buf) + self._proc_gnusparse_00(next, raw_headers) elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": # GNU extended sparse format version 1.0. @@ -1497,15 +1521,24 @@ def _proc_pax(self, tarfile): return next - def _proc_gnusparse_00(self, next, pax_headers, buf): + def _proc_gnusparse_00(self, next, raw_headers): """Process a GNU tar extended sparse header, version 0.0. """ offsets = [] - for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): - offsets.append(int(match.group(1))) numbytes = [] - for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): - numbytes.append(int(match.group(1))) + for _, keyword, value in raw_headers: + if keyword == b"GNU.sparse.offset": + try: + offsets.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + + elif keyword == b"GNU.sparse.numbytes": + try: + numbytes.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + next.sparse = list(zip(offsets, numbytes)) def _proc_gnusparse_01(self, next, pax_headers): diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 3fbd25e742b181..e28d0311826e2b 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1237,6 +1237,48 @@ def test_pax_number_fields(self): finally: tar.close() + def test_pax_header_bad_formats(self): + # The fields from the pax header have priority over the + # TarInfo. + pax_header_replacements = ( + b" foo=bar\n", + b"0 \n", + b"1 \n", + b"2 \n", + b"3 =\n", + b"4 =a\n", + b"1000000 foo=bar\n", + b"0 foo=bar\n", + b"-12 foo=bar\n", + b"000000000000000000000000036 foo=bar\n", + ) + pax_headers = {"foo": "bar"} + + for replacement in pax_header_replacements: + with self.subTest(header=replacement): + tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT, + encoding="iso8859-1") + try: + t = tarfile.TarInfo() + t.name = "pax" # non-ASCII + t.uid = 1 + t.pax_headers = pax_headers + tar.addfile(t) + finally: + tar.close() + + with open(tmpname, "rb") as f: + data = f.read() + self.assertIn(b"11 foo=bar\n", data) + data = data.replace(b"11 foo=bar\n", replacement) + + with open(tmpname, "wb") as f: + f.truncate() + f.write(data) + + with self.assertRaisesRegex(tarfile.ReadError, r"method tar: ReadError$'invalid header'$"): + tarfile.open(tmpname, encoding="iso8859-1") + class WriteTestBase(TarTest): # Put all write tests in here that are supposed to be tested diff --git a/Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst b/Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst new file mode 100644 index 00000000000000..81f918bfe2b255 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst @@ -0,0 +1,2 @@ +Remove backtracking from tarfile header parsing for ``hdrcharset``, PAX, and +GNU sparse headers.

1 0

gh-121285: Remove backtracking when parsing tarfile headers (GH-121286)
by gpshead 31 Aug '24

31 Aug '24

https://github.com/python/cpython/commit/34ddb64d088dd7ccc321f6103d23153256… commit: 34ddb64d088dd7ccc321f6103d23153256caa5d4 branch: main author: Seth Michael Larson <seth(a)python.org> committer: gpshead <greg(a)krypto.org> date: 2024-08-31T15:17:05-07:00 summary: gh-121285: Remove backtracking when parsing tarfile headers (GH-121286) * Remove backtracking when parsing tarfile headers * Rewrite PAX header parsing to be stricter * Optimize parsing of GNU extended sparse headers v0.0 Co-authored-by: Kirill Podoprigora <kirill.bast9(a)mail.ru> Co-authored-by: Gregory P. Smith <greg(a)krypto.org> files: A Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 4fa7bb6740adbb..1475b3da2d3293 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -845,6 +845,9 @@ def data_filter(member, dest_path): # Sentinel for replace() defaults, meaning "don't change the attribute" _KEEP = object() +# Header length is digits followed by a space. +_header_length_prefix_re = re.compile(br"([0-9]{1,20}) ") + class TarInfo(object): """Informational class which holds the details about an archive member given by a tar header block. @@ -1432,37 +1435,59 @@ def _proc_pax(self, tarfile): else: pax_headers = tarfile.pax_headers.copy() - # Check if the pax header contains a hdrcharset field. This tells us - # the encoding of the path, linkpath, uname and gname fields. Normally, - # these fields are UTF-8 encoded but since POSIX.1-2008 tar - # implementations are allowed to store them as raw binary strings if - # the translation to UTF-8 fails. - match = re.search(br"\d+ hdrcharset=([^\n]+)\n", buf) - if match is not None: - pax_headers["hdrcharset"] = match.group(1).decode("utf-8") - - # For the time being, we don't care about anything other than "BINARY". - # The only other value that is currently allowed by the standard is - # "ISO-IR 10646 2000 UTF-8" in other words UTF-8. - hdrcharset = pax_headers.get("hdrcharset") - if hdrcharset == "BINARY": - encoding = tarfile.encoding - else: - encoding = "utf-8" - # Parse pax header information. A record looks like that: # "%d %s=%s\n" % (length, keyword, value). length is the size # of the complete record including the length field itself and - # the newline. keyword and value are both UTF-8 encoded strings. - regex = re.compile(br"(\d+) ([^=]+)=") + # the newline. pos = 0 - while match := regex.match(buf, pos): - length, keyword = match.groups() - length = int(length) - if length == 0: + encoding = None + raw_headers = [] + while len(buf) > pos and buf[pos] != 0x00: + if not (match := _header_length_prefix_re.match(buf, pos)): + raise InvalidHeaderError("invalid header") + try: + length = int(match.group(1)) + except ValueError: + raise InvalidHeaderError("invalid header") + # Headers must be at least 5 bytes, shortest being '5 x=\n'. + # Value is allowed to be empty. + if length < 5: + raise InvalidHeaderError("invalid header") + if pos + length > len(buf): + raise InvalidHeaderError("invalid header") + + header_value_end_offset = match.start(1) + length - 1 # Last byte of the header + keyword_and_value = buf[match.end(1) + 1:header_value_end_offset] + raw_keyword, equals, raw_value = keyword_and_value.partition(b"=") + + # Check the framing of the header. The last character must be '\n' (0x0A) + if not raw_keyword or equals != b"=" or buf[header_value_end_offset] != 0x0A: raise InvalidHeaderError("invalid header") - value = buf[match.end(2) + 1:match.start(1) + length - 1] + raw_headers.append((length, raw_keyword, raw_value)) + + # Check if the pax header contains a hdrcharset field. This tells us + # the encoding of the path, linkpath, uname and gname fields. Normally, + # these fields are UTF-8 encoded but since POSIX.1-2008 tar + # implementations are allowed to store them as raw binary strings if + # the translation to UTF-8 fails. For the time being, we don't care about + # anything other than "BINARY". The only other value that is currently + # allowed by the standard is "ISO-IR 10646 2000 UTF-8" in other words UTF-8. + # Note that we only follow the initial 'hdrcharset' setting to preserve + # the initial behavior of the 'tarfile' module. + if raw_keyword == b"hdrcharset" and encoding is None: + if raw_value == b"BINARY": + encoding = tarfile.encoding + else: # This branch ensures only the first 'hdrcharset' header is used. + encoding = "utf-8" + pos += length + + # If no explicit hdrcharset is set, we use UTF-8 as a default. + if encoding is None: + encoding = "utf-8" + + # After parsing the raw headers we can decode them to text. + for length, raw_keyword, raw_value in raw_headers: # Normally, we could just use "utf-8" as the encoding and "strict" # as the error handler, but we better not take the risk. For # example, GNU tar <= 1.23 is known to store filenames it cannot @@ -1470,17 +1495,16 @@ def _proc_pax(self, tarfile): # hdrcharset=BINARY header). # We first try the strict standard encoding, and if that fails we # fall back on the user's encoding and error handler. - keyword = self._decode_pax_field(keyword, "utf-8", "utf-8", + keyword = self._decode_pax_field(raw_keyword, "utf-8", "utf-8", tarfile.errors) if keyword in PAX_NAME_FIELDS: - value = self._decode_pax_field(value, encoding, tarfile.encoding, + value = self._decode_pax_field(raw_value, encoding, tarfile.encoding, tarfile.errors) else: - value = self._decode_pax_field(value, "utf-8", "utf-8", + value = self._decode_pax_field(raw_value, "utf-8", "utf-8", tarfile.errors) pax_headers[keyword] = value - pos += length # Fetch the next header. try: @@ -1495,7 +1519,7 @@ def _proc_pax(self, tarfile): elif "GNU.sparse.size" in pax_headers: # GNU extended sparse format version 0.0. - self._proc_gnusparse_00(next, pax_headers, buf) + self._proc_gnusparse_00(next, raw_headers) elif pax_headers.get("GNU.sparse.major") == "1" and pax_headers.get("GNU.sparse.minor") == "0": # GNU extended sparse format version 1.0. @@ -1517,15 +1541,24 @@ def _proc_pax(self, tarfile): return next - def _proc_gnusparse_00(self, next, pax_headers, buf): + def _proc_gnusparse_00(self, next, raw_headers): """Process a GNU tar extended sparse header, version 0.0. """ offsets = [] - for match in re.finditer(br"\d+ GNU.sparse.offset=(\d+)\n", buf): - offsets.append(int(match.group(1))) numbytes = [] - for match in re.finditer(br"\d+ GNU.sparse.numbytes=(\d+)\n", buf): - numbytes.append(int(match.group(1))) + for _, keyword, value in raw_headers: + if keyword == b"GNU.sparse.offset": + try: + offsets.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + + elif keyword == b"GNU.sparse.numbytes": + try: + numbytes.append(int(value.decode())) + except ValueError: + raise InvalidHeaderError("invalid header") + next.sparse = list(zip(offsets, numbytes)) def _proc_gnusparse_01(self, next, pax_headers): diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 5600f0746770b8..54d329a15d4d25 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1261,6 +1261,48 @@ def test_pax_number_fields(self): finally: tar.close() + def test_pax_header_bad_formats(self): + # The fields from the pax header have priority over the + # TarInfo. + pax_header_replacements = ( + b" foo=bar\n", + b"0 \n", + b"1 \n", + b"2 \n", + b"3 =\n", + b"4 =a\n", + b"1000000 foo=bar\n", + b"0 foo=bar\n", + b"-12 foo=bar\n", + b"000000000000000000000000036 foo=bar\n", + ) + pax_headers = {"foo": "bar"} + + for replacement in pax_header_replacements: + with self.subTest(header=replacement): + tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT, + encoding="iso8859-1") + try: + t = tarfile.TarInfo() + t.name = "pax" # non-ASCII + t.uid = 1 + t.pax_headers = pax_headers + tar.addfile(t) + finally: + tar.close() + + with open(tmpname, "rb") as f: + data = f.read() + self.assertIn(b"11 foo=bar\n", data) + data = data.replace(b"11 foo=bar\n", replacement) + + with open(tmpname, "wb") as f: + f.truncate() + f.write(data) + + with self.assertRaisesRegex(tarfile.ReadError, r"method tar: ReadError$'invalid header'$"): + tarfile.open(tmpname, encoding="iso8859-1") + class WriteTestBase(TarTest): # Put all write tests in here that are supposed to be tested diff --git a/Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst b/Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst new file mode 100644 index 00000000000000..81f918bfe2b255 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-07-02-13-39-20.gh-issue-121285.hrl-yI.rst @@ -0,0 +1,2 @@ +Remove backtracking from tarfile header parsing for ``hdrcharset``, PAX, and +GNU sparse headers.

1 0

[3.13] Enable colour for doctest on GitHub Actions (GH-123536) (#123538)
by AA-Turner 31 Aug '24

31 Aug '24

https://github.com/python/cpython/commit/59a6d48a8953d4a30fb828e4c956887b4d… commit: 59a6d48a8953d4a30fb828e4c956887b4db88739 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: AA-Turner <9087854+AA-Turner(a)users.noreply.github.com> date: 2024-08-31T14:39:15Z summary: [3.13] Enable colour for doctest on GitHub Actions (GH-123536) (#123538) Enable colour for doctest on GitHub Actions (GH-123536) (cherry picked from commit 0cba289870d5cd41f24b2f63b9480e4593aa2330) Co-authored-by: Hugo van Kemenade <1324225+hugovk(a)users.noreply.github.com> files: M .github/workflows/reusable-docs.yml diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 4b384f4b3fa602..4b021b3dc32f15 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -11,6 +11,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: build_doc: name: 'Docs'

1 0

Enable colour for doctest on GitHub Actions (#123536)
by AA-Turner 31 Aug '24

31 Aug '24

https://github.com/python/cpython/commit/0cba289870d5cd41f24b2f63b9480e4593… commit: 0cba289870d5cd41f24b2f63b9480e4593aa2330 branch: main author: Hugo van Kemenade <1324225+hugovk(a)users.noreply.github.com> committer: AA-Turner <9087854+AA-Turner(a)users.noreply.github.com> date: 2024-08-31T15:15:44+01:00 summary: Enable colour for doctest on GitHub Actions (#123536) files: M .github/workflows/reusable-docs.yml diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 4b384f4b3fa602..4b021b3dc32f15 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -11,6 +11,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: build_doc: name: 'Docs'

1 0

gh-123309: Remove check for redefined memo entry in pickletools.dis() (GH-123374)
by serhiy-storchaka 31 Aug '24

31 Aug '24

https://github.com/python/cpython/commit/5414b97ce22c2c2c80d698bd50c3c620e4… commit: 5414b97ce22c2c2c80d698bd50c3c620e4aec66f branch: main author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2024-08-31T16:21:49+03:00 summary: gh-123309: Remove check for redefined memo entry in pickletools.dis() (GH-123374) Such pickles are supported by the Unpickler even if the Pickler does not produce them. files: A Misc/NEWS.d/next/Library/2024-08-31-12-34-44.gh-issue-123374.3kE7rb.rst M Lib/pickletools.py M Lib/test/test_pickletools.py diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 51ee4a7a2632ac..c462d26da97ce1 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -2429,8 +2429,6 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): + A memo entry isn't referenced before it's defined. + The markobject isn't stored in the memo. - - + A memo entry isn't redefined. """ # Most of the hair here is for sanity checks, but most of it is needed @@ -2484,7 +2482,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): assert opcode.name == "POP" numtopop = 0 else: - errormsg = markmsg = "no MARK exists on stack" + errormsg = "no MARK exists on stack" # Check for correct memo usage. if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT", "MEMOIZE"): @@ -2494,9 +2492,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): else: assert arg is not None memo_idx = arg - if memo_idx in memo: - errormsg = "memo key %r already defined" % arg - elif not stack: + if not stack: errormsg = "stack is empty -- can't store into memo" elif stack[-1] is markobject: errormsg = "can't store markobject in the memo" diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 8cb1f6dffcc6be..d8ff7a25cbc4b7 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -206,7 +206,7 @@ def test_mark_without_pos(self): def test_no_mark(self): self.check_dis_error(b'Nt.', '''\ 0: N NONE - 1: t TUPLE no MARK exists on stack + 1: t TUPLE ''', 'no MARK exists on stack') def test_put(self): @@ -221,26 +221,16 @@ def test_put(self): ''') def test_put_redefined(self): - self.check_dis_error(b'Np1\np1\n.', '''\ + self.check_dis(b'Np1\np1\nq\x01r\x01\x00\x00\x00\x94.', '''\ 0: N NONE 1: p PUT 1 4: p PUT 1 -''', 'memo key 1 already defined') - self.check_dis_error(b'Np1\nq\x01.', '''\ - 0: N NONE - 1: p PUT 1 - 4: q BINPUT 1 -''', 'memo key 1 already defined') - self.check_dis_error(b'Np1\nr\x01\x00\x00\x00.', '''\ - 0: N NONE - 1: p PUT 1 - 4: r LONG_BINPUT 1 -''', 'memo key 1 already defined') - self.check_dis_error(b'Np1\n\x94.', '''\ - 0: N NONE - 1: p PUT 1 - 4: \\x94 MEMOIZE (as 1) -''', 'memo key None already defined') + 7: q BINPUT 1 + 9: r LONG_BINPUT 1 + 14: \\x94 MEMOIZE (as 1) + 15: . STOP +highest protocol among opcodes = 4 +''') def test_put_empty_stack(self): self.check_dis_error(b'p0\n', '''\ diff --git a/Misc/NEWS.d/next/Library/2024-08-31-12-34-44.gh-issue-123374.3kE7rb.rst b/Misc/NEWS.d/next/Library/2024-08-31-12-34-44.gh-issue-123374.3kE7rb.rst new file mode 100644 index 00000000000000..2fac9079b69e44 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-31-12-34-44.gh-issue-123374.3kE7rb.rst @@ -0,0 +1 @@ +Remove check for redefined memo entry in :func:`pickletools.dis`.

1 0

gh-76960: Fix urljoin() and urldefrag() for URIs with empty components (GH-123273)
by serhiy-storchaka 31 Aug '24

31 Aug '24

https://github.com/python/cpython/commit/fc897fcc01964649f023e0baa4c95d142e… commit: fc897fcc01964649f023e0baa4c95d142e4e8a10 branch: main author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2024-08-31T12:42:08+03:00 summary: gh-76960: Fix urljoin() and urldefrag() for URIs with empty components (GH-123273) * urljoin() with relative reference "?" sets empty query and removes fragment. * Preserve empty components (authority, params, query, fragment) in urljoin(). * Preserve empty components (authority, params, query) in urldefrag(). Also refactor the code and get rid of double _coerce_args() and _coerce_result() calls in urljoin(), urldefrag(), urlparse() and urlunparse(). files: A Misc/NEWS.d/next/Library/2024-08-23-22-01-30.gh-issue-76960.vsANPu.rst M Lib/test/test_urlparse.py M Lib/urllib/parse.py diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 3dbbd9cf1d3364..d49e4388696ab4 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -349,7 +349,7 @@ def _encode(t): split = (scheme,) + split self.checkRoundtrips(url, parsed, split) - def checkJoin(self, base, relurl, expected): + def checkJoin(self, base, relurl, expected, *, relroundtrip=True): with self.subTest(base=base, relurl=relurl): self.assertEqual(urllib.parse.urljoin(base, relurl), expected) baseb = base.encode('ascii') @@ -357,10 +357,11 @@ def checkJoin(self, base, relurl, expected): expectedb = expected.encode('ascii') self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) - relurl = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurl)) - self.assertEqual(urllib.parse.urljoin(base, relurl), expected) - relurlb = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurlb)) - self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + if relroundtrip: + relurl = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurl)) + self.assertEqual(urllib.parse.urljoin(base, relurl), expected) + relurlb = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurlb)) + self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) def test_unparse_parse(self): str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] @@ -526,8 +527,6 @@ def test_RFC3986(self): def test_urljoins(self): self.checkJoin(SIMPLE_BASE, 'g:h','g:h') - self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') - self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') self.checkJoin(SIMPLE_BASE, 'g','http://a/b/c/g') self.checkJoin(SIMPLE_BASE, './g','http://a/b/c/g') self.checkJoin(SIMPLE_BASE, 'g/','http://a/b/c/g/') @@ -548,8 +547,6 @@ def test_urljoins(self): self.checkJoin(SIMPLE_BASE, 'g/./h','http://a/b/c/g/h') self.checkJoin(SIMPLE_BASE, 'g/../h','http://a/b/c/h') self.checkJoin(SIMPLE_BASE, 'http:g','http://a/b/c/g') - self.checkJoin(SIMPLE_BASE, 'http:','http://a/b/c/d') - self.checkJoin(SIMPLE_BASE, 'http:?y','http://a/b/c/d?y') self.checkJoin(SIMPLE_BASE, 'http:g?y','http://a/b/c/g?y') self.checkJoin(SIMPLE_BASE, 'http:g?y/./x','http://a/b/c/g?y/./x') self.checkJoin('http:///', '..','http:///') @@ -579,6 +576,53 @@ def test_urljoins(self): # issue 23703: don't duplicate filename self.checkJoin('a', 'b', 'b') + # Test with empty (but defined) components. + self.checkJoin(RFC1808_BASE, '', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, '#', 'http://a/b/c/d;p?q#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, '?', 'http://a/b/c/d;p?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '?#z', 'http://a/b/c/d;p?#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, '?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, ';', 'http://a/b/c/;') + self.checkJoin(RFC1808_BASE, ';?y', 'http://a/b/c/;?y') + self.checkJoin(RFC1808_BASE, ';#z', 'http://a/b/c/;#z') + self.checkJoin(RFC1808_BASE, ';x', 'http://a/b/c/;x') + self.checkJoin(RFC1808_BASE, '/w', 'http://a/w') + self.checkJoin(RFC1808_BASE, '//', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, '//#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, '//?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, '//;x', 'http://;x') + self.checkJoin(RFC1808_BASE, '///w', 'http://a/w') + self.checkJoin(RFC1808_BASE, '//v', 'http://v') + # For backward compatibility with RFC1630, the scheme name is allowed + # to be present in a relative reference if it is the same as the base + # URI scheme. + self.checkJoin(RFC1808_BASE, 'http:', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, 'http:#', 'http://a/b/c/d;p?q#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, 'http:?', 'http://a/b/c/d;p?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:?#z', 'http://a/b/c/d;p?#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'http:?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, 'http:;', 'http://a/b/c/;') + self.checkJoin(RFC1808_BASE, 'http:;?y', 'http://a/b/c/;?y') + self.checkJoin(RFC1808_BASE, 'http:;#z', 'http://a/b/c/;#z') + self.checkJoin(RFC1808_BASE, 'http:;x', 'http://a/b/c/;x') + self.checkJoin(RFC1808_BASE, 'http:/w', 'http://a/w') + self.checkJoin(RFC1808_BASE, 'http://', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, 'http://#z', 'http://a/b/c/d;p?q#z') + self.checkJoin(RFC1808_BASE, 'http://?y', 'http://a/b/c/d;p?y') + self.checkJoin(RFC1808_BASE, 'http://;x', 'http://;x') + self.checkJoin(RFC1808_BASE, 'http:///w', 'http://a/w') + self.checkJoin(RFC1808_BASE, 'http://v', 'http://v') + # Different scheme is not ignored. + self.checkJoin(RFC1808_BASE, 'https:', 'https:', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:#', 'https:#', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:#z', 'https:#z', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:?', 'https:?', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:?y', 'https:?y', relroundtrip=False) + self.checkJoin(RFC1808_BASE, 'https:;', 'https:;') + self.checkJoin(RFC1808_BASE, 'https:;x', 'https:;x') + def test_RFC2732(self): str_cases = [ ('http://Test.python.org:5432/foo/', 'test.python.org', 5432), @@ -641,16 +685,31 @@ def test_urldefrag(self): ('http://python.org/p?q', 'http://python.org/p?q', ''), (RFC1808_BASE, 'http://a/b/c/d;p?q', 'f'), (RFC2396_BASE, 'http://a/b/c/d;p?q', ''), + ('http://a/b/c;p?q#f', 'http://a/b/c;p?q', 'f'), + ('http://a/b/c;p?q#', 'http://a/b/c;p?q', ''), + ('http://a/b/c;p?q', 'http://a/b/c;p?q', ''), + ('http://a/b/c;p?#f', 'http://a/b/c;p?', 'f'), + ('http://a/b/c;p#f', 'http://a/b/c;p', 'f'), + ('http://a/b/c;?q#f', 'http://a/b/c;?q', 'f'), + ('http://a/b/c?q#f', 'http://a/b/c?q', 'f'), + ('http:///b/c;p?q#f', 'http:///b/c;p?q', 'f'), + ('http:b/c;p?q#f', 'http:b/c;p?q', 'f'), + ('http:;?q#f', 'http:;?q', 'f'), + ('http:?q#f', 'http:?q', 'f'), + ('//a/b/c;p?q#f', '//a/b/c;p?q', 'f'), + ('://a/b/c;p?q#f', '://a/b/c;p?q', 'f'), ] def _encode(t): return type(t)(x.encode('ascii') for x in t) bytes_cases = [_encode(x) for x in str_cases] for url, defrag, frag in str_cases + bytes_cases: - result = urllib.parse.urldefrag(url) - self.assertEqual(result.geturl(), url) - self.assertEqual(result, (defrag, frag)) - self.assertEqual(result.url, defrag) - self.assertEqual(result.fragment, frag) + with self.subTest(url): + result = urllib.parse.urldefrag(url) + hash = '#' if isinstance(url, str) else b'#' + self.assertEqual(result.geturl(), url.rstrip(hash)) + self.assertEqual(result, (defrag, frag)) + self.assertEqual(result.url, defrag) + self.assertEqual(result.fragment, frag) def test_urlsplit_scoped_IPv6(self): p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 33165309daf0c4..5b00ab25c6b4ca 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -392,20 +392,23 @@ def urlparse(url, scheme='', allow_fragments=True): Note that % escapes are not expanded. """ url, scheme, _coerce_result = _coerce_args(url, scheme) - splitresult = urlsplit(url, scheme, allow_fragments) - scheme, netloc, url, query, fragment = splitresult - if scheme in uses_params and ';' in url: - url, params = _splitparams(url) - else: - params = '' - result = ParseResult(scheme, netloc, url, params, query, fragment) + scheme, netloc, url, params, query, fragment = _urlparse(url, scheme, allow_fragments) + result = ParseResult(scheme or '', netloc or '', url, params or '', query or '', fragment or '') return _coerce_result(result) -def _splitparams(url): +def _urlparse(url, scheme=None, allow_fragments=True): + scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) + if (scheme or '') in uses_params and ';' in url: + url, params = _splitparams(url, allow_none=True) + else: + params = None + return (scheme, netloc, url, params, query, fragment) + +def _splitparams(url, allow_none=False): if '/' in url: i = url.find(';', url.rfind('/')) if i < 0: - return url, '' + return url, None if allow_none else '' else: i = url.find(';') return url[:i], url[i+1:] @@ -472,17 +475,23 @@ def urlsplit(url, scheme='', allow_fragments=True): """ url, scheme, _coerce_result = _coerce_args(url, scheme) + scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) + v = SplitResult(scheme or '', netloc or '', url, query or '', fragment or '') + return _coerce_result(v) + +def _urlsplit(url, scheme=None, allow_fragments=True): # Only lstrip url as some applications rely on preserving trailing space. # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) - scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) - for b in _UNSAFE_URL_BYTES_TO_REMOVE: url = url.replace(b, "") - scheme = scheme.replace(b, "") + if scheme is not None: + scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + scheme = scheme.replace(b, "") allow_fragments = bool(allow_fragments) - netloc = query = fragment = '' + netloc = query = fragment = None i = url.find(':') if i > 0 and url[0].isascii() and url[0].isalpha(): for c in url[:i]: @@ -503,8 +512,7 @@ def urlsplit(url, scheme='', allow_fragments=True): if '?' in url: url, query = url.split('?', 1) _checknetloc(netloc) - v = SplitResult(scheme, netloc, url, query, fragment) - return _coerce_result(v) + return (scheme, netloc, url, query, fragment) def urlunparse(components): """Put a parsed URL back together again. This may result in a @@ -513,9 +521,15 @@ def urlunparse(components): (the draft states that these are equivalent).""" scheme, netloc, url, params, query, fragment, _coerce_result = ( _coerce_args(*components)) + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None if params: url = "%s;%s" % (url, params) - return _coerce_result(urlunsplit((scheme, netloc, url, query, fragment))) + return _coerce_result(_urlunsplit(scheme or None, netloc, url, + query or None, fragment or None)) def urlunsplit(components): """Combine the elements of a tuple as returned by urlsplit() into a @@ -525,20 +539,27 @@ def urlunsplit(components): empty query; the RFC states that these are equivalent).""" scheme, netloc, url, query, fragment, _coerce_result = ( _coerce_args(*components)) - if netloc: + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None + return _coerce_result(_urlunsplit(scheme or None, netloc, url, + query or None, fragment or None)) + +def _urlunsplit(scheme, netloc, url, query, fragment): + if netloc is not None: if url and url[:1] != '/': url = '/' + url url = '//' + netloc + url elif url[:2] == '//': url = '//' + url - elif scheme and scheme in uses_netloc and (not url or url[:1] == '/'): - url = '//' + url if scheme: url = scheme + ':' + url - if query: + if query is not None: url = url + '?' + query - if fragment: + if fragment is not None: url = url + '#' + fragment - return _coerce_result(url) + return url def urljoin(base, url, allow_fragments=True): """Join a base URL and a possibly relative URL to form an absolute @@ -549,26 +570,29 @@ def urljoin(base, url, allow_fragments=True): return base base, url, _coerce_result = _coerce_args(base, url) - bscheme, bnetloc, bpath, bparams, bquery, bfragment = \ - urlparse(base, '', allow_fragments) - scheme, netloc, path, params, query, fragment = \ - urlparse(url, bscheme, allow_fragments) + bscheme, bnetloc, bpath, bquery, bfragment = \ + _urlsplit(base, None, allow_fragments) + scheme, netloc, path, query, fragment = \ + _urlsplit(url, None, allow_fragments) + if scheme is None: + scheme = bscheme if scheme != bscheme or scheme not in uses_relative: return _coerce_result(url) if scheme in uses_netloc: if netloc: - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) + return _coerce_result(_urlunsplit(scheme, netloc, path, + query, fragment)) netloc = bnetloc - if not path and not params: + if not path: path = bpath - params = bparams - if not query: + if query is None: query = bquery - return _coerce_result(urlunparse((scheme, netloc, path, - params, query, fragment))) + if fragment is None: + fragment = bfragment + return _coerce_result(_urlunsplit(scheme, netloc, path, + query, fragment)) base_parts = bpath.split('/') if base_parts[-1] != '': @@ -605,8 +629,8 @@ def urljoin(base, url, allow_fragments=True): # then we need to append the trailing '/' resolved_path.append('') - return _coerce_result(urlunparse((scheme, netloc, '/'.join( - resolved_path) or '/', params, query, fragment))) + return _coerce_result(_urlunsplit(scheme, netloc, '/'.join( + resolved_path) or '/', query, fragment)) def urldefrag(url): @@ -618,12 +642,12 @@ def urldefrag(url): """ url, _coerce_result = _coerce_args(url) if '#' in url: - s, n, p, a, q, frag = urlparse(url) - defrag = urlunparse((s, n, p, a, q, '')) + s, n, p, q, frag = _urlsplit(url) + defrag = _urlunsplit(s, n, p, q, None) else: frag = '' defrag = url - return _coerce_result(DefragResult(defrag, frag)) + return _coerce_result(DefragResult(defrag, frag or '')) _hexdig = '0123456789ABCDEFabcdef' _hextobyte = None diff --git a/Misc/NEWS.d/next/Library/2024-08-23-22-01-30.gh-issue-76960.vsANPu.rst b/Misc/NEWS.d/next/Library/2024-08-23-22-01-30.gh-issue-76960.vsANPu.rst new file mode 100644 index 00000000000000..7ce2baee3f9ab6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-23-22-01-30.gh-issue-76960.vsANPu.rst @@ -0,0 +1,5 @@ +Fix :func:`urllib.parse.urljoin` and :func:`urllib.parse.urldefrag` for URIs +containing empty components. For example, :func:`!urljoin()` with relative +reference "?" now sets empty query and removes fragment. +Preserve empty components (authority, params, query, fragment) in :func:`!urljoin`. +Preserve empty components (authority, params, query) in :func:`!urldefrag`.

1 0

gh-123309: Add more tests for the pickletools module (GH-123355)
by serhiy-storchaka 31 Aug '24

31 Aug '24

https://github.com/python/cpython/commit/e5a567b0a721c26c79530249d9aa159afb… commit: e5a567b0a721c26c79530249d9aa159afbd11955 branch: main author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2024-08-31T12:30:05+03:00 summary: gh-123309: Add more tests for the pickletools module (GH-123355) Add tests for genops() and dis(). files: M Lib/test/test_pickletools.py diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index d37af79e878a2e..8cb1f6dffcc6be 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -1,3 +1,4 @@ +import io import pickle import pickletools from test import support @@ -62,6 +63,315 @@ def test_optimize_binput_and_memoize(self): self.assertNotIn(pickle.BINPUT, pickled2) +class SimpleReader: + def __init__(self, data): + self.data = data + self.pos = 0 + + def read(self, n): + data = self.data[self.pos: self.pos + n] + self.pos += n + return data + + def readline(self): + nl = self.data.find(b'\n', self.pos) + 1 + if not nl: + nl = len(self.data) + data = self.data[self.pos: nl] + self.pos = nl + return data + + +class GenopsTests(unittest.TestCase): + def test_genops(self): + it = pickletools.genops(b'(I123\nK\x12J\x12\x34\x56\x78t.') + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, 0), + ('INT', 123, 1), + ('BININT1', 0x12, 6), + ('BININT', 0x78563412, 8), + ('TUPLE', None, 13), + ('STOP', None, 14), + ]) + + def test_from_file(self): + f = io.BytesIO(b'prefix(I123\nK\x12J\x12\x34\x56\x78t.suffix') + self.assertEqual(f.read(6), b'prefix') + it = pickletools.genops(f) + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, 6), + ('INT', 123, 7), + ('BININT1', 0x12, 12), + ('BININT', 0x78563412, 14), + ('TUPLE', None, 19), + ('STOP', None, 20), + ]) + self.assertEqual(f.read(), b'suffix') + + def test_without_pos(self): + f = SimpleReader(b'(I123\nK\x12J\x12\x34\x56\x78t.') + it = pickletools.genops(f) + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, None), + ('INT', 123, None), + ('BININT1', 0x12, None), + ('BININT', 0x78563412, None), + ('TUPLE', None, None), + ('STOP', None, None), + ]) + + def test_no_stop(self): + it = pickletools.genops(b'N') + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + 'pickle exhausted before seeing STOP'): + next(it) + + def test_truncated_data(self): + it = pickletools.genops(b'I123') + with self.assertRaisesRegex(ValueError, + 'no newline found when trying to read stringnl'): + next(it) + it = pickletools.genops(b'J\x12\x34') + with self.assertRaisesRegex(ValueError, + 'not enough data in stream to read int4'): + next(it) + + def test_unknown_opcode(self): + it = pickletools.genops(b'N\xff') + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + r"at position 1, opcode b'\\xff' unknown"): + next(it) + + def test_unknown_opcode_without_pos(self): + f = SimpleReader(b'N\xff') + it = pickletools.genops(f) + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + r"at position <unknown>, opcode b'\\xff' unknown"): + next(it) + + +class DisTests(unittest.TestCase): + maxDiff = None + + def check_dis(self, data, expected, **kwargs): + out = io.StringIO() + pickletools.dis(data, out=out, **kwargs) + self.assertEqual(out.getvalue(), expected) + + def check_dis_error(self, data, expected, expected_error, **kwargs): + out = io.StringIO() + with self.assertRaisesRegex(ValueError, expected_error): + pickletools.dis(data, out=out, **kwargs) + self.assertEqual(out.getvalue(), expected) + + def test_mark(self): + self.check_dis(b'(N(tl.', '''\ + 0: ( MARK + 1: N NONE + 2: ( MARK + 3: t TUPLE (MARK at 2) + 4: l LIST (MARK at 0) + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_indentlevel(self): + self.check_dis(b'(N(tl.', '''\ + 0: ( MARK + 1: N NONE + 2: ( MARK + 3: t TUPLE (MARK at 2) + 4: l LIST (MARK at 0) + 5: . STOP +highest protocol among opcodes = 0 +''', indentlevel=2) + + def test_mark_without_pos(self): + self.check_dis(SimpleReader(b'(N(tl.'), '''\ +( MARK +N NONE +( MARK +t TUPLE (MARK at unknown opcode offset) +l LIST (MARK at unknown opcode offset) +. STOP +highest protocol among opcodes = 0 +''') + + def test_no_mark(self): + self.check_dis_error(b'Nt.', '''\ + 0: N NONE + 1: t TUPLE no MARK exists on stack +''', 'no MARK exists on stack') + + def test_put(self): + self.check_dis(b'Np0\nq\x01r\x02\x00\x00\x00\x94.', '''\ + 0: N NONE + 1: p PUT 0 + 4: q BINPUT 1 + 6: r LONG_BINPUT 2 + 11: \\x94 MEMOIZE (as 3) + 12: . STOP +highest protocol among opcodes = 4 +''') + + def test_put_redefined(self): + self.check_dis_error(b'Np1\np1\n.', '''\ + 0: N NONE + 1: p PUT 1 + 4: p PUT 1 +''', 'memo key 1 already defined') + self.check_dis_error(b'Np1\nq\x01.', '''\ + 0: N NONE + 1: p PUT 1 + 4: q BINPUT 1 +''', 'memo key 1 already defined') + self.check_dis_error(b'Np1\nr\x01\x00\x00\x00.', '''\ + 0: N NONE + 1: p PUT 1 + 4: r LONG_BINPUT 1 +''', 'memo key 1 already defined') + self.check_dis_error(b'Np1\n\x94.', '''\ + 0: N NONE + 1: p PUT 1 + 4: \\x94 MEMOIZE (as 1) +''', 'memo key None already defined') + + def test_put_empty_stack(self): + self.check_dis_error(b'p0\n', '''\ + 0: p PUT 0 +''', "stack is empty -- can't store into memo") + + def test_put_markobject(self): + self.check_dis_error(b'(p0\n', '''\ + 0: ( MARK + 1: p PUT 0 +''', "can't store markobject in the memo") + + def test_get(self): + self.check_dis(b'(Np1\ng1\nh\x01j\x01\x00\x00\x00t.', '''\ + 0: ( MARK + 1: N NONE + 2: p PUT 1 + 5: g GET 1 + 8: h BINGET 1 + 10: j LONG_BINGET 1 + 15: t TUPLE (MARK at 0) + 16: . STOP +highest protocol among opcodes = 1 +''') + + def test_get_without_put(self): + self.check_dis_error(b'g1\n.', '''\ + 0: g GET 1 +''', 'memo key 1 has never been stored into') + self.check_dis_error(b'h\x01.', '''\ + 0: h BINGET 1 +''', 'memo key 1 has never been stored into') + self.check_dis_error(b'j\x01\x00\x00\x00.', '''\ + 0: j LONG_BINGET 1 +''', 'memo key 1 has never been stored into') + + def test_memo(self): + memo = {} + self.check_dis(b'Np1\n.', '''\ + 0: N NONE + 1: p PUT 1 + 4: . STOP +highest protocol among opcodes = 0 +''', memo=memo) + self.check_dis(b'g1\n.', '''\ + 0: g GET 1 + 3: . STOP +highest protocol among opcodes = 0 +''', memo=memo) + + def test_mark_pop(self): + self.check_dis(b'(N00N.', '''\ + 0: ( MARK + 1: N NONE + 2: 0 POP + 3: 0 POP (MARK at 0) + 4: N NONE + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_too_small_stack(self): + self.check_dis_error(b'a', '''\ + 0: a APPEND +''', 'tries to pop 2 items from stack with only 0 items') + self.check_dis_error(b']a', '''\ + 0: ] EMPTY_LIST + 1: a APPEND +''', 'tries to pop 2 items from stack with only 1 items') + + def test_no_stop(self): + self.check_dis_error(b'N', '''\ + 0: N NONE +''', 'pickle exhausted before seeing STOP') + + def test_truncated_data(self): + self.check_dis_error(b'NI123', '''\ + 0: N NONE +''', 'no newline found when trying to read stringnl') + self.check_dis_error(b'NJ\x12\x34', '''\ + 0: N NONE +''', 'not enough data in stream to read int4') + + def test_unknown_opcode(self): + self.check_dis_error(b'N\xff', '''\ + 0: N NONE +''', r"at position 1, opcode b'\\xff' unknown") + + def test_stop_not_empty_stack(self): + self.check_dis_error(b']N.', '''\ + 0: ] EMPTY_LIST + 1: N NONE + 2: . STOP +highest protocol among opcodes = 1 +''', r'stack not empty after STOP: \[list\]') + + def test_annotate(self): + self.check_dis(b'(Nt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: N NONE Push None on the stack. + 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 3: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=1) + self.check_dis(b'(Nt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: N NONE Push None on the stack. + 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 3: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=20) + self.check_dis(b'(((((((ttttttt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: ( MARK Push markobject onto the stack. + 2: ( MARK Push markobject onto the stack. + 3: ( MARK Push markobject onto the stack. + 4: ( MARK Push markobject onto the stack. + 5: ( MARK Push markobject onto the stack. + 6: ( MARK Push markobject onto the stack. + 7: t TUPLE (MARK at 6) Build a tuple out of the topmost stack slice, after markobject. + 8: t TUPLE (MARK at 5) Build a tuple out of the topmost stack slice, after markobject. + 9: t TUPLE (MARK at 4) Build a tuple out of the topmost stack slice, after markobject. + 10: t TUPLE (MARK at 3) Build a tuple out of the topmost stack slice, after markobject. + 11: t TUPLE (MARK at 2) Build a tuple out of the topmost stack slice, after markobject. + 12: t TUPLE (MARK at 1) Build a tuple out of the topmost stack slice, after markobject. + 13: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 14: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=20) + + class MiscTestCase(unittest.TestCase): def test__all__(self): not_exported = {

1 0

[3.13] gh-123407: Enable translating literal and code blocks (GH-123408) (#123530)
by AA-Turner 31 Aug '24

31 Aug '24

https://github.com/python/cpython/commit/578742a9a1a94ec0b890516b25e490c4ba… commit: 578742a9a1a94ec0b890516b25e490c4baa9f97c branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: AA-Turner <9087854+AA-Turner(a)users.noreply.github.com> date: 2024-08-31T06:49:08Z summary: [3.13] gh-123407: Enable translating literal and code blocks (GH-123408) (#123530) gh-123407: Enable translating literal and code blocks (GH-123408) (cherry picked from commit 5332d989af45378e6ae99aeda72bfa82042b8659) Co-authored-by: Maciej Olko <maciej.olko(a)affirm.com> files: M Doc/conf.py diff --git a/Doc/conf.py b/Doc/conf.py index 3860d146a27e85..915e9317fa4bab 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -310,6 +310,7 @@ # Allow translation of index directives gettext_additional_targets = [ 'index', + 'literal-block', ] # Options for HTML output

1 0

[3.12] gh-123407: Enable translating literal and code blocks (GH-123408) (#123531)
by AA-Turner 31 Aug '24

31 Aug '24

https://github.com/python/cpython/commit/5b3fcc04e30971ef4a8002003454cae3de… commit: 5b3fcc04e30971ef4a8002003454cae3de910435 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: AA-Turner <9087854+AA-Turner(a)users.noreply.github.com> date: 2024-08-31T06:38:11Z summary: [3.12] gh-123407: Enable translating literal and code blocks (GH-123408) (#123531) gh-123407: Enable translating literal and code blocks (GH-123408) (cherry picked from commit 5332d989af45378e6ae99aeda72bfa82042b8659) Co-authored-by: Maciej Olko <maciej.olko(a)affirm.com> files: M Doc/conf.py diff --git a/Doc/conf.py b/Doc/conf.py index 75c65fa2dfc4f5..319cdf6079027c 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -294,6 +294,7 @@ # Allow translation of index directives gettext_additional_targets = [ 'index', + 'literal-block', ] # Options for HTML output

1 0

[3.12] gh-108172: do not override OS preferred browser if it is a super-string of a known browser (GH-113011) (GH-123528)
by gpshead 31 Aug '24

31 Aug '24

https://github.com/python/cpython/commit/bf9981830393edf9cc91003a5f89de15d6… commit: bf9981830393edf9cc91003a5f89de15d66d0f2f branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: gpshead <greg(a)krypto.org> date: 2024-08-31T06:33:06Z summary: [3.12] gh-108172: do not override OS preferred browser if it is a super-string of a known browser (GH-113011) (GH-123528) gh-108172: do not override OS preferred browser if it is a super-string of a known browser (GH-113011) When checking if the registering browser is the "OS preferred browser", do not use a substring search - that makes no sense: one can have a preferred browser that looks like a super-string of a known browser, e.g. "firefox-nightly" vs "firefox". https://github.com/python/cpython/issues/108172 explains in more detail, and lays out a potential better future enhancement for this case of just using xdg-open. We'll go with this for now. --------- (cherry picked from commit 10bf615bab9f832971a098f0a42b0d617aea6993) Co-authored-by: Oded Arbel <oded(a)geek.co.il> Co-authored-by: Alex Waygood <Alex.Waygood(a)Gmail.com> files: A Misc/NEWS.d/next/Library/2023-12-12-15-19-58.gh-issue-108172.KyDPuG.rst M Lib/webbrowser.py diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index ba6711e4ef5c33..e9cc71fa331e2d 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -30,7 +30,7 @@ def register(name, klass, instance=None, *, preferred=False): # Preferred browsers go to the front of the list. # Need to match to the default browser returned by xdg-settings, which # may be of the form e.g. "firefox.desktop". - if preferred or (_os_preferred_browser and name in _os_preferred_browser): + if preferred or (_os_preferred_browser and f'{name}.desktop' == _os_preferred_browser): _tryorder.insert(0, name) else: _tryorder.append(name) diff --git a/Misc/NEWS.d/next/Library/2023-12-12-15-19-58.gh-issue-108172.KyDPuG.rst b/Misc/NEWS.d/next/Library/2023-12-12-15-19-58.gh-issue-108172.KyDPuG.rst new file mode 100644 index 00000000000000..5c6b9cd3f81a21 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-12-15-19-58.gh-issue-108172.KyDPuG.rst @@ -0,0 +1 @@ +``webbrowser`` honors OS preferred browser on Linux when its desktop entry name contains the text of a known browser name.

1 0