Mailman 3 April 2024 - Python-checkins

gh-117860: Add tests for resolving names when import rebind names (GH-118176)
by serhiy-storchaka 30 Apr '24

30 Apr '24

https://github.com/python/cpython/commit/c0eaa232f63a62e0e0408911ab5f118dca… commit: c0eaa232f63a62e0e0408911ab5f118dca2af607 branch: main author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2024-04-30T17:23:44+03:00 summary: gh-117860: Add tests for resolving names when import rebind names (GH-118176) Add tests for "import", pkgutil.resolve_name() and unittest.mock.path() for cases when "import a.b as x" and "from a import b as x" give different results. files: A Lib/test/test_import/data/package3/__init__.py A Lib/test/test_import/data/package3/submodule.py A Lib/test/test_import/data/package4/__init__.py A Lib/test/test_import/data/package4/submodule.py M Lib/test/test_import/__init__.py M Lib/test/test_pkgutil.py M Lib/test/test_unittest/testmock/testpatch.py M Makefile.pre.in diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 40c3023a827067..f88b51f916cc81 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1327,6 +1327,34 @@ def test_import_from_unloaded_package(self): import package2.submodule1 package2.submodule1.submodule2 + def test_rebinding(self): + # The same data is also used for testing pkgutil.resolve_name() + # in test_pkgutil and mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + + def test_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + from package4 import submodule + self.assertEqual(submodule.attr, 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + from package4 import submodule + self.assertEqual(submodule.attr, 'origin') + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + class OverridingImportBuiltinTests(unittest.TestCase): def test_override_builtin(self): diff --git a/Lib/test/test_import/data/package3/__init__.py b/Lib/test/test_import/data/package3/__init__.py new file mode 100644 index 00000000000000..7033c22a719ec4 --- /dev/null +++ b/Lib/test/test_import/data/package3/__init__.py @@ -0,0 +1,2 @@ +"""Rebinding the package attribute after importing the module.""" +from .submodule import submodule diff --git a/Lib/test/test_import/data/package3/submodule.py b/Lib/test/test_import/data/package3/submodule.py new file mode 100644 index 00000000000000..cd7b30db15e449 --- /dev/null +++ b/Lib/test/test_import/data/package3/submodule.py @@ -0,0 +1,7 @@ +attr = 'submodule' +class A: + attr = 'submodule' +class submodule: + attr = 'rebound' + class B: + attr = 'rebound' diff --git a/Lib/test/test_import/data/package4/__init__.py b/Lib/test/test_import/data/package4/__init__.py new file mode 100644 index 00000000000000..d8af60ab38a319 --- /dev/null +++ b/Lib/test/test_import/data/package4/__init__.py @@ -0,0 +1,5 @@ +"""Binding the package attribute without importing the module.""" +class submodule: + attr = 'origin' + class B: + attr = 'origin' diff --git a/Lib/test/test_import/data/package4/submodule.py b/Lib/test/test_import/data/package4/submodule.py new file mode 100644 index 00000000000000..c861417aece9c3 --- /dev/null +++ b/Lib/test/test_import/data/package4/submodule.py @@ -0,0 +1,3 @@ +attr = 'submodule' +class A: + attr = 'submodule' diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 6fcd726345eeac..e19dce1dbd2583 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -12,6 +12,9 @@ import shutil import zipfile +from test.support.import_helper import DirsOnSysPath +from test.test_importlib.util import uncache + # Note: pkgutil.walk_packages is currently tested in test_runpy. This is # a hack to get a major issue resolved for 3.3b2. Longer term, it should # be moved back here, perhaps by factoring out the helper code for @@ -318,6 +321,38 @@ def test_name_resolution(self): with self.assertRaises(exc): pkgutil.resolve_name(s) + def test_name_resolution_import_rebinding(self): + # The same data is also used for testing import in test_import and + # mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + class PkgutilPEP302Tests(unittest.TestCase): diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index d0046d702a53f4..be75fda7826af1 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -7,9 +7,11 @@ from collections import OrderedDict import unittest +import test from test.test_unittest.testmock import support from test.test_unittest.testmock.support import SomeClass, is_instance +from test.support.import_helper import DirsOnSysPath from test.test_importlib.util import uncache from unittest.mock import ( NonCallableMock, CallableMixin, sentinel, @@ -1728,6 +1730,71 @@ def test(mock): 'exception traceback not propagated') + def test_name_resolution_import_rebinding(self): + # Currently mock.patch uses pkgutil.resolve_name(), but repeat + # similar tests just for the case. + # The same data is also used for testing import in test_import and + # pkgutil.resolve_name() in test_pkgutil. + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + + def test_create_and_specs(self): for kwarg in ('spec', 'spec_set', 'autospec'): p = patch('%s.doesnotexist' % __name__, create=True, diff --git a/Makefile.pre.in b/Makefile.pre.in index 0e52e10602cf85..4e4d01d14ee3cb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2380,6 +2380,8 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_import/data/circular_imports/subpkg2/parent \ test/test_import/data/package \ test/test_import/data/package2 \ + test/test_import/data/package3 \ + test/test_import/data/package4 \ test/test_import/data/unwritable \ test/test_importlib \ test/test_importlib/builtin \

1 0

gh-116622: Redirect stdout and stderr to system log when embedded in an Android app (#118063)
by vstinner 30 Apr '24

30 Apr '24

https://github.com/python/cpython/commit/3b268f4edc02b22257d745363b5cae199b… commit: 3b268f4edc02b22257d745363b5cae199b6e5720 branch: main author: Malcolm Smith <smith(a)chaquo.com> committer: vstinner <vstinner(a)python.org> date: 2024-04-30T16:00:31+02:00 summary: gh-116622: Redirect stdout and stderr to system log when embedded in an Android app (#118063) files: A Lib/_android_support.py A Lib/test/test_android.py A Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst M Python/pylifecycle.c M Python/stdlib_module_names.h M configure M configure.ac diff --git a/Lib/_android_support.py b/Lib/_android_support.py new file mode 100644 index 00000000000000..590e85ea8c2db1 --- /dev/null +++ b/Lib/_android_support.py @@ -0,0 +1,94 @@ +import io +import sys + + +# The maximum length of a log message in bytes, including the level marker and +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in +# platform/system/logging/liblog/include/log/log.h. As of API level 30, messages +# longer than this will be be truncated by logcat. This limit has already been +# reduced at least once in the history of Android (from 4076 to 4068 between API +# level 23 and 26), so leave some headroom. +MAX_BYTES_PER_WRITE = 4000 + +# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this +# size ensures that TextIOWrapper can always avoid exceeding MAX_BYTES_PER_WRITE. +# However, if the actual number of bytes per character is smaller than that, +# then TextIOWrapper may still join multiple consecutive text writes into binary +# writes containing a larger number of characters. +MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 + + +# When embedded in an app on current versions of Android, there's no easy way to +# monitor the C-level stdout and stderr. The testbed comes with a .c file to +# redirect them to the system log using a pipe, but that wouldn't be convenient +# or appropriate for all apps. So we redirect at the Python level instead. +def init_streams(android_log_write, stdout_prio, stderr_prio): + if sys.executable: + return # Not embedded in an app. + + sys.stdout = TextLogStream( + android_log_write, stdout_prio, "python.stdout", errors=sys.stdout.errors) + sys.stderr = TextLogStream( + android_log_write, stderr_prio, "python.stderr", errors=sys.stderr.errors) + + +class TextLogStream(io.TextIOWrapper): + def __init__(self, android_log_write, prio, tag, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(BinaryLogStream(android_log_write, prio, tag), **kwargs) + self._CHUNK_SIZE = MAX_BYTES_PER_WRITE + + def __repr__(self): + return f"<TextLogStream {self.buffer.tag!r}>" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line wherever possible, so split + # the string before sending it to the superclass. Note that + # "".splitlines() == [], so nothing will be logged for an empty string. + for line in s.splitlines(keepends=True): + while line: + super().write(line[:MAX_CHARS_PER_WRITE]) + line = line[MAX_CHARS_PER_WRITE:] + + return len(s) + + +class BinaryLogStream(io.RawIOBase): + def __init__(self, android_log_write, prio, tag): + self.android_log_write = android_log_write + self.prio = prio + self.tag = tag + + def __repr__(self): + return f"<BinaryLogStream {self.tag!r}>" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + self.android_log_write(self.prio, self.tag, + b.replace(b"\x00", b"\xc0\x80")) + + return len(b) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py new file mode 100644 index 00000000000000..115882a4c281f6 --- /dev/null +++ b/Lib/test/test_android.py @@ -0,0 +1,332 @@ +import platform +import queue +import re +import subprocess +import sys +import unittest +from array import array +from contextlib import contextmanager +from threading import Thread +from test.support import LOOPBACK_TIMEOUT +from time import time + + +if sys.platform != "android": + raise unittest.SkipTest("Android-specific") + +api_level = platform.android_ver().api_level + + +# Test redirection of stdout and stderr to the Android log. +(a)unittest.skipIf( + api_level < 23 and platform.machine() == "aarch64", + "SELinux blocks reading logs on older ARM64 emulators" +) +class TestAndroidOutput(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.logcat_process = subprocess.Popen( + ["logcat", "-v", "tag"], stdout=subprocess.PIPE, + errors="backslashreplace" + ) + self.logcat_queue = queue.Queue() + + def logcat_thread(): + for line in self.logcat_process.stdout: + self.logcat_queue.put(line.rstrip("\n")) + self.logcat_process.stdout.close() + Thread(target=logcat_thread).start() + + from ctypes import CDLL, c_char_p, c_int + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + ANDROID_LOG_INFO = 4 + + # Separate tests using a marker line with a different tag. + tag, message = "python.test", f"{self.id()} {time()}" + android_log_write( + ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8")) + self.assert_log("I", tag, message, skip=True, timeout=5) + + def assert_logs(self, level, tag, expected, **kwargs): + for line in expected: + self.assert_log(level, tag, line, **kwargs) + + def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): + deadline = time() + timeout + while True: + try: + line = self.logcat_queue.get(timeout=(deadline - time())) + except queue.Empty: + self.fail(f"line not found: {expected!r}") + if match := re.fullmatch(fr"(.)/{tag}: (.*)", line): + try: + self.assertEqual(level, match[1]) + self.assertEqual(expected, match[2]) + break + except AssertionError: + if not skip: + raise + + def tearDown(self): + self.logcat_process.terminate() + self.logcat_process.wait(LOOPBACK_TIMEOUT) + + @contextmanager + def unbuffered(self, stream): + stream.reconfigure(write_through=True) + try: + yield + finally: + stream.reconfigure(write_through=False) + + def test_str(self): + for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + with self.subTest(stream=stream_name): + stream = getattr(sys, stream_name) + tag = f"python.{stream_name}" + self.assertEqual(f"<TextLogStream '{tag}'>", repr(stream)) + + self.assertTrue(stream.writable()) + self.assertFalse(stream.readable()) + self.assertEqual("UTF-8", stream.encoding) + self.assertTrue(stream.line_buffering) + self.assertFalse(stream.write_through) + + # stderr is backslashreplace by default; stdout is configured + # that way by libregrtest.main. + self.assertEqual("backslashreplace", stream.errors) + + def write(s, lines=None, *, write_len=None): + if write_len is None: + write_len = len(s) + self.assertEqual(write_len, stream.write(s)) + if lines is None: + lines = [s] + self.assert_logs(level, tag, lines) + + # Single-line messages, + with self.unbuffered(stream): + write("", []) + + write("a") + write("Hello") + write("Hello world") + write(" ") + write(" ") + + # Non-ASCII text + write("ol\u00e9") # Spanish + write("\u4e2d\u6587") # Chinese + + # Non-BMP emoji + write("\U0001f600") + + # Non-encodable surrogates + write("\ud800\udc00", [r"\ud800\udc00"]) + + # Code used by surrogateescape (which isn't enabled here) + write("\udc80", [r"\udc80"]) + + # Null characters are logged using "modified UTF-8". + write("\u0000", [r"\xc0\x80"]) + write("a\u0000", [r"a\xc0\x80"]) + write("\u0000b", [r"\xc0\x80b"]) + write("a\u0000b", [r"a\xc0\x80b"]) + + # Multi-line messages. Avoid identical consecutive lines, as + # they may activate "chatty" filtering and break the tests. + write("\nx", [""]) + write("\na\n", ["x", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d"]) + write("xx", []) + write("f\n\ng", ["exxf", ""]) + write("\n", ["g"]) + + with self.unbuffered(stream): + write("\nx", ["", "x"]) + write("\na\n", ["", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d", "e"]) + write("xx", ["xx"]) + write("f\n\ng", ["f", "", "g"]) + write("\n", [""]) + + # "\r\n" should be translated into "\n". + write("hello\r\n", ["hello"]) + write("hello\r\nworld\r\n", ["hello", "world"]) + write("\r\n", [""]) + + # Non-standard line separators should be preserved. + write("before form feed\x0cafter form feed\n", + ["before form feed\x0cafter form feed"]) + write("before line separator\u2028after line separator\n", + ["before line separator\u2028after line separator"]) + + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + write(CustomStr("custom\n"), ["custom"], write_len=7) + + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write argument must be str, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + # Manual flushing is supported. + write("hello", []) + stream.flush() + self.assert_log(level, tag, "hello") + write("hello", []) + write("world", []) + stream.flush() + self.assert_log(level, tag, "helloworld") + + # Long lines are split into blocks of 1000 characters + # (MAX_CHARS_PER_WRITE in _android_support.py), but + # TextIOWrapper should then join them back together as much as + # possible without exceeding 4000 UTF-8 bytes + # (MAX_BYTES_PER_WRITE). + # + # ASCII (1 byte per character) + write(("foobar" * 700) + "\n", + [("foobar" * 666) + "foob", # 4000 bytes + "ar" + ("foobar" * 33)]) # 200 bytes + + # "Full-width" digits 0-9 (3 bytes per character) + s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" + write((s * 150) + "\n", + [s * 100, # 3000 bytes + s * 50]) # 1500 bytes + + s = "0123456789" + write(s * 200, []) + write(s * 150, []) + write(s * 51, [s * 350]) # 3500 bytes + write("\n", [s * 51]) # 510 bytes + + def test_bytes(self): + for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + with self.subTest(stream=stream_name): + stream = getattr(sys, stream_name).buffer + tag = f"python.{stream_name}" + self.assertEqual(f"<BinaryLogStream '{tag}'>", repr(stream)) + self.assertTrue(stream.writable()) + self.assertFalse(stream.readable()) + + def write(b, lines=None, *, write_len=None): + if write_len is None: + write_len = len(b) + self.assertEqual(write_len, stream.write(b)) + if lines is None: + lines = [b.decode()] + self.assert_logs(level, tag, lines) + + # Single-line messages, + write(b"", []) + + write(b"a") + write(b"Hello") + write(b"Hello world") + write(b" ") + write(b" ") + + # Non-ASCII text + write(b"ol\xc3\xa9") # Spanish + write(b"\xe4\xb8\xad\xe6\x96\x87") # Chinese + + # Non-BMP emoji + write(b"\xf0\x9f\x98\x80") + + # Null bytes are logged using "modified UTF-8". + write(b"\x00", [r"\xc0\x80"]) + write(b"a\x00", [r"a\xc0\x80"]) + write(b"\x00b", [r"\xc0\x80b"]) + write(b"a\x00b", [r"a\xc0\x80b"]) + + # Invalid UTF-8 + write(b"\xff", [r"\xff"]) + write(b"a\xff", [r"a\xff"]) + write(b"\xffb", [r"\xffb"]) + write(b"a\xffb", [r"a\xffb"]) + + # Log entries containing newlines are shown differently by + # `logcat -v tag`, `logcat -v long`, and Android Studio. We + # currently use `logcat -v tag`, which shows each line as if it + # was a separate log entry, but strips a single trailing + # newline. + # + # On newer versions of Android, all three of the above tools (or + # maybe Logcat itself) will also strip any number of leading + # newlines. + write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) + write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) + write(b"\n", [""]) + write(b"b\n", ["b"]) + write(b"c\n\n", ["c", ""]) + write(b"d\ne", ["d", "e"]) + write(b"xx", ["xx"]) + write(b"f\n\ng", ["f", "", "g"]) + write(b"\n", [""]) + + # "\r\n" should be translated into "\n". + write(b"hello\r\n", ["hello"]) + write(b"hello\r\nworld\r\n", ["hello", "world"]) + write(b"\r\n", [""]) + + # Other bytes-like objects are accepted. + write(bytearray(b"bytearray")) + + mv = memoryview(b"memoryview") + write(mv, ["memoryview"]) # Continuous + write(mv[::2], ["mmrve"]) # Discontinuous + + write( + # Android only supports little-endian architectures, so the + # bytes representation is as follows: + array("H", [ + 0, # 00 00 + 1, # 01 00 + 65534, # FE FF + 65535, # FF FF + ]), + + # After encoding null bytes with modified UTF-8, the only + # valid UTF-8 sequence is \x01. All other bytes are handled + # by backslashreplace. + ["\\xc0\\x80\\xc0\\x80" + "\x01\\xc0\\x80" + "\\xfe\\xff" + "\\xff\\xff"], + write_len=8, + ) + + # Non-bytes-like classes are not accepted. + for obj in ["", "hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst new file mode 100644 index 00000000000000..04f84792f667a0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst @@ -0,0 +1 @@ +Redirect stdout and stderr to system log when embedded in an Android app. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0f3ca4a6687514..a672d8c7915ccd 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -71,6 +71,9 @@ static PyStatus add_main_module(PyInterpreterState *interp); static PyStatus init_import_site(void); static PyStatus init_set_builtins_open(void); static PyStatus init_sys_streams(PyThreadState *tstate); +#ifdef __ANDROID__ +static PyStatus init_android_streams(PyThreadState *tstate); +#endif static void wait_for_thread_shutdown(PyThreadState *tstate); static void call_ll_exitfuncs(_PyRuntimeState *runtime); @@ -1223,6 +1226,13 @@ init_interp_main(PyThreadState *tstate) return status; } +#ifdef __ANDROID__ + status = init_android_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + #ifdef Py_DEBUG run_presite(tstate); #endif @@ -2719,6 +2729,73 @@ init_sys_streams(PyThreadState *tstate) } +#ifdef __ANDROID__ +#include <android/log.h> + +static PyObject * +android_log_write_impl(PyObject *self, PyObject *args) +{ + int prio = 0; + const char *tag = NULL; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) { + return NULL; + } + + // Despite its name, this function is part of the public API + // (https://developer.android.com/ndk/reference/group/logging). + __android_log_write(prio, tag, text); + Py_RETURN_NONE; +} + + +static PyMethodDef android_log_write_method = { + "android_log_write", android_log_write_impl, METH_VARARGS +}; + + +static PyStatus +init_android_streams(PyThreadState *tstate) +{ + PyStatus status = _PyStatus_OK(); + PyObject *_android_support = NULL; + PyObject *android_log_write = NULL; + PyObject *result = NULL; + + _android_support = PyImport_ImportModule("_android_support"); + if (_android_support == NULL) { + goto error; + } + + android_log_write = PyCFunction_New(&android_log_write_method, NULL); + if (android_log_write == NULL) { + goto error; + } + + // These log priorities match those used by Java's System.out and System.err. + result = PyObject_CallMethod( + _android_support, "init_streams", "Oii", + android_log_write, ANDROID_LOG_INFO, ANDROID_LOG_WARN); + if (result == NULL) { + goto error; + } + + goto done; + +error: + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Android streams"); + +done: + Py_XDECREF(result); + Py_XDECREF(android_log_write); + Py_XDECREF(_android_support); + return status; +} + +#endif // __ANDROID__ + + static void _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, PyThreadState *tstate) diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 08a66f447e2258..f44abf1f9c3121 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -5,6 +5,7 @@ static const char* _Py_stdlib_module_names[] = { "__future__", "_abc", "_aix_support", +"_android_support", "_ast", "_asyncio", "_bisect", diff --git a/configure b/configure index 571ab8c882aa68..e1d0f36463c44b 100755 --- a/configure +++ b/configure @@ -7103,6 +7103,9 @@ printf "%s\n" "$ANDROID_API_LEVEL" >&6; } printf "%s\n" "#define ANDROID_API_LEVEL $ANDROID_API_LEVEL" >>confdefs.h + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the Android arm ABI" >&5 printf %s "checking for the Android arm ABI... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $_arm_arch" >&5 diff --git a/configure.ac b/configure.ac index a2d6b1357efbc9..7681ea3d3fe3c6 100644 --- a/configure.ac +++ b/configure.ac @@ -1192,6 +1192,9 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then AC_DEFINE_UNQUOTED([ANDROID_API_LEVEL], [$ANDROID_API_LEVEL], [The Android API level.]) + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + AC_MSG_CHECKING([for the Android arm ABI]) AC_MSG_RESULT([$_arm_arch]) if test "$_arm_arch" = 7; then

1 0

[3.12] gh-118404: Fix inspect.signature() for non-comparable callables (GH-118405) (GH-118424)
by serhiy-storchaka 30 Apr '24

30 Apr '24

https://github.com/python/cpython/commit/1b12ad59703b49a9240cbe037abe3d1c2f… commit: 1b12ad59703b49a9240cbe037abe3d1c2f550877 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2024-04-30T12:33:54Z summary: [3.12] gh-118404: Fix inspect.signature() for non-comparable callables (GH-118405) (GH-118424) (cherry picked from commit 11f8348d78c22f85694d7a424541b34d6054a8ee) Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: A Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst M Lib/inspect.py M Lib/test/test_inspect/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index e8c60b77e27201..41dea936b4dd7a 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2145,8 +2145,10 @@ def _signature_is_builtin(obj): ismethoddescriptor(obj) or isinstance(obj, _NonUserDefinedCallables) or # Can't test 'isinstance(type)' here, as it would - # also be True for regular python classes - obj in (type, object)) + # also be True for regular python classes. + # Can't use the `in` operator here, as it would + # invoke the custom __eq__ method. + obj is type or obj is object) def _signature_is_functionlike(obj): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 6db011a1de3242..80e68511e428d1 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -4341,6 +4341,16 @@ class D2(D1): self.assertEqual(inspect.signature(D2), inspect.signature(D1)) + def test_signature_on_non_comparable(self): + class NoncomparableCallable: + def __call__(self, a): + pass + def __eq__(self, other): + 1/0 + self.assertEqual(self.signature(NoncomparableCallable()), + ((('a', ..., ..., 'positional_or_keyword'),), + ...)) + class TestParameterObject(unittest.TestCase): def test_signature_parameter_kinds(self): diff --git a/Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst b/Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst new file mode 100644 index 00000000000000..b8f9ee061ac164 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst @@ -0,0 +1 @@ +Fix :func:`inspect.signature` for non-comparable callables.

1 0

gh-118404: Fix inspect.signature() for non-comparable callables (GH-118405)
by serhiy-storchaka 30 Apr '24

30 Apr '24

https://github.com/python/cpython/commit/11f8348d78c22f85694d7a424541b34d60… commit: 11f8348d78c22f85694d7a424541b34d6054a8ee branch: main author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2024-04-30T15:04:16+03:00 summary: gh-118404: Fix inspect.signature() for non-comparable callables (GH-118405) files: A Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst M Lib/inspect.py M Lib/test/test_inspect/test_inspect.py diff --git a/Lib/inspect.py b/Lib/inspect.py index 1f4216f0389d28..d46514f4b10467 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2179,8 +2179,10 @@ def _signature_is_builtin(obj): ismethoddescriptor(obj) or isinstance(obj, _NonUserDefinedCallables) or # Can't test 'isinstance(type)' here, as it would - # also be True for regular python classes - obj in (type, object)) + # also be True for regular python classes. + # Can't use the `in` operator here, as it would + # invoke the custom __eq__ method. + obj is type or obj is object) def _signature_is_functionlike(obj): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 6b577090bdff68..fbef34eddacb3a 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -4690,6 +4690,16 @@ class D2(D1): self.assertEqual(inspect.signature(D2), inspect.signature(D1)) + def test_signature_on_non_comparable(self): + class NoncomparableCallable: + def __call__(self, a): + pass + def __eq__(self, other): + 1/0 + self.assertEqual(self.signature(NoncomparableCallable()), + ((('a', ..., ..., 'positional_or_keyword'),), + ...)) + class TestParameterObject(unittest.TestCase): def test_signature_parameter_kinds(self): diff --git a/Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst b/Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst new file mode 100644 index 00000000000000..b8f9ee061ac164 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst @@ -0,0 +1 @@ +Fix :func:`inspect.signature` for non-comparable callables.

1 0

GH-118095: Add tier 2 support for YIELD_VALUE (GH-118380)
by markshannon 30 Apr '24

30 Apr '24

https://github.com/python/cpython/commit/5b05d452cd20d9f0cfecdeec90adad3af5… commit: 5b05d452cd20d9f0cfecdeec90adad3af5e4dfff branch: main author: Mark Shannon <mark(a)hotpy.org> committer: markshannon <mark(a)hotpy.org> date: 2024-04-30T11:33:13+01:00 summary: GH-118095: Add tier 2 support for YIELD_VALUE (GH-118380) files: M Include/internal/pycore_opcode_metadata.h M Include/internal/pycore_uop_ids.h M Include/internal/pycore_uop_metadata.h M Python/bytecodes.c M Python/executor_cases.c.h M Python/generated_cases.c.h M Python/optimizer.c M Python/optimizer_bytecodes.c M Python/optimizer_cases.c.h diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 4b1f43cf2af06e..2ccc548ca6c5fd 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1347,6 +1347,7 @@ _PyOpcode_macro_expansion[256] = { [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, 0, 0 } } }, + [YIELD_VALUE] = { .nuops = 1, .uops = { { _YIELD_VALUE, 0, 0 } } }, }; #endif // NEED_OPCODE_METADATA diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 030321ef4fcb23..0a2231e88488c9 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -282,6 +282,7 @@ extern "C" { #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START +#define _YIELD_VALUE YIELD_VALUE #define MAX_UOP_ID 447 #ifdef __cplusplus diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 59e690f3aace35..a84212c1ec0b69 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -91,6 +91,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_ASSERTION_ERROR] = 0, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -500,6 +501,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", + [_YIELD_VALUE] = "_YIELD_VALUE", }; int _PyUop_num_popped(int opcode, int oparg) { @@ -648,6 +650,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GET_AWAITABLE: return 1; + case _YIELD_VALUE: + return 1; case _POP_EXCEPT: return 1; case _LOAD_ASSERTION_ERROR: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f688856d6909ca..eee8b328a65b8c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1089,32 +1089,38 @@ dummy_func( goto resume_frame; } - tier1 inst(YIELD_VALUE, (retval -- unused)) { + inst(YIELD_VALUE, (retval -- value)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. + #if TIER_ONE assert(frame != &entry_frame); - frame->instr_ptr = next_instr; + #endif + frame->instr_ptr++; PyGenObject *gen = _PyFrame_GetGenerator(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); + SYNC_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); } inst(POP_EXCEPT, (exc_value -- )) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2d9acfeea432bc..b17f3762714c72 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1110,6 +1110,47 @@ /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */ + case _YIELD_VALUE: { + PyObject *retval; + PyObject *value; + oparg = CURRENT_OPARG(); + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + case _POP_EXCEPT: { PyObject *exc_value; exc_value = stack_pointer[-1]; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c27505fde3d9fa..7c1cc147de4e1a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6012,31 +6012,41 @@ next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; + PyObject *value; retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. + #if TIER_ONE assert(frame != &entry_frame); - frame->instr_ptr = next_instr; + #endif + frame->instr_ptr++; PyGenObject *gen = _PyFrame_GetGenerator(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); } #undef TIER_ONE diff --git a/Python/optimizer.c b/Python/optimizer.c index fcd7d18f2c2e22..a9a35fc902018a 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -704,7 +704,7 @@ translate_bytecode_to_trace( int nuops = expansion->nuops; RESERVE(nuops + 1); /* One extra for exit */ int16_t last_op = expansion->uops[nuops-1].uop; - if (last_op == _POP_FRAME || last_op == _RETURN_GENERATOR) { + if (last_op == _POP_FRAME || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) { // Check for trace stack underflow now: // We can't bail e.g. in the middle of // LOAD_CONST + _POP_FRAME. @@ -763,7 +763,7 @@ translate_bytecode_to_trace( Py_FatalError("garbled expansion"); } - if (uop == _POP_FRAME || uop == _RETURN_GENERATOR) { + if (uop == _POP_FRAME || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { TRACE_STACK_POP(); /* Set the operand to the function or code object returned to, * to assist optimization passes. (See _PUSH_FRAME below.) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 8bc56342774790..60763286178c71 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -673,6 +673,15 @@ dummy_func(void) { } } + op(_YIELD_VALUE, (unused -- res)) { + OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + } + + op(_FOR_ITER_GEN_FRAME, ( -- )) { + /* We are about to hit the end of the trace */ + goto done; + } + op(_CHECK_STACK_SPACE, ( --)) { assert(corresponding_check_stack == NULL); corresponding_check_stack = this_instr; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4102d00171fbaf..0b696921ebfc9f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -684,6 +684,13 @@ /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + case _YIELD_VALUE: { + _Py_UopsSymbol *res; + OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + stack_pointer[-1] = res; + break; + } + case _POP_EXCEPT: { stack_pointer += -1; break; @@ -1440,11 +1447,8 @@ } case _FOR_ITER_GEN_FRAME: { - _PyInterpreterFrame *gen_frame; - gen_frame = sym_new_not_null(ctx); - if (gen_frame == NULL) goto out_of_space; - stack_pointer[0] = (_Py_UopsSymbol *)gen_frame; - stack_pointer += 1; + /* We are about to hit the end of the trace */ + goto done; break; }

1 0

[3.12] gh-85453: Consistent backquotes on None occurences across datetime.rst (GH-118282) (#118419)
by erlend-aasland 30 Apr '24

30 Apr '24

https://github.com/python/cpython/commit/668163865c96ae141761b23895e128c8f7… commit: 668163865c96ae141761b23895e128c8f7a9f29a branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2024-04-30T10:44:23+02:00 summary: [3.12] gh-85453: Consistent backquotes on None occurences across datetime.rst (GH-118282) (#118419) (cherry picked from commit 0f797402bc77192c76a952410ca8e17359feab3c) Co-authored-by: edson duarte <eduarte.uatach(a)gmail.com> files: M Doc/library/datetime.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index c7545db2f241a6..26bd9f5e9bac97 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1691,7 +1691,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. * ``fold in [0, 1]``. If an argument outside those ranges is given, :exc:`ValueError` is raised. All - default to 0 except *tzinfo*, which defaults to :const:`None`. + default to 0 except *tzinfo*, which defaults to ``None``. Class attributes:

1 0

gh-85453: Consistent backquotes on None occurences across datetime.rst (#118282)
by erlend-aasland 30 Apr '24

30 Apr '24

https://github.com/python/cpython/commit/0f797402bc77192c76a952410ca8e17359… commit: 0f797402bc77192c76a952410ca8e17359feab3c branch: main author: edson duarte <eduarte.uatach(a)gmail.com> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2024-04-30T10:34:15+02:00 summary: gh-85453: Consistent backquotes on None occurences across datetime.rst (#118282) files: M Doc/library/datetime.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index b1b5e04c74b066..f40c6cea83820c 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1735,7 +1735,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. * ``fold in [0, 1]``. If an argument outside those ranges is given, :exc:`ValueError` is raised. All - default to 0 except *tzinfo*, which defaults to :const:`None`. + default to 0 except *tzinfo*, which defaults to ``None``. Class attributes:

1 0

gh-118392: Add note about random.random for multi thread app (gh-118396)
by corona10 30 Apr '24

30 Apr '24

https://github.com/python/cpython/commit/11cbf77f9799e86603bca927a98959c7b9… commit: 11cbf77f9799e86603bca927a98959c7b94fff80 branch: main author: Donghee Na <donghee.na(a)python.org> committer: corona10 <donghee.na92(a)gmail.com> date: 2024-04-30T13:42:13+09:00 summary: gh-118392: Add note about random.random for multi thread app (gh-118396) files: M Doc/library/random.rst diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 8fbce18c56f17c..61798263d61195 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -59,6 +59,12 @@ from sources provided by the operating system. random number generator with a long period and comparatively simple update operations. +.. note:: + The global random number generator and instances of :class:`Random` are thread-safe. + However, in the free-threaded build, concurrent calls to the global generator or + to the same instance of :class:`Random` may encounter contention and poor performance. + Consider using separate instances of :class:`Random` per thread instead. + Bookkeeping functions ---------------------

1 0

gh-118347: Fix Windows installer not updating launcher (GH-118386)
by zooba 29 Apr '24

29 Apr '24

https://github.com/python/cpython/commit/97c0194c9f334f8c7b5366aa639e18d267… commit: 97c0194c9f334f8c7b5366aa639e18d267057e6d branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: zooba <steve.dower(a)microsoft.com> date: 2024-04-29T21:16:17Z summary: gh-118347: Fix Windows installer not updating launcher (GH-118386) (cherry picked from commit 96d8ca7ad6c4f23f023dd6ed0abd042b29dff4a2) Co-authored-by: Steve Dower <steve.dower(a)python.org> files: A Misc/NEWS.d/next/Windows/2024-04-29-13-53-25.gh-issue-118347.U5ZRm_.rst M Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp M Tools/msi/bundle/packagegroups/launcher.wxs diff --git a/Misc/NEWS.d/next/Windows/2024-04-29-13-53-25.gh-issue-118347.U5ZRm_.rst b/Misc/NEWS.d/next/Windows/2024-04-29-13-53-25.gh-issue-118347.U5ZRm_.rst new file mode 100644 index 00000000000000..8f02ed94f100fe --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-04-29-13-53-25.gh-issue-118347.U5ZRm_.rst @@ -0,0 +1 @@ +Fixes launcher updates not being installed. diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp index e0e179e3aede6d..7cddda9b06555d 100644 --- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp +++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp @@ -464,11 +464,11 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { LOC_STRING *pLocString = nullptr; LPCWSTR locKey = L"#(loc.Include_launcherHelp)"; - LONGLONG detectedLauncher; + LONGLONG blockedLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"DetectedLauncher", &detectedLauncher)) && detectedLauncher) { + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherRemove)"; - } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &detectedLauncher)) && detectedLauncher) { + } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherUpgrade)"; } @@ -2671,7 +2671,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { /*Elevate when installing for all users*/ L"InstallAllUsers or " /*Elevate when installing the launcher for all users and it was not detected*/ - L"(Include_launcher and InstallLauncherAllUsers and not DetectedLauncher)" + L"(Include_launcher and InstallLauncherAllUsers and not BlockedLauncher)" L")", L"" }; diff --git a/Tools/msi/bundle/packagegroups/launcher.wxs b/Tools/msi/bundle/packagegroups/launcher.wxs index a6922758f31f14..080598a0a486ef 100644 --- a/Tools/msi/bundle/packagegroups/launcher.wxs +++ b/Tools/msi/bundle/packagegroups/launcher.wxs @@ -11,7 +11,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> <?if $(var.Platform)~="ARM64" ?> <MsiProperty Name="ARM64_SHELLEXT" Value="1" /> <?endif ?> @@ -25,7 +25,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> <?if $(var.Platform)~="ARM64" ?> <MsiProperty Name="ARM64_SHELLEXT" Value="1" /> <?endif ?>

1 0

GH-118306: Update JIT to use LLVM 18 (GH-118307)
by brandtbucher 29 Apr '24

29 Apr '24

https://github.com/python/cpython/commit/8b56d82c59c2983b4292a7f506982f2cab… commit: 8b56d82c59c2983b4292a7f506982f2cab352bb2 branch: main author: Savannah Ostrowski <savannahostrowski(a)gmail.com> committer: brandtbucher <brandtbucher(a)gmail.com> date: 2024-04-29T21:09:16Z summary: GH-118306: Update JIT to use LLVM 18 (GH-118307) files: A Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst M .github/workflows/jit.yml M Python/jit.c M Tools/jit/README.md M Tools/jit/_llvm.py M Tools/jit/_schema.py M Tools/jit/_stencils.py M Tools/jit/_targets.py diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index b37eb727955909..b969e6d13e61a3 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -41,7 +41,7 @@ jobs: - true - false llvm: - - 16 + - 18 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -94,6 +94,7 @@ jobs: - name: Native Windows if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | + choco upgrade llvm -y choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -102,27 +103,31 @@ jobs: - name: Emulated Windows if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | + choco upgrade llvm -y choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} - name: Native macOS if: runner.os == 'macOS' run: | + brew update brew install llvm@${{ matrix.llvm }} SDKROOT="$(xcrun --show-sdk-path)" \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Native Linux if: runner.os == 'Linux' && matrix.architecture == 'x86_64' run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }} make all --jobs 4 ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Emulated Linux if: runner.os == 'Linux' && matrix.architecture != 'x86_64' run: | @@ -139,6 +144,6 @@ jobs: CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ CPP="$CC --preprocess" \ HOSTRUNNER=qemu-${{ matrix.architecture }} \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst new file mode 100644 index 00000000000000..051295541ab7e2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 18 diff --git a/Python/jit.c b/Python/jit.c index 8782adb847cfd6..75ec4fb9756eb7 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -234,6 +234,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) assert((int64_t)value < (1LL << 31)); *loc32 = (uint32_t)value; continue; + case HoleKind_ARM64_RELOC_BRANCH26: case HoleKind_IMAGE_REL_ARM64_BRANCH26: case HoleKind_R_AARCH64_CALL26: case HoleKind_R_AARCH64_JUMP26: @@ -278,6 +279,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21: case HoleKind_IMAGE_REL_ARM64_PAGEBASE_REL21: case HoleKind_R_AARCH64_ADR_GOT_PAGE: + case HoleKind_R_AARCH64_ADR_PREL_PG_HI21: // 21-bit count of pages between this page and an absolute address's // page... I know, I know, it's weird. Pairs nicely with // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below). @@ -341,6 +343,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) case HoleKind_ARM64_RELOC_PAGEOFF12: case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12A: case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L: + case HoleKind_R_AARCH64_ADD_ABS_LO12_NC: case HoleKind_R_AARCH64_LD64_GOT_LO12_NC: // 12-bit low part of an absolute address. Pairs nicely with // ARM64_RELOC_GOT_LOAD_PAGE21 (above). diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 04a6c0780bf972..7b33f99d23f75d 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -23,17 +23,21 @@ sudo ./llvm.sh 16 ### macOS -Install LLVM 16 with [Homebrew](https://brew.sh): +Install LLVM 18 with [Homebrew](https://brew.sh): ```sh -brew install llvm@16 +brew install llvm@18 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. ### Windows -Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Install LLVM 18 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=18), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** + +### Dev Containers + +If you are working CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using…, there's no need to install LLVM as the Fedora 40 base image includes LLVM 18 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 603bbef59ba2e6..74a048ccbfcc58 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -7,7 +7,7 @@ import subprocess import typing -_LLVM_VERSION = 16 +_LLVM_VERSION = 18 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+") _P = typing.ParamSpec("_P") diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 045fd502a03c12..6aef887e12475b 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -2,6 +2,7 @@ import typing HoleKind: typing.TypeAlias = typing.Literal[ + "ARM64_RELOC_BRANCH26", "ARM64_RELOC_GOT_LOAD_PAGE21", "ARM64_RELOC_GOT_LOAD_PAGEOFF12", "ARM64_RELOC_PAGE21", @@ -16,8 +17,10 @@ "IMAGE_REL_I386_REL32", "R_AARCH64_ABS64", "R_AARCH64_ADR_GOT_PAGE", + "R_AARCH64_ADR_PREL_PG_HI21", "R_AARCH64_CALL26", "R_AARCH64_JUMP26", + "R_AARCH64_ADD_ABS_LO12_NC", "R_AARCH64_LD64_GOT_LO12_NC", "R_AARCH64_MOVW_UABS_G0_NC", "R_AARCH64_MOVW_UABS_G1_NC", @@ -68,12 +71,12 @@ class _COFFSymbol(typing.TypedDict): class _ELFSymbol(typing.TypedDict): - Name: dict[typing.Literal["Value"], str] + Name: dict[typing.Literal["Name"], str] Value: int class _MachOSymbol(typing.TypedDict): - Name: dict[typing.Literal["Value"], str] + Name: dict[typing.Literal["Name"], str] Value: int @@ -99,7 +102,7 @@ class ELFSection(typing.TypedDict): Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]] SectionData: dict[typing.Literal["Bytes"], list[int]] Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]] - Type: dict[typing.Literal["Value"], str] + Type: dict[typing.Literal["Name"], str] class MachOSection(typing.TypedDict): diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 243bb3dd134f70..f8ecffcf3ddda2 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -203,7 +203,9 @@ def process_relocations(self, *, alignment: int = 1) -> None: """Fix up all GOT and internal relocations for this stencil group.""" for hole in self.code.holes.copy(): if ( - hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"} + hole.kind in { + "R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26" + } and hole.value is HoleValue.ZERO ): self.code.pad(alignment) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 66db358679239e..91734b36b4ab1b 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -221,7 +221,7 @@ def _handle_relocation( case { "Offset": offset, "Symbol": s, - "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, + "Type": {"Name": "IMAGE_REL_I386_DIR32" as kind}, }: offset += base value, symbol = self._unwrap_dllimport(s) @@ -230,7 +230,7 @@ def _handle_relocation( "Offset": offset, "Symbol": s, "Type": { - "Value": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind + "Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind }, }: offset += base @@ -242,7 +242,7 @@ def _handle_relocation( "Offset": offset, "Symbol": s, "Type": { - "Value": "IMAGE_REL_ARM64_BRANCH26" + "Name": "IMAGE_REL_ARM64_BRANCH26" | "IMAGE_REL_ARM64_PAGEBASE_REL21" | "IMAGE_REL_ARM64_PAGEOFFSET_12A" | "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind @@ -262,7 +262,7 @@ class _ELF( def _handle_section( self, section: _schema.ELFSection, group: _stencils.StencilGroup ) -> None: - section_type = section["Type"]["Value"] + section_type = section["Type"]["Name"] flags = {flag["Name"] for flag in section["Flags"]["Flags"]} if section_type == "SHT_RELA": assert "SHF_INFO_LINK" in flags, flags @@ -290,7 +290,7 @@ def _handle_section( for wrapped_symbol in section["Symbols"]: symbol = wrapped_symbol["Symbol"] offset = len(stencil.body) + symbol["Value"] - name = symbol["Name"]["Value"] + name = symbol["Name"]["Name"] name = name.removeprefix(self.prefix) group.symbols[name] = value, offset stencil.body.extend(section["SectionData"]["Bytes"]) @@ -312,9 +312,9 @@ def _handle_relocation( case { "Addend": addend, "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "R_AARCH64_ADR_GOT_PAGE" + "Name": "R_AARCH64_ADR_GOT_PAGE" | "R_AARCH64_LD64_GOT_LO12_NC" | "R_X86_64_GOTPCREL" | "R_X86_64_GOTPCRELX" @@ -327,8 +327,8 @@ def _handle_relocation( case { "Addend": addend, "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": kind}, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -371,7 +371,7 @@ def _handle_section( for wrapped_symbol in section["Symbols"]: symbol = wrapped_symbol["Symbol"] offset = symbol["Value"] - start_address - name = symbol["Name"]["Value"] + name = symbol["Name"]["Name"] name = name.removeprefix(self.prefix) group.symbols[name] = value, offset assert "Relocations" in section @@ -387,9 +387,9 @@ def _handle_relocation( match relocation: case { "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "ARM64_RELOC_GOT_LOAD_PAGE21" + "Name": "ARM64_RELOC_GOT_LOAD_PAGE21" | "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind }, }: @@ -399,8 +399,8 @@ def _handle_relocation( addend = 0 case { "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, + "Symbol": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -410,13 +410,13 @@ def _handle_relocation( ) case { "Offset": offset, - "Section": {"Value": s}, - "Type": {"Value": "X86_64_RELOC_SIGNED" as kind}, + "Section": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_SIGNED" as kind}, } | { "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind + "Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind }, }: offset += base @@ -427,12 +427,12 @@ def _handle_relocation( ) case { "Offset": offset, - "Section": {"Value": s}, - "Type": {"Value": kind}, + "Section": {"Name": s}, + "Type": {"Name": kind}, } | { "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": kind}, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -446,13 +446,12 @@ def _handle_relocation( def get_target(host: str) -> _COFF | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" if re.fullmatch(r"aarch64-apple-darwin.*", host): - args = ["-mcmodel=large"] - return _MachO(host, alignment=8, args=args, prefix="_") + return _MachO(host, alignment=8, prefix="_") if re.fullmatch(r"aarch64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] return _COFF(host, alignment=8, args=args) if re.fullmatch(r"aarch64-.*-linux-gnu", host): - args = ["-mcmodel=large"] + args = ["-fpic"] return _ELF(host, alignment=8, args=args) if re.fullmatch(r"i686-pc-windows-msvc", host): args = ["-DPy_NO_ENABLE_SHARED"] @@ -463,5 +462,6 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: args = ["-fms-runtime-lib=dll"] return _COFF(host, args=args) if re.fullmatch(r"x86_64-.*-linux-gnu", host): - return _ELF(host) + args = ["-fpic"] + return _ELF(host, args=args) raise ValueError(host)

1 0