Mailman 3 April 2022 - Python-checkins

bpo-40280: Detect if WASM platform supports threading (GH-32243)
by miss-islington April 2, 2022

April 2, 2022

https://github.com/python/cpython/commit/59be9cd748728b03ac61287681c3010bce… commit: 59be9cd748728b03ac61287681c3010bcec5e558 branch: main author: Christian Heimes <christian(a)python.org> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-04-02T01:12:44-07:00 summary: bpo-40280: Detect if WASM platform supports threading (GH-32243) Automerge-Triggered-By: GH:tiran files: M Lib/test/libregrtest/runtest.py M Lib/test/support/threading_helper.py diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 83c5f249841cc..62cf1a3f1175e 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -11,6 +11,7 @@ from test import support from test.support import os_helper +from test.support import threading_helper from test.libregrtest.cmdline import Namespace from test.libregrtest.save_env import saved_test_environment from test.libregrtest.utils import clear_caches, format_duration, print_warning @@ -179,7 +180,9 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult: output_on_failure = ns.verbose3 - use_timeout = (ns.timeout is not None) + use_timeout = ( + ns.timeout is not None and threading_helper.can_start_thread + ) if use_timeout: faulthandler.dump_traceback_later(ns.timeout, exit=True) diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index 92a64e8354acb..2ae4577f41f7b 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -207,3 +207,30 @@ def __exit__(self, *exc_info): del self.exc_value del self.exc_traceback del self.thread + + +def _can_start_thread() -> bool: + """Detect if Python can start new threads. + + Some WebAssembly platforms do not provide a working pthread + implementation. Thread support is stubbed and any attempt + to create a new thread fails. + + - wasm32-wasi does not have threading. + - wasm32-emscripten can be compiled with or without pthread + support (-s USE_PTHREADS / __EMSCRIPTEN_PTHREADS__). + """ + if sys.platform == "emscripten": + try: + _thread.start_new_thread(lambda: None, ()) + except RuntimeError: + return False + else: + return True + elif sys.platform == "wasi": + return False + else: + # assume all other platforms have working thread support. + return True + +can_start_thread = _can_start_thread()

1 0

bpo-47196: Fix function pointer cast in test_imp (GH-32244)
by miss-islington April 2, 2022

April 2, 2022

https://github.com/python/cpython/commit/7000cd70164707e28ba045b5c036ca7ed7… commit: 7000cd70164707e28ba045b5c036ca7ed73f5dc4 branch: main author: Hood Chatham <roberthoodchatham(a)gmail.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-04-02T01:00:49-07:00 summary: bpo-47196: Fix function pointer cast in test_imp (GH-32244) The function PyInit_imp_dummy is declared as void f(PyObject* spec) but called as void f(void). On wasm targets without the call trampolines this causes a fatal error. Automerge-Triggered-By: GH:tiran files: M Modules/_testmultiphase.c diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 4905269177bf3..e9a37a019134f 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -450,7 +450,7 @@ static PyModuleDef_Slot main_slots[] = { static PyModuleDef main_def = TEST_MODULE_DEF("main", main_slots, testexport_methods); PyMODINIT_FUNC -PyInit__testmultiphase(PyObject *spec) +PyInit__testmultiphase(void) { return PyModuleDef_Init(&main_def); } @@ -495,7 +495,7 @@ static PyModuleDef def_nonmodule = TEST_MODULE_DEF( "_testmultiphase_nonmodule", slots_create_nonmodule, NULL); PyMODINIT_FUNC -PyInit__testmultiphase_nonmodule(PyObject *spec) +PyInit__testmultiphase_nonmodule(void) { return PyModuleDef_Init(&def_nonmodule); } @@ -525,7 +525,7 @@ static PyModuleDef def_nonmodule_with_methods = TEST_MODULE_DEF( "_testmultiphase_nonmodule_with_methods", slots_create_nonmodule, nonmodule_methods); PyMODINIT_FUNC -PyInit__testmultiphase_nonmodule_with_methods(PyObject *spec) +PyInit__testmultiphase_nonmodule_with_methods(void) { return PyModuleDef_Init(&def_nonmodule_with_methods); } @@ -545,7 +545,7 @@ static PyModuleDef def_nonascii_latin = { \ }; PyMODINIT_FUNC -PyInitU__testmultiphase_zkouka_naten_evc07gi8e(PyObject *spec) +PyInitU__testmultiphase_zkouka_naten_evc07gi8e(void) { return PyModuleDef_Init(&def_nonascii_latin); } @@ -563,7 +563,7 @@ static PyModuleDef def_nonascii_kana = { \ }; PyMODINIT_FUNC -PyInitU_eckzbwbhc6jpgzcx415x(PyObject *spec) +PyInitU_eckzbwbhc6jpgzcx415x(void) { return PyModuleDef_Init(&def_nonascii_kana); } @@ -571,7 +571,7 @@ PyInitU_eckzbwbhc6jpgzcx415x(PyObject *spec) /*** Module with a single-character name ***/ PyMODINIT_FUNC -PyInit_x(PyObject *spec) +PyInit_x(void) { return PyModuleDef_Init(&main_def); } @@ -582,7 +582,7 @@ static PyModuleDef null_slots_def = TEST_MODULE_DEF( "_testmultiphase_null_slots", NULL, NULL); PyMODINIT_FUNC -PyInit__testmultiphase_null_slots(PyObject *spec) +PyInit__testmultiphase_null_slots(void) { return PyModuleDef_Init(&null_slots_def); } @@ -598,7 +598,7 @@ static PyModuleDef def_bad_large = TEST_MODULE_DEF( "_testmultiphase_bad_slot_large", slots_bad_large, NULL); PyMODINIT_FUNC -PyInit__testmultiphase_bad_slot_large(PyObject *spec) +PyInit__testmultiphase_bad_slot_large(void) { return PyModuleDef_Init(&def_bad_large); } @@ -612,7 +612,7 @@ static PyModuleDef def_bad_negative = TEST_MODULE_DEF( "_testmultiphase_bad_slot_negative", slots_bad_negative, NULL); PyMODINIT_FUNC -PyInit__testmultiphase_bad_slot_negative(PyObject *spec) +PyInit__testmultiphase_bad_slot_negative(void) { return PyModuleDef_Init(&def_bad_negative); } @@ -630,7 +630,7 @@ static PyModuleDef def_create_int_with_state = { \ }; PyMODINIT_FUNC -PyInit__testmultiphase_create_int_with_state(PyObject *spec) +PyInit__testmultiphase_create_int_with_state(void) { return PyModuleDef_Init(&def_create_int_with_state); } @@ -649,7 +649,7 @@ static PyModuleDef def_negative_size = { \ }; PyMODINIT_FUNC -PyInit__testmultiphase_negative_size(PyObject *spec) +PyInit__testmultiphase_negative_size(void) { return PyModuleDef_Init(&def_negative_size); } @@ -658,26 +658,26 @@ PyInit__testmultiphase_negative_size(PyObject *spec) static PyModuleDef uninitialized_def = TEST_MODULE_DEF("main", main_slots, testexport_methods); PyMODINIT_FUNC -PyInit__testmultiphase_export_uninitialized(PyObject *spec) +PyInit__testmultiphase_export_uninitialized(void) { return (PyObject*) &uninitialized_def; } PyMODINIT_FUNC -PyInit__testmultiphase_export_null(PyObject *spec) +PyInit__testmultiphase_export_null(void) { return NULL; } PyMODINIT_FUNC -PyInit__testmultiphase_export_raise(PyObject *spec) +PyInit__testmultiphase_export_raise(void) { PyErr_SetString(PyExc_SystemError, "bad export function"); return NULL; } PyMODINIT_FUNC -PyInit__testmultiphase_export_unreported_exception(PyObject *spec) +PyInit__testmultiphase_export_unreported_exception(void) { PyErr_SetString(PyExc_SystemError, "bad export function"); return PyModuleDef_Init(&main_def); @@ -698,7 +698,7 @@ static PyModuleDef def_create_null = TEST_MODULE_DEF( "_testmultiphase_create_null", slots_create_null, NULL); PyMODINIT_FUNC -PyInit__testmultiphase_create_null(PyObject *spec) +PyInit__testmultiphase_create_null(void) { return PyModuleDef_Init(&def_create_null); } @@ -719,7 +719,7 @@ static PyModuleDef def_create_raise = TEST_MODULE_DEF( "_testmultiphase_create_null", slots_create_raise, NULL); PyMODINIT_FUNC -PyInit__testmultiphase_create_raise(PyObject *spec) +PyInit__testmultiphase_create_raise(void) { return PyModuleDef_Init(&def_create_raise); } @@ -740,7 +740,7 @@ static PyModuleDef def_create_unreported_exception = TEST_MODULE_DEF( "_testmultiphase_create_unreported_exception", slots_create_unreported_exception, NULL); PyMODINIT_FUNC -PyInit__testmultiphase_create_unreported_exception(PyObject *spec) +PyInit__testmultiphase_create_unreported_exception(void) { return PyModuleDef_Init(&def_create_unreported_exception); } @@ -755,7 +755,7 @@ static PyModuleDef def_nonmodule_with_exec_slots = TEST_MODULE_DEF( "_testmultiphase_nonmodule_with_exec_slots", slots_nonmodule_with_exec_slots, NULL); PyMODINIT_FUNC -PyInit__testmultiphase_nonmodule_with_exec_slots(PyObject *spec) +PyInit__testmultiphase_nonmodule_with_exec_slots(void) { return PyModuleDef_Init(&def_nonmodule_with_exec_slots); } @@ -775,7 +775,7 @@ static PyModuleDef def_exec_err = TEST_MODULE_DEF( "_testmultiphase_exec_err", slots_exec_err, NULL); PyMODINIT_FUNC -PyInit__testmultiphase_exec_err(PyObject *spec) +PyInit__testmultiphase_exec_err(void) { return PyModuleDef_Init(&def_exec_err); } @@ -817,7 +817,7 @@ static PyModuleDef def_exec_unreported_exception = TEST_MODULE_DEF( "_testmultiphase_exec_unreported_exception", slots_exec_unreported_exception, NULL); PyMODINIT_FUNC -PyInit__testmultiphase_exec_unreported_exception(PyObject *spec) +PyInit__testmultiphase_exec_unreported_exception(void) { return PyModuleDef_Init(&def_exec_unreported_exception); } @@ -861,7 +861,7 @@ static PyModuleDef def_meth_state_access = { }; PyMODINIT_FUNC -PyInit__testmultiphase_meth_state_access(PyObject *spec) +PyInit__testmultiphase_meth_state_access(void) { return PyModuleDef_Init(&def_meth_state_access); } @@ -874,7 +874,7 @@ static PyModuleDef def_module_state_shared = { }; PyMODINIT_FUNC -PyInit__test_module_state_shared(PyObject *spec) +PyInit__test_module_state_shared(void) { PyObject *module = PyModule_Create(&def_module_state_shared); if (module == NULL) { @@ -894,7 +894,7 @@ PyInit__test_module_state_shared(PyObject *spec) static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods); PyMODINIT_FUNC -PyInit_imp_dummy(PyObject *spec) +PyInit_imp_dummy(void) { return PyModuleDef_Init(&imp_dummy_def); }

1 0

bpo-47131: Speedup AST comparisons in test_unparse by using node traversal (GH-32132)
by pablogsal April 1, 2022

April 1, 2022

https://github.com/python/cpython/commit/0f68c208fa6a36b6c8ad3d775e64292a66… commit: 0f68c208fa6a36b6c8ad3d775e64292a665ba108 branch: main author: Jeremy Kloth <jeremy.kloth(a)gmail.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2022-04-02T02:54:04+01:00 summary: bpo-47131: Speedup AST comparisons in test_unparse by using node traversal (GH-32132) files: M Lib/test/test_unparse.py diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index e38b33574cccc..f999ae8c16cea 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -130,7 +130,43 @@ class Foo: pass class ASTTestCase(unittest.TestCase): def assertASTEqual(self, ast1, ast2): - self.assertEqual(ast.dump(ast1), ast.dump(ast2)) + # Ensure the comparisons start at an AST node + self.assertIsInstance(ast1, ast.AST) + self.assertIsInstance(ast2, ast.AST) + + # An AST comparison routine modeled after ast.dump(), but + # instead of string building, it traverses the two trees + # in lock-step. + def traverse_compare(a, b, missing=object()): + if type(a) is not type(b): + self.fail(f"{type(a)!r} is not {type(b)!r}") + if isinstance(a, ast.AST): + for field in a._fields: + value1 = getattr(a, field, missing) + value2 = getattr(b, field, missing) + # Singletons are equal by definition, so further + # testing can be skipped. + if value1 is not value2: + traverse_compare(value1, value2) + elif isinstance(a, list): + try: + for node1, node2 in zip(a, b, strict=True): + traverse_compare(node1, node2) + except ValueError: + # Attempt a "pretty" error ala assertSequenceEqual() + len1 = len(a) + len2 = len(b) + if len1 > len2: + what = "First" + diff = len1 - len2 + else: + what = "Second" + diff = len2 - len1 + msg = f"{what} list contains {diff} additional elements." + raise self.failureException(msg) from None + elif a != b: + self.fail(f"{a!r} != {b!r}") + traverse_compare(ast1, ast2) def check_ast_roundtrip(self, code1, **kwargs): with self.subTest(code1=code1, ast_parse_kwargs=kwargs):

1 0

[3.9] bpo-47089: Avoid test_compileall failures on Windows (GH-32037). (GH-32240)
by vstinner April 1, 2022

April 1, 2022

https://github.com/python/cpython/commit/306a93b4819db611041da735ca3b34117a… commit: 306a93b4819db611041da735ca3b34117a3bc961 branch: 3.9 author: Jeremy Kloth <jeremy.kloth(a)gmail.com> committer: vstinner <vstinner(a)python.org> date: 2022-04-02T03:40:27+02:00 summary: [3.9] bpo-47089: Avoid test_compileall failures on Windows (GH-32037). (GH-32240) * [3.9] bpo-47089: Avoid test_compileall failures on Windows (GH-32037). (cherry picked from commit 76b8a075b8a79b08468fd0ed06a489a5c815bc11) Co-authored-by: Jeremy Kloth <jeremy.kloth(a)gmail.com> files: M Lib/test/test_compileall.py diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index a904f426b172a..ab647d66f0d12 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -449,32 +449,29 @@ def test_error(self): class CommandLineTestsBase: """Test compileall's CLI.""" - @classmethod - def setUpClass(cls): - for path in filter(os.path.isdir, sys.path): - directory_created = False - directory = pathlib.Path(path) / '__pycache__' - path = directory / 'test.try' - try: - if not directory.is_dir(): - directory.mkdir() - directory_created = True - with path.open('w') as file: - file.write('# for test_compileall') - except OSError: - sys_path_writable = False - break - finally: - support.unlink(str(path)) - if directory_created: - directory.rmdir() - else: - sys_path_writable = True - cls._sys_path_writable = sys_path_writable - - def _skip_if_sys_path_not_writable(self): - if not self._sys_path_writable: - raise unittest.SkipTest('not all entries on sys.path are writable') + def setUp(self): + self.directory = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.directory) + self.pkgdir = os.path.join(self.directory, 'foo') + os.mkdir(self.pkgdir) + self.pkgdir_cachedir = os.path.join(self.pkgdir, '__pycache__') + # Create the __init__.py and a package module. + self.initfn = script_helper.make_script(self.pkgdir, '__init__', '') + self.barfn = script_helper.make_script(self.pkgdir, 'bar', '') + + @contextlib.contextmanager + def temporary_pycache_prefix(self): + """Adjust and restore sys.pycache_prefix.""" + old_prefix = sys.pycache_prefix + new_prefix = os.path.join(self.directory, '__testcache__') + try: + sys.pycache_prefix = new_prefix + yield { + 'PYTHONPATH': self.directory, + 'PYTHONPYCACHEPREFIX': new_prefix, + } + finally: + sys.pycache_prefix = old_prefix def _get_run_args(self, args): return [*support.optim_args_from_interpreter_flags(), @@ -502,49 +499,39 @@ def assertNotCompiled(self, fn): path = importlib.util.cache_from_source(fn) self.assertFalse(os.path.exists(path)) - def setUp(self): - self.directory = tempfile.mkdtemp() - self.addCleanup(support.rmtree, self.directory) - self.pkgdir = os.path.join(self.directory, 'foo') - os.mkdir(self.pkgdir) - self.pkgdir_cachedir = os.path.join(self.pkgdir, '__pycache__') - # Create the __init__.py and a package module. - self.initfn = script_helper.make_script(self.pkgdir, '__init__', '') - self.barfn = script_helper.make_script(self.pkgdir, 'bar', '') - def test_no_args_compiles_path(self): # Note that -l is implied for the no args case. - self._skip_if_sys_path_not_writable() bazfn = script_helper.make_script(self.directory, 'baz', '') - self.assertRunOK(PYTHONPATH=self.directory) - self.assertCompiled(bazfn) - self.assertNotCompiled(self.initfn) - self.assertNotCompiled(self.barfn) + with self.temporary_pycache_prefix() as env: + self.assertRunOK(**env) + self.assertCompiled(bazfn) + self.assertNotCompiled(self.initfn) + self.assertNotCompiled(self.barfn) @without_source_date_epoch # timestamp invalidation test def test_no_args_respects_force_flag(self): - self._skip_if_sys_path_not_writable() bazfn = script_helper.make_script(self.directory, 'baz', '') - self.assertRunOK(PYTHONPATH=self.directory) - pycpath = importlib.util.cache_from_source(bazfn) + with self.temporary_pycache_prefix() as env: + self.assertRunOK(**env) + pycpath = importlib.util.cache_from_source(bazfn) # Set atime/mtime backward to avoid file timestamp resolution issues os.utime(pycpath, (time.time()-60,)*2) mtime = os.stat(pycpath).st_mtime # Without force, no recompilation - self.assertRunOK(PYTHONPATH=self.directory) + self.assertRunOK(**env) mtime2 = os.stat(pycpath).st_mtime self.assertEqual(mtime, mtime2) # Now force it. - self.assertRunOK('-f', PYTHONPATH=self.directory) + self.assertRunOK('-f', **env) mtime2 = os.stat(pycpath).st_mtime self.assertNotEqual(mtime, mtime2) def test_no_args_respects_quiet_flag(self): - self._skip_if_sys_path_not_writable() script_helper.make_script(self.directory, 'baz', '') - noisy = self.assertRunOK(PYTHONPATH=self.directory) + with self.temporary_pycache_prefix() as env: + noisy = self.assertRunOK(**env) self.assertIn(b'Listing ', noisy) - quiet = self.assertRunOK('-q', PYTHONPATH=self.directory) + quiet = self.assertRunOK('-q', **env) self.assertNotIn(b'Listing ', quiet) # Ensure that the default behavior of compileall's CLI is to create

1 0

no-issue: Add assertion to PyModule_GetName for understanding (GH-32236)
by corona10 April 1, 2022

April 1, 2022

https://github.com/python/cpython/commit/b183f486493e7e4c332566392ef18c6b34… commit: b183f486493e7e4c332566392ef18c6b346a6561 branch: main author: Dong-hee Na <donghee.na(a)python.org> committer: corona10 <donghee.na92(a)gmail.com> date: 2022-04-02T09:56:30+09:00 summary: no-issue: Add assertion to PyModule_GetName for understanding (GH-32236) files: M Objects/moduleobject.c diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 738b262288bcd..dba20a0b36463 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -510,8 +510,10 @@ const char * PyModule_GetName(PyObject *m) { PyObject *name = PyModule_GetNameObject(m); - if (name == NULL) + if (name == NULL) { return NULL; + } + assert(Py_REFCNT(name) >= 2); Py_DECREF(name); /* module dict has still a reference */ return PyUnicode_AsUTF8(name); }

1 0

bpo-47089: Avoid test_compileall failures on Windows (GH-32037)
by miss-islington April 1, 2022

April 1, 2022

https://github.com/python/cpython/commit/1069d529590850e87a11b8c559a7fb296e… commit: 1069d529590850e87a11b8c559a7fb296e9c626a branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-04-01T13:38:35-07:00 summary: bpo-47089: Avoid test_compileall failures on Windows (GH-32037) (cherry picked from commit 76b8a075b8a79b08468fd0ed06a489a5c815bc11) Co-authored-by: Jeremy Kloth <jeremy.kloth(a)gmail.com> files: M Lib/test/test_compileall.py diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 33f0c939325f5..a7318bf404c3b 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -457,31 +457,29 @@ def test_error(self): class CommandLineTestsBase: """Test compileall's CLI.""" - @classmethod - def setUpClass(cls): - for path in filter(os.path.isdir, sys.path): - directory_created = False - directory = pathlib.Path(path) / '__pycache__' - path = directory / 'test.try' - try: - if not directory.is_dir(): - directory.mkdir() - directory_created = True - path.write_text('# for test_compileall', encoding="utf-8") - except OSError: - sys_path_writable = False - break - finally: - os_helper.unlink(str(path)) - if directory_created: - directory.rmdir() - else: - sys_path_writable = True - cls._sys_path_writable = sys_path_writable - - def _skip_if_sys_path_not_writable(self): - if not self._sys_path_writable: - raise unittest.SkipTest('not all entries on sys.path are writable') + def setUp(self): + self.directory = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, self.directory) + self.pkgdir = os.path.join(self.directory, 'foo') + os.mkdir(self.pkgdir) + self.pkgdir_cachedir = os.path.join(self.pkgdir, '__pycache__') + # Create the __init__.py and a package module. + self.initfn = script_helper.make_script(self.pkgdir, '__init__', '') + self.barfn = script_helper.make_script(self.pkgdir, 'bar', '') + + @contextlib.contextmanager + def temporary_pycache_prefix(self): + """Adjust and restore sys.pycache_prefix.""" + old_prefix = sys.pycache_prefix + new_prefix = os.path.join(self.directory, '__testcache__') + try: + sys.pycache_prefix = new_prefix + yield { + 'PYTHONPATH': self.directory, + 'PYTHONPYCACHEPREFIX': new_prefix, + } + finally: + sys.pycache_prefix = old_prefix def _get_run_args(self, args): return [*support.optim_args_from_interpreter_flags(), @@ -509,49 +507,39 @@ def assertNotCompiled(self, fn): path = importlib.util.cache_from_source(fn) self.assertFalse(os.path.exists(path)) - def setUp(self): - self.directory = tempfile.mkdtemp() - self.addCleanup(os_helper.rmtree, self.directory) - self.pkgdir = os.path.join(self.directory, 'foo') - os.mkdir(self.pkgdir) - self.pkgdir_cachedir = os.path.join(self.pkgdir, '__pycache__') - # Create the __init__.py and a package module. - self.initfn = script_helper.make_script(self.pkgdir, '__init__', '') - self.barfn = script_helper.make_script(self.pkgdir, 'bar', '') - def test_no_args_compiles_path(self): # Note that -l is implied for the no args case. - self._skip_if_sys_path_not_writable() bazfn = script_helper.make_script(self.directory, 'baz', '') - self.assertRunOK(PYTHONPATH=self.directory) - self.assertCompiled(bazfn) - self.assertNotCompiled(self.initfn) - self.assertNotCompiled(self.barfn) + with self.temporary_pycache_prefix() as env: + self.assertRunOK(**env) + self.assertCompiled(bazfn) + self.assertNotCompiled(self.initfn) + self.assertNotCompiled(self.barfn) @without_source_date_epoch # timestamp invalidation test def test_no_args_respects_force_flag(self): - self._skip_if_sys_path_not_writable() bazfn = script_helper.make_script(self.directory, 'baz', '') - self.assertRunOK(PYTHONPATH=self.directory) - pycpath = importlib.util.cache_from_source(bazfn) + with self.temporary_pycache_prefix() as env: + self.assertRunOK(**env) + pycpath = importlib.util.cache_from_source(bazfn) # Set atime/mtime backward to avoid file timestamp resolution issues os.utime(pycpath, (time.time()-60,)*2) mtime = os.stat(pycpath).st_mtime # Without force, no recompilation - self.assertRunOK(PYTHONPATH=self.directory) + self.assertRunOK(**env) mtime2 = os.stat(pycpath).st_mtime self.assertEqual(mtime, mtime2) # Now force it. - self.assertRunOK('-f', PYTHONPATH=self.directory) + self.assertRunOK('-f', **env) mtime2 = os.stat(pycpath).st_mtime self.assertNotEqual(mtime, mtime2) def test_no_args_respects_quiet_flag(self): - self._skip_if_sys_path_not_writable() script_helper.make_script(self.directory, 'baz', '') - noisy = self.assertRunOK(PYTHONPATH=self.directory) + with self.temporary_pycache_prefix() as env: + noisy = self.assertRunOK(**env) self.assertIn(b'Listing ', noisy) - quiet = self.assertRunOK('-q', PYTHONPATH=self.directory) + quiet = self.assertRunOK('-q', **env) self.assertNotIn(b'Listing ', quiet) # Ensure that the default behavior of compileall's CLI is to create

1 0

bpo-40280: Emscripten fork_exec now fails early (GH-32224)
by tiran April 1, 2022

April 1, 2022

https://github.com/python/cpython/commit/082d3495d0c820972f09f6109a98ed7eb5… commit: 082d3495d0c820972f09f6109a98ed7eb5a7b79f branch: main author: Christian Heimes <christian(a)python.org> committer: tiran <christian(a)python.org> date: 2022-04-01T21:20:56+02:00 summary: bpo-40280: Emscripten fork_exec now fails early (GH-32224) files: M Lib/subprocess.py M Lib/test/test_subprocess.py M configure M configure.ac diff --git a/Lib/subprocess.py b/Lib/subprocess.py index ad08339b25ddc..b58c578428390 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -96,7 +96,13 @@ "CREATE_NO_WINDOW", "DETACHED_PROCESS", "CREATE_DEFAULT_ERROR_MODE", "CREATE_BREAKAWAY_FROM_JOB"]) else: - import _posixsubprocess + if sys.platform in {"emscripten", "wasi"}: + def _fork_exec(*args, **kwargs): + raise OSError( + errno.ENOTSUP, f"{sys.platform} does not support processes." + ) + else: + from _posixsubprocess import fork_exec as _fork_exec import select import selectors @@ -1777,7 +1783,7 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, for dir in os.get_exec_path(env)) fds_to_keep = set(pass_fds) fds_to_keep.add(errpipe_write) - self.pid = _posixsubprocess.fork_exec( + self.pid = _fork_exec( args, executable_list, close_fds, tuple(sorted(map(int, fds_to_keep))), cwd, env_list, diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 99a25e279df92..8603b9888bb98 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1805,7 +1805,7 @@ class PopenNoDestructor(subprocess.Popen): def __del__(self): pass - @mock.patch("subprocess._posixsubprocess.fork_exec") + @mock.patch("subprocess._fork_exec") def test_exception_errpipe_normal(self, fork_exec): """Test error passing done through errpipe_write in the good case""" def proper_error(*args): @@ -1822,7 +1822,7 @@ def proper_error(*args): with self.assertRaises(IsADirectoryError): self.PopenNoDestructor(["non_existent_command"]) - @mock.patch("subprocess._posixsubprocess.fork_exec") + @mock.patch("subprocess._fork_exec") def test_exception_errpipe_bad_data(self, fork_exec): """Test error passing done through errpipe_write where its not in the expected format""" @@ -2112,7 +2112,7 @@ def raise_it(): preexec_fn=raise_it) except subprocess.SubprocessError as e: self.assertTrue( - subprocess._posixsubprocess, + subprocess._fork_exec, "Expected a ValueError from the preexec_fn") except ValueError as e: self.assertIn("coconut", e.args[0]) @@ -2600,11 +2600,11 @@ def prepare(): preexec_fn=prepare) except ValueError as err: # Pure Python implementations keeps the message - self.assertIsNone(subprocess._posixsubprocess) + self.assertIsNone(subprocess._fork_exec) self.assertEqual(str(err), "surrogate:\uDCff") except subprocess.SubprocessError as err: # _posixsubprocess uses a default message - self.assertIsNotNone(subprocess._posixsubprocess) + self.assertIsNotNone(subprocess._fork_exec) self.assertEqual(str(err), "Exception occurred in preexec_fn.") else: self.fail("Expected ValueError or subprocess.SubprocessError") diff --git a/configure b/configure index f08a01caafdd5..a5062d7b81573 100755 --- a/configure +++ b/configure @@ -21772,6 +21772,9 @@ case $ac_sys_system/$ac_sys_emscripten_target in #( py_cv_module__curses_panel=n/a py_cv_module__dbm=n/a py_cv_module__gdbm=n/a + py_cv_module__multiprocessing=n/a + py_cv_module__posixshmem=n/a + py_cv_module__posixsubprocess=n/a py_cv_module__scproxy=n/a py_cv_module__tkinter=n/a py_cv_module__xxsubinterpreters=n/a diff --git a/configure.ac b/configure.ac index bc3d8b87def9c..84bc9b3ca5cf0 100644 --- a/configure.ac +++ b/configure.ac @@ -6518,6 +6518,9 @@ AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], [_curses_panel], [_dbm], [_gdbm], + [_multiprocessing], + [_posixshmem], + [_posixsubprocess], [_scproxy], [_tkinter], [_xxsubinterpreters],

1 0

bpo-47089: Avoid test_compileall failures on Windows (GH-32037)
by vstinner April 1, 2022

April 1, 2022

https://github.com/python/cpython/commit/76b8a075b8a79b08468fd0ed06a489a5c8… commit: 76b8a075b8a79b08468fd0ed06a489a5c815bc11 branch: main author: Jeremy Kloth <jeremy.kloth(a)gmail.com> committer: vstinner <vstinner(a)python.org> date: 2022-04-01T20:13:01+02:00 summary: bpo-47089: Avoid test_compileall failures on Windows (GH-32037) files: M Lib/test/test_compileall.py diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 98dab339caa1c..1a9fc973c3aa1 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -460,31 +460,29 @@ def test_error(self): class CommandLineTestsBase: """Test compileall's CLI.""" - @classmethod - def setUpClass(cls): - for path in filter(os.path.isdir, sys.path): - directory_created = False - directory = pathlib.Path(path) / '__pycache__' - path = directory / 'test.try' - try: - if not directory.is_dir(): - directory.mkdir() - directory_created = True - path.write_text('# for test_compileall', encoding="utf-8") - except OSError: - sys_path_writable = False - break - finally: - os_helper.unlink(str(path)) - if directory_created: - directory.rmdir() - else: - sys_path_writable = True - cls._sys_path_writable = sys_path_writable - - def _skip_if_sys_path_not_writable(self): - if not self._sys_path_writable: - raise unittest.SkipTest('not all entries on sys.path are writable') + def setUp(self): + self.directory = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, self.directory) + self.pkgdir = os.path.join(self.directory, 'foo') + os.mkdir(self.pkgdir) + self.pkgdir_cachedir = os.path.join(self.pkgdir, '__pycache__') + # Create the __init__.py and a package module. + self.initfn = script_helper.make_script(self.pkgdir, '__init__', '') + self.barfn = script_helper.make_script(self.pkgdir, 'bar', '') + + @contextlib.contextmanager + def temporary_pycache_prefix(self): + """Adjust and restore sys.pycache_prefix.""" + old_prefix = sys.pycache_prefix + new_prefix = os.path.join(self.directory, '__testcache__') + try: + sys.pycache_prefix = new_prefix + yield { + 'PYTHONPATH': self.directory, + 'PYTHONPYCACHEPREFIX': new_prefix, + } + finally: + sys.pycache_prefix = old_prefix def _get_run_args(self, args): return [*support.optim_args_from_interpreter_flags(), @@ -512,49 +510,39 @@ def assertNotCompiled(self, fn): path = importlib.util.cache_from_source(fn) self.assertFalse(os.path.exists(path)) - def setUp(self): - self.directory = tempfile.mkdtemp() - self.addCleanup(os_helper.rmtree, self.directory) - self.pkgdir = os.path.join(self.directory, 'foo') - os.mkdir(self.pkgdir) - self.pkgdir_cachedir = os.path.join(self.pkgdir, '__pycache__') - # Create the __init__.py and a package module. - self.initfn = script_helper.make_script(self.pkgdir, '__init__', '') - self.barfn = script_helper.make_script(self.pkgdir, 'bar', '') - def test_no_args_compiles_path(self): # Note that -l is implied for the no args case. - self._skip_if_sys_path_not_writable() bazfn = script_helper.make_script(self.directory, 'baz', '') - self.assertRunOK(PYTHONPATH=self.directory) - self.assertCompiled(bazfn) - self.assertNotCompiled(self.initfn) - self.assertNotCompiled(self.barfn) + with self.temporary_pycache_prefix() as env: + self.assertRunOK(**env) + self.assertCompiled(bazfn) + self.assertNotCompiled(self.initfn) + self.assertNotCompiled(self.barfn) @without_source_date_epoch # timestamp invalidation test def test_no_args_respects_force_flag(self): - self._skip_if_sys_path_not_writable() bazfn = script_helper.make_script(self.directory, 'baz', '') - self.assertRunOK(PYTHONPATH=self.directory) - pycpath = importlib.util.cache_from_source(bazfn) + with self.temporary_pycache_prefix() as env: + self.assertRunOK(**env) + pycpath = importlib.util.cache_from_source(bazfn) # Set atime/mtime backward to avoid file timestamp resolution issues os.utime(pycpath, (time.time()-60,)*2) mtime = os.stat(pycpath).st_mtime # Without force, no recompilation - self.assertRunOK(PYTHONPATH=self.directory) + self.assertRunOK(**env) mtime2 = os.stat(pycpath).st_mtime self.assertEqual(mtime, mtime2) # Now force it. - self.assertRunOK('-f', PYTHONPATH=self.directory) + self.assertRunOK('-f', **env) mtime2 = os.stat(pycpath).st_mtime self.assertNotEqual(mtime, mtime2) def test_no_args_respects_quiet_flag(self): - self._skip_if_sys_path_not_writable() script_helper.make_script(self.directory, 'baz', '') - noisy = self.assertRunOK(PYTHONPATH=self.directory) + with self.temporary_pycache_prefix() as env: + noisy = self.assertRunOK(**env) self.assertIn(b'Listing ', noisy) - quiet = self.assertRunOK('-q', PYTHONPATH=self.directory) + quiet = self.assertRunOK('-q', **env) self.assertNotIn(b'Listing ', quiet) # Ensure that the default behavior of compileall's CLI is to create

1 0

bpo-40280: Add debug Emscripten flavors (GH-32233)
by tiran April 1, 2022

April 1, 2022

https://github.com/python/cpython/commit/17245c815e44e79d4dad6a99b54000956a… commit: 17245c815e44e79d4dad6a99b54000956a4a7229 branch: main author: Christian Heimes <christian(a)python.org> committer: tiran <christian(a)python.org> date: 2022-04-01T17:24:00+02:00 summary: bpo-40280: Add debug Emscripten flavors (GH-32233) files: M configure M configure.ac diff --git a/configure b/configure index f261a86a28d1d..f08a01caafdd5 100755 --- a/configure +++ b/configure @@ -6262,6 +6262,10 @@ if test "${with_emscripten_target+set}" = set; then : ac_sys_emscripten_target=browser ;; #( node) : ac_sys_emscripten_target=node ;; #( + browser-debug) : + ac_sys_emscripten_target=browser-debug ;; #( + node-debug) : + ac_sys_emscripten_target=node-debug ;; #( *) : as_fn_error $? "Invalid argument: --with-emscripten-target=browser|node" "$LINENO" 5 ;; @@ -6303,9 +6307,9 @@ esac else case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser) : + Emscripten/browser*) : EXEEXT=.html ;; #( - Emscripten/node) : + Emscripten/node*) : EXEEXT=.js ;; #( WASI/*) : EXEEXT=.wasm ;; #( @@ -6615,7 +6619,7 @@ $as_echo "$LDLIBRARY" >&6; } # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser) : + Emscripten/browser*) : LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB)' ;; #( *) : LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)' @@ -7758,29 +7762,29 @@ fi # The option disables code elimination, which increases code size of main # binary. All objects must be built with -fPIC. case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser) : + Emscripten/browser*) : LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1" LINKFORSHARED="--preload-file \$(WASM_ASSETS_DIR)" WASM_ASSETS_DIR=".\$(prefix)" WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py" - if test "$Py_DEBUG" = 'true'; then + if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "browser-debug"; then LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" LINKFORSHARED="$LINKFORSHARED -gsource-map --emit-symbol-map" else LINKFORSHARED="$LINKFORSHARED -O2 -g0" fi ;; #( - Emscripten/node) : + Emscripten/node*) : LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s USE_PTHREADS=1" LINKFORSHARED="-s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1" CFLAGS_NODIST="$CFLAGS_NODIST -pthread" - if test "$Py_DEBUG" = 'true'; then + if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "node-debug"; then LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" LINKFORSHARED="$LINKFORSHARED -gseparate-dwarf --emit-symbol-map" else - LINKFORSHARED="$LINKFORSHARED -O2 -gseparate-dwarf" + LINKFORSHARED="$LINKFORSHARED -O2 -g0" fi ;; #( WASI/*) : @@ -21672,7 +21676,7 @@ if test "$enable_test_modules" = no; then TEST_MODULES=no else case $ac_sys_system/$ac_sys_emscripten_target in #( - Emscripten/browser) : + Emscripten/browser*) : TEST_MODULES=no ;; #( *) : TEST_MODULES=yes @@ -21731,7 +21735,7 @@ case $ac_sys_system/$ac_sys_emscripten_target in #( py_cv_module__scproxy=n/a py_cv_module_spwd=n/a ;; #( - Emscripten/browser) : + Emscripten/browser*) : @@ -21759,7 +21763,7 @@ case $ac_sys_system/$ac_sys_emscripten_target in #( py_cv_module_=n/a ;; #( - Emscripten/node) : + Emscripten/node*) : diff --git a/configure.ac b/configure.ac index 7e8203ba85d1b..bc3d8b87def9c 100644 --- a/configure.ac +++ b/configure.ac @@ -1091,6 +1091,12 @@ AC_ARG_WITH([emscripten-target], AS_CASE([$with_emscripten_target], [browser], [ac_sys_emscripten_target=browser], [node], [ac_sys_emscripten_target=node], +dnl Debug builds with source map / dwarf symbols. Py_DEBUG builds easily +dnl run out of stack space. Detached sybmols and map prohibit some +dnl optimizations and increase file size. Options are undocumented so we +dnl are free to remove them in the future. + [browser-debug], [ac_sys_emscripten_target=browser-debug], + [node-debug], [ac_sys_emscripten_target=node-debug], [AC_MSG_ERROR([Invalid argument: --with-emscripten-target=browser|node])] ) ], [ @@ -1112,8 +1118,8 @@ AC_ARG_WITH([suffix], ) ], [ AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser], [EXEEXT=.html], - [Emscripten/node], [EXEEXT=.js], + [Emscripten/browser*], [EXEEXT=.html], + [Emscripten/node*], [EXEEXT=.js], [WASI/*], [EXEEXT=.wasm], [EXEEXT=] ) @@ -1376,7 +1382,7 @@ AC_MSG_RESULT($LDLIBRARY) # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser], [LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB)'], + [Emscripten/browser*], [LIBRARY_DEPS='$(PY3LIBRARY) $(WASM_STDLIB)'], [LIBRARY_DEPS='$(PY3LIBRARY) $(EXPORTSYMS)'] ) LINK_PYTHON_DEPS='$(LIBRARY_DEPS)' @@ -1888,28 +1894,28 @@ fi # The option disables code elimination, which increases code size of main # binary. All objects must be built with -fPIC. AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser], [ + [Emscripten/browser*], [ LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1" LINKFORSHARED="--preload-file \$(WASM_ASSETS_DIR)" WASM_ASSETS_DIR=".\$(prefix)" WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py" dnl separate-dwarf does not seem to work in Chrome DevTools Support. - if test "$Py_DEBUG" = 'true'; then + if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "browser-debug"; then LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" LINKFORSHARED="$LINKFORSHARED -gsource-map --emit-symbol-map" else LINKFORSHARED="$LINKFORSHARED -O2 -g0" fi ], - [Emscripten/node], [ + [Emscripten/node*], [ LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s USE_PTHREADS=1" LINKFORSHARED="-s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1" CFLAGS_NODIST="$CFLAGS_NODIST -pthread" - if test "$Py_DEBUG" = 'true'; then + if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "node-debug"; then LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" LINKFORSHARED="$LINKFORSHARED -gseparate-dwarf --emit-symbol-map" else - LINKFORSHARED="$LINKFORSHARED -O2 -gseparate-dwarf" + LINKFORSHARED="$LINKFORSHARED -O2 -g0" fi ], [WASI/*], [ @@ -6453,7 +6459,7 @@ if test "$enable_test_modules" = no; then TEST_MODULES=no else AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser], [TEST_MODULES=no], + [Emscripten/browser*], [TEST_MODULES=no], [TEST_MODULES=yes] ) fi @@ -6478,7 +6484,7 @@ AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], [CYGWIN*/*], [PY_STDLIB_MOD_SET_NA([_scproxy], [nis])], [QNX*/*], [PY_STDLIB_MOD_SET_NA([_scproxy], [nis])], [FreeBSD*/*], [PY_STDLIB_MOD_SET_NA([_scproxy], [spwd])], - [Emscripten/browser], [ + [Emscripten/browser*], [ PY_STDLIB_MOD_SET_NA( [_ctypes], [_curses], @@ -6505,7 +6511,7 @@ AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], ], dnl Some modules like _posixsubprocess do not work. We build them anyway dnl so imports in tests do not fail. - [Emscripten/node], [ + [Emscripten/node*], [ PY_STDLIB_MOD_SET_NA( [_ctypes], [_curses],

1 0

bpo-46023: makesetup: skip all duplicate modules (GH-32234)
by tiran April 1, 2022

April 1, 2022

https://github.com/python/cpython/commit/abdd69c95c1711c2dc75be4e784c6d6c80… commit: abdd69c95c1711c2dc75be4e784c6d6c80a409b9 branch: main author: Christian Heimes <christian(a)python.org> committer: tiran <christian(a)python.org> date: 2022-04-01T17:23:12+02:00 summary: bpo-46023: makesetup: skip all duplicate modules (GH-32234) files: A Misc/NEWS.d/next/Build/2022-04-01-16-12-53.bpo-46023.1Z1OcC.rst M Modules/makesetup diff --git a/Misc/NEWS.d/next/Build/2022-04-01-16-12-53.bpo-46023.1Z1OcC.rst b/Misc/NEWS.d/next/Build/2022-04-01-16-12-53.bpo-46023.1Z1OcC.rst new file mode 100644 index 0000000000000..cb2f7b760e1ec --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-04-01-16-12-53.bpo-46023.1Z1OcC.rst @@ -0,0 +1,2 @@ +``makesetup`` now detects and skips all duplicated module definitions. The +first entry wins. diff --git a/Modules/makesetup b/Modules/makesetup index 3909650ed7c41..9b20e3c9f6344 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -117,6 +117,7 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | BUILT= BUILT_SHARED= DISABLED= + CONFIGURED= MODS= SHAREDMODS= OBJS= @@ -206,12 +207,17 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' | cpps="\$(MODULE_${mods_upper}_CFLAGS)" libs="\$(MODULE_${mods_upper}_LDFLAGS)" fi - case $DISABLED in - *$mods*) - # disabled by previous rule / Setup file - continue - ;; - esac + for mod in $mods + do + case $CONFIGURED in + *,${mod},*) + # Detected multiple rules for a module, first rule wins. This + # allows users to disable modules in Setup.local. + echo 1>&2 "maksetup: '$mod' was handled by previous rule." + continue 2;; + esac + CONFIGURED="$CONFIGURED,${mod}," + done case $doconfig in yes) LIBS="$LIBS $libs"

1 0