Mailman 3 August 2022 - Python-checkins

gh-46845: clean up unused DK_IXSIZE (GH-96405)
by miss-islington Aug. 30, 2022

Aug. 30, 2022

https://github.com/python/cpython/commit/1901ee7a5251a3a2a2d9f65f25205b0b91… commit: 1901ee7a5251a3a2a2d9f65f25205b0b918d5c97 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-08-30T00:28:51-07:00 summary: gh-46845: clean up unused DK_IXSIZE (GH-96405) (cherry picked from commit d21d2f0793ce32d72759d5cfc11622d13e3e6b81) Co-authored-by: Matthias Görgens <matthias.goergens(a)gmail.com> files: M Include/internal/pycore_dict.h diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 24d2a711878c..dc308fe5e218 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -141,17 +141,8 @@ struct _dictvalues { #define DK_LOG_SIZE(dk) ((dk)->dk_log2_size) #if SIZEOF_VOID_P > 4 #define DK_SIZE(dk) (((int64_t)1)<<DK_LOG_SIZE(dk)) -#define DK_IXSIZE(dk) \ - (DK_LOG_SIZE(dk) <= 7 ? \ - 1 : DK_LOG_SIZE(dk) <= 15 ? \ - 2 : DK_LOG_SIZE(dk) <= 31 ? \ - 4 : sizeof(int64_t)) #else #define DK_SIZE(dk) (1<<DK_LOG_SIZE(dk)) -#define DK_IXSIZE(dk) \ - (DK_LOG_SIZE(dk) <= 7 ? \ - 1 : DK_LOG_SIZE(dk) <= 15 ? \ - 2 : sizeof(int32_t)) #endif #define DK_ENTRIES(dk) \ (assert(dk->dk_kind == DICT_KEYS_GENERAL), (PyDictKeyEntry*)(&((int8_t*)((dk)->dk_indices))[(size_t)1 << (dk)->dk_log2_index_bytes]))

1 0

gh-46845: clean up unused DK_IXSIZE (GH-96405)
by methane Aug. 30, 2022

Aug. 30, 2022

https://github.com/python/cpython/commit/d21d2f0793ce32d72759d5cfc11622d13e… commit: d21d2f0793ce32d72759d5cfc11622d13e3e6b81 branch: main author: Matthias Görgens <matthias.goergens(a)gmail.com> committer: methane <songofacandy(a)gmail.com> date: 2022-08-30T16:03:30+09:00 summary: gh-46845: clean up unused DK_IXSIZE (GH-96405) files: M Include/internal/pycore_dict.h diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 5370106d529f..464092996cae 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -141,17 +141,8 @@ struct _dictvalues { #define DK_LOG_SIZE(dk) ((dk)->dk_log2_size) #if SIZEOF_VOID_P > 4 #define DK_SIZE(dk) (((int64_t)1)<<DK_LOG_SIZE(dk)) -#define DK_IXSIZE(dk) \ - (DK_LOG_SIZE(dk) <= 7 ? \ - 1 : DK_LOG_SIZE(dk) <= 15 ? \ - 2 : DK_LOG_SIZE(dk) <= 31 ? \ - 4 : sizeof(int64_t)) #else #define DK_SIZE(dk) (1<<DK_LOG_SIZE(dk)) -#define DK_IXSIZE(dk) \ - (DK_LOG_SIZE(dk) <= 7 ? \ - 1 : DK_LOG_SIZE(dk) <= 15 ? \ - 2 : sizeof(int32_t)) #endif #define DK_ENTRIES(dk) \ (assert((dk)->dk_kind == DICT_KEYS_GENERAL), \

1 0

gh-95853: Improve WASM build script (GH-96389)
by tiran Aug. 30, 2022

Aug. 30, 2022

https://github.com/python/cpython/commit/2eb9008d726e15838a2c8b36c8bd0bf67f… commit: 2eb9008d726e15838a2c8b36c8bd0bf67f229458 branch: main author: Christian Heimes <christian(a)python.org> committer: tiran <christian(a)python.org> date: 2022-08-30T07:56:26+02:00 summary: gh-95853: Improve WASM build script (GH-96389) - pre-build Emscripten ports and system libraries - check for broken EMSDK versions - use EMSDK's node for wasm32-emscripten - warn when PKG_CONFIG_PATH is set - add support level information files: A Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst M Tools/wasm/wasm_build.py diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst new file mode 100644 index 000000000000..1cd1ce14fac0 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-08-29-17-25-13.gh-issue-95853.Ce17cT.rst @@ -0,0 +1,2 @@ +The ``wasm_build.py`` script now pre-builds Emscripten ports, checks for +broken EMSDK versions, and warns about pkg-config env vars. diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index 5ccf88cbc44f..9054370e5e2f 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -25,9 +25,11 @@ import shutil import subprocess import sysconfig +import tempfile +import warnings # for Python 3.8 -from typing import Any, Dict, Callable, Iterable, List, Optional, Union +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() WASMTOOLS = SRCDIR / "Tools" / "wasm" @@ -45,6 +47,11 @@ EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten")) # 3.1.16 has broken utime() EMSDK_MIN_VERSION = (3, 1, 17) +EMSDK_BROKEN_VERSION = { + (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338", + (3, 1, 16): "https://github.com/emscripten-core/emscripten/issues/17393", + (3, 1, 20): "https://github.com/emscripten-core/emscripten/issues/17720", +} _MISSING = pathlib.PurePath("MISSING") # WASM_WEBSERVER = WASMTOOLS / "wasmwebserver.py" @@ -80,24 +87,28 @@ """ -def get_emscripten_root(emconfig: pathlib.Path = EM_CONFIG) -> pathlib.PurePath: - """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT +def parse_emconfig( + emconfig: pathlib.Path = EM_CONFIG, +) -> Tuple[pathlib.PurePath, pathlib.PurePath]: + """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS. The ".emscripten" config file is a Python snippet that uses "EM_CONFIG" environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten" subdirectory with tools like "emconfigure". """ if not emconfig.exists(): - return _MISSING + return _MISSING, _MISSING with open(emconfig, encoding="utf-8") as f: code = f.read() # EM_CONFIG file is a Python snippet local: Dict[str, Any] = {} exec(code, globals(), local) - return pathlib.Path(local["EMSCRIPTEN_ROOT"]) + emscripten_root = pathlib.Path(local["EMSCRIPTEN_ROOT"]) + node_js = pathlib.Path(local["NODE_JS"]) + return emscripten_root, node_js -EMSCRIPTEN_ROOT = get_emscripten_root() +EMSCRIPTEN_ROOT, NODE_JS = parse_emconfig() def read_python_version(configure: pathlib.Path = CONFIGURE) -> str: @@ -153,6 +164,9 @@ class Platform: make_wrapper: Optional[pathlib.PurePath] environ: dict check: Callable[[], None] + # Used for build_emports(). + ports: Optional[pathlib.PurePath] + cc: Optional[pathlib.PurePath] def getenv(self, profile: "BuildProfile") -> dict: return self.environ.copy() @@ -174,6 +188,8 @@ def _check_clean_src(): pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python", config_site=None, configure_wrapper=None, + ports=None, + cc=None, make_wrapper=None, environ={}, check=_check_clean_src, @@ -198,12 +214,26 @@ def _check_emscripten(): version = version[:-4] version_tuple = tuple(int(v) for v in version.split(".")) if version_tuple < EMSDK_MIN_VERSION: - raise MissingDependency( + raise ConditionError( os.fspath(version_txt), f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than " "minimum required version " f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.", ) + broken = EMSDK_BROKEN_VERSION.get(version_tuple) + if broken is not None: + raise ConditionError( + os.fspath(version_txt), + ( + f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known " + f"bugs, see {broken}." + ), + ) + if os.environ.get("PKG_CONFIG_PATH"): + warnings.warn( + "PKG_CONFIG_PATH is set and not empty. emconfigure overrides " + "this environment variable. Use EM_PKG_CONFIG_PATH instead." + ) _check_clean_src() @@ -212,11 +242,14 @@ def _check_emscripten(): pythonexe="python.js", config_site=WASMTOOLS / "config.site-wasm32-emscripten", configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure", + ports=EMSCRIPTEN_ROOT / "embuilder", + cc=EMSCRIPTEN_ROOT / "emcc", make_wrapper=EMSCRIPTEN_ROOT / "emmake", environ={ # workaround for https://github.com/emscripten-core/emscripten/issues/17635 "TZ": "UTC", "EM_COMPILER_WRAPPER": "ccache" if HAS_CCACHE else None, + "PATH": [EMSCRIPTEN_ROOT, os.environ["PATH"]], }, check=_check_emscripten, ) @@ -237,6 +270,8 @@ def _check_wasi(): pythonexe="python.wasm", config_site=WASMTOOLS / "config.site-wasm32-wasi", configure_wrapper=WASMTOOLS / "wasi-env", + ports=None, + cc=WASI_SDK_PATH / "bin" / "clang", make_wrapper=None, environ={ "WASI_SDK_PATH": WASI_SDK_PATH, @@ -246,6 +281,7 @@ def _check_wasi(): "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib " "--mapdir /::{srcdir} --" ), + "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]], }, check=_check_wasi, ) @@ -280,6 +316,42 @@ def is_wasi(self) -> bool: cls = type(self) return self in {cls.wasm32_wasi, cls.wasm64_wasi} + def get_extra_paths(self) -> Iterable[pathlib.PurePath]: + """Host-specific os.environ["PATH"] entries. + + Emscripten's Node version 14.x works well for wasm32-emscripten. + wasm64-emscripten requires more recent v8 version, e.g. node 16.x. + Attempt to use system's node command. + """ + cls = type(self) + if self == cls.wasm32_emscripten: + return [NODE_JS.parent] + elif self == cls.wasm64_emscripten: + # TODO: look for recent node + return [] + else: + return [] + + @property + def emport_args(self) -> List[str]: + """Host-specific port args (Emscripten).""" + cls = type(self) + if self is cls.wasm64_emscripten: + return ["-sMEMORY64=1"] + elif self is cls.wasm32_emscripten: + return ["-sMEMORY64=0"] + else: + return [] + + @property + def embuilder_args(self) -> List[str]: + """Host-specific embuilder args (Emscripten).""" + cls = type(self) + if self is cls.wasm64_emscripten: + return ["--wasm64"] + else: + return [] + class EmscriptenTarget(enum.Enum): """Emscripten-specific targets (--with-emscripten-target)""" @@ -294,10 +366,32 @@ def can_execute(self) -> bool: cls = type(self) return self not in {cls.browser, cls.browser_debug} + @property + def emport_args(self) -> List[str]: + """Target-specific port args.""" + cls = type(self) + if self in {cls.browser_debug, cls.node_debug}: + # some libs come in debug and non-debug builds + return ["-O0"] + else: + return ["-O2"] + + +class SupportLevel(enum.Enum): + supported = "tier 3, supported" + working = "working, unsupported" + experimental = "experimental, may be broken" + broken = "broken / unavailable" + + def __bool__(self): + cls = type(self) + return self in {cls.supported, cls.working} + @dataclasses.dataclass class BuildProfile: name: str + support_level: SupportLevel host: Host target: Union[EmscriptenTarget, None] = None dynamic_linking: Union[bool, None] = None @@ -380,6 +474,12 @@ def getenv(self) -> dict: for key, value in platenv.items(): if value is None: env.pop(key, None) + elif key == "PATH": + # list of path items, prefix with extra paths + new_path: List[pathlib.PurePath] = [] + new_path.extend(self.host.get_extra_paths()) + new_path.extend(value) + env[key] = os.pathsep.join(os.fspath(p) for p in new_path) elif isinstance(value, str): env[key] = value.format( relbuilddir=self.builddir.relative_to(SRCDIR), @@ -390,12 +490,19 @@ def getenv(self) -> dict: env[key] = value return env - def _run_cmd(self, cmd: Iterable[str], args: Iterable[str]): + def _run_cmd( + self, + cmd: Iterable[str], + args: Iterable[str] = (), + cwd: Optional[pathlib.Path] = None, + ): cmd = list(cmd) cmd.extend(args) + if cwd is None: + cwd = self.builddir return subprocess.check_call( cmd, - cwd=os.fspath(self.builddir), + cwd=os.fspath(cwd), env=self.getenv(), ) @@ -443,10 +550,58 @@ def clean(self, all: bool = False): elif self.makefile.exists(): self.run_make("clean") + def build_emports(self, force: bool = False): + """Pre-build emscripten ports.""" + platform = self.host.platform + if platform.ports is None or platform.cc is None: + raise ValueError("Need ports and CC command") + + embuilder_cmd = [os.fspath(platform.ports)] + embuilder_cmd.extend(self.host.embuilder_args) + if force: + embuilder_cmd.append("--force") + + ports_cmd = [os.fspath(platform.cc)] + ports_cmd.extend(self.host.emport_args) + if self.target: + ports_cmd.extend(self.target.emport_args) + + if self.dynamic_linking: + # Trigger PIC build. + ports_cmd.append("-sMAIN_MODULE") + embuilder_cmd.append("--pic") + if self.pthreads: + # Trigger multi-threaded build. + ports_cmd.append("-sUSE_PTHREADS") + # https://github.com/emscripten-core/emscripten/pull/17729 + # embuilder_cmd.append("--pthreads") + + # Pre-build libbz2, libsqlite3, libz, and some system libs. + ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"]) + embuilder_cmd.extend(["build", "bzip2", "sqlite3", "zlib"]) + + if not self.pthreads: + # Emscripten <= 3.1.20 has no option to build multi-threaded ports. + self._run_cmd(embuilder_cmd, cwd=SRCDIR) + + with tempfile.TemporaryDirectory(suffix="-py-emport") as tmpdir: + tmppath = pathlib.Path(tmpdir) + main_c = tmppath / "main.c" + main_js = tmppath / "main.js" + with main_c.open("w") as f: + f.write("int main(void) { return 0; }\n") + args = [ + os.fspath(main_c), + "-o", + os.fspath(main_js), + ] + self._run_cmd(ports_cmd, args, cwd=tmppath) + # native build (build Python) BUILD = BuildProfile( "build", + support_level=SupportLevel.working, host=Host.build, ) @@ -455,43 +610,59 @@ def clean(self, all: bool = False): # wasm32-emscripten BuildProfile( "emscripten-browser", + support_level=SupportLevel.supported, host=Host.wasm32_emscripten, target=EmscriptenTarget.browser, dynamic_linking=True, ), BuildProfile( "emscripten-browser-debug", + support_level=SupportLevel.working, host=Host.wasm32_emscripten, target=EmscriptenTarget.browser_debug, dynamic_linking=True, ), BuildProfile( "emscripten-node-dl", + support_level=SupportLevel.supported, host=Host.wasm32_emscripten, target=EmscriptenTarget.node, dynamic_linking=True, ), BuildProfile( "emscripten-node-dl-debug", + support_level=SupportLevel.working, host=Host.wasm32_emscripten, target=EmscriptenTarget.node_debug, dynamic_linking=True, ), BuildProfile( "emscripten-node-pthreads", + support_level=SupportLevel.supported, host=Host.wasm32_emscripten, target=EmscriptenTarget.node, pthreads=True, ), BuildProfile( "emscripten-node-pthreads-debug", + support_level=SupportLevel.working, + host=Host.wasm32_emscripten, + target=EmscriptenTarget.node_debug, + pthreads=True, + ), + # Emscripten build with both pthreads and dynamic linking is crashing. + BuildProfile( + "emscripten-node-dl-pthreads-debug", + support_level=SupportLevel.broken, host=Host.wasm32_emscripten, target=EmscriptenTarget.node_debug, + dynamic_linking=True, pthreads=True, ), - # wasm64-emscripten (currently not working) + # wasm64-emscripten (requires unreleased Emscripten >= 3.1.21) BuildProfile( "wasm64-emscripten-node-debug", + support_level=SupportLevel.experimental, host=Host.wasm64_emscripten, target=EmscriptenTarget.node_debug, # MEMORY64 is not compatible with dynamic linking @@ -501,6 +672,7 @@ def clean(self, all: bool = False): # wasm32-wasi BuildProfile( "wasi", + support_level=SupportLevel.supported, host=Host.wasm32_wasi, # skip sysconfig test_srcdir testopts="-i '*.test_srcdir' -j2", @@ -508,6 +680,7 @@ def clean(self, all: bool = False): # no SDK available yet # BuildProfile( # "wasm64-wasi", + # support_level=SupportLevel.broken, # host=Host.wasm64_wasi, # ), ] @@ -523,15 +696,17 @@ def clean(self, all: bool = False): "--clean", "-c", help="Clean build directories first", action="store_true" ) -platforms = list(PROFILES) + ["cleanall"] +# Don't list broken and experimental variants in help +platforms_choices = list(p.name for p in _profiles) + ["cleanall"] +platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"] parser.add_argument( "platform", metavar="PLATFORM", - help=f"Build platform: {', '.join(platforms)}", - choices=platforms, + help=f"Build platform: {', '.join(platforms_help)}", + choices=platforms_choices, ) -ops = ["compile", "pythoninfo", "test", "repl", "clean", "cleanall"] +ops = ["compile", "pythoninfo", "test", "repl", "clean", "cleanall", "emports"] parser.add_argument( "op", metavar="OP", @@ -572,6 +747,8 @@ def main(): builder.clean(all=False) if args.op == "compile": + if builder.host.is_emscripten: + builder.build_emports() builder.run_build(force_configure=True) else: if not builder.makefile.exists(): @@ -586,6 +763,8 @@ def main(): builder.clean(all=False) elif args.op == "cleanall": builder.clean(all=True) + elif args.op == "emports": + builder.build_emports(force=args.clean) else: raise ValueError(args.op)

1 0

gh-96320: WASI socket fixes (#96388)
by tiran Aug. 30, 2022

Aug. 30, 2022

https://github.com/python/cpython/commit/d0b3d235dbc560ada459e386b6fa5b5ce1… commit: d0b3d235dbc560ada459e386b6fa5b5ce1952c7e branch: main author: Christian Heimes <christian(a)python.org> committer: tiran <christian(a)python.org> date: 2022-08-30T06:36:11+02:00 summary: gh-96320: WASI socket fixes (#96388) * gh-96320: WASI socket fixes - ignore missing functions in ``socket.__repr__`` - bundle network files with assets * blurb files: A Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst M Lib/socket.py M Tools/wasm/wasm_assets.py diff --git a/Lib/socket.py b/Lib/socket.py index 0ed86afb4a9e..1c8cef6ce658 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -255,17 +255,18 @@ def __repr__(self): self.type, self.proto) if not closed: + # getsockname and getpeername may not be available on WASI. try: laddr = self.getsockname() if laddr: s += ", laddr=%s" % str(laddr) - except error: + except (error, AttributeError): pass try: raddr = self.getpeername() if raddr: s += ", raddr=%s" % str(raddr) - except error: + except (error, AttributeError): pass s += '>' return s diff --git a/Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst b/Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst new file mode 100644 index 000000000000..3a35c4734871 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-08-29-16-54-36.gh-issue-96388.dCpJcu.rst @@ -0,0 +1,2 @@ +Work around missing socket functions in :class:`~socket.socket`'s +``__repr__``. diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/wasm_assets.py index a300d594414a..a35acfd9387b 100755 --- a/Tools/wasm/wasm_assets.py +++ b/Tools/wasm/wasm_assets.py @@ -230,7 +230,8 @@ def main(): extmods = detect_extension_modules(args) omit_files = list(OMIT_FILES) - omit_files.extend(OMIT_NETWORKING_FILES) + if sysconfig.get_platform().startswith("emscripten"): + omit_files.extend(OMIT_NETWORKING_FILES) for modname, modfiles in OMIT_MODULE_FILES.items(): if not extmods.get(modname): omit_files.extend(modfiles)

1 0

Docs: normalize SQL style in sqlite3 docs (GH-96403)
by miss-islington Aug. 29, 2022

Aug. 29, 2022

https://github.com/python/cpython/commit/4cfb6395e18b4846e92c9f685d534e6ec1… commit: 4cfb6395e18b4846e92c9f685d534e6ec17822a2 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com> date: 2022-08-29T15:59:13-07:00 summary: Docs: normalize SQL style in sqlite3 docs (GH-96403) (cherry picked from commit 6d403e264a7dcd1544a91708f139c6dd8612204d) Co-authored-by: Erlend E. Aasland <erlend.aasland(a)protonmail.com> files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 50499b001e2..21ad3540ccb 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -384,7 +384,7 @@ Module functions ... print(f"Error message: {unraisable.err_msg}") >>> import sys >>> sys.unraisablehook = debug - >>> cur = con.execute("select 1") + >>> cur = con.execute("SELECT 1") ZeroDivisionError('division by zero') in callback evil_trace Error message: None @@ -1206,7 +1206,7 @@ Cursor objects ("row2",), ] # cur is an sqlite3.Cursor object - cur.executemany("insert into data values(?)", rows) + cur.executemany("INSERT INTO data VALUES(?)", rows) .. method:: executescript(sql_script, /) @@ -1224,11 +1224,11 @@ Cursor objects # cur is an sqlite3.Cursor object cur.executescript(""" - begin; - create table person(firstname, lastname, age); - create table book(title, author, published); - create table publisher(name, address); - commit; + BEGIN; + CREATE TABLE person(firstname, lastname, age); + CREATE TABLE book(title, author, published); + CREATE TABLE publisher(name, address); + COMMIT; """)

1 0

Docs: normalize SQL style in sqlite3 docs (#96403)
by erlend-aasland Aug. 29, 2022

Aug. 29, 2022

https://github.com/python/cpython/commit/6d403e264a7dcd1544a91708f139c6dd86… commit: 6d403e264a7dcd1544a91708f139c6dd8612204d branch: main author: Erlend E. Aasland <erlend.aasland(a)protonmail.com> committer: erlend-aasland <erlend.aasland(a)protonmail.com> date: 2022-08-30T00:50:42+02:00 summary: Docs: normalize SQL style in sqlite3 docs (#96403) files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index f6a86e69f1e..31a0e7f0b53 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -387,7 +387,7 @@ Module functions ... print(f"Error message: {unraisable.err_msg}") >>> import sys >>> sys.unraisablehook = debug - >>> cur = con.execute("select 1") + >>> cur = con.execute("SELECT 1") ZeroDivisionError('division by zero') in callback evil_trace Error message: None @@ -1219,7 +1219,7 @@ Cursor objects ("row2",), ] # cur is an sqlite3.Cursor object - cur.executemany("insert into data values(?)", rows) + cur.executemany("INSERT INTO data VALUES(?)", rows) .. method:: executescript(sql_script, /) @@ -1237,11 +1237,11 @@ Cursor objects # cur is an sqlite3.Cursor object cur.executescript(""" - begin; - create table person(firstname, lastname, age); - create table book(title, author, published); - create table publisher(name, address); - commit; + BEGIN; + CREATE TABLE person(firstname, lastname, age); + CREATE TABLE book(title, author, published); + CREATE TABLE publisher(name, address); + COMMIT; """)

1 0

gh-91305: Add a note about DatagramHandler and DNS latency. (GH-96380)
by vsajip Aug. 29, 2022

Aug. 29, 2022

https://github.com/python/cpython/commit/6324b135acb92042017c2364540c37426b… commit: 6324b135acb92042017c2364540c37426b0ea65f branch: main author: Vinay Sajip <vinay_sajip(a)yahoo.co.uk> committer: vsajip <vinay_sajip(a)yahoo.co.uk> date: 2022-08-29T22:29:22+01:00 summary: gh-91305: Add a note about DatagramHandler and DNS latency. (GH-96380) files: M Doc/library/logging.handlers.rst diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index f125dfe64a03..a0129279f4db 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -572,6 +572,13 @@ over UDP sockets. Returns a new instance of the :class:`DatagramHandler` class intended to communicate with a remote machine whose address is given by *host* and *port*. + .. note:: As UDP is not a streaming protocol, there is no persistent connection + between an instance of this handler and *host*. For this reason, when using a + network socket, a DNS lookup might have to be made each time an event is + logged, which can introduce some latency into the system. If this affects you, + you can do a lookup yourself and initialize this handler using the looked-up IP + address rather than the hostname. + .. versionchanged:: 3.4 If ``port`` is specified as ``None``, a Unix domain socket is created using the value in ``host`` - otherwise, a UDP socket is created.

1 0

GH-74116: Allow multiple drain waiters for asyncio.StreamWriter (GH-94705)
by gvanrossum Aug. 29, 2022

Aug. 29, 2022

https://github.com/python/cpython/commit/e5b2453e61ba5376831093236d598ef5f9… commit: e5b2453e61ba5376831093236d598ef5f9f1de61 branch: main author: Kumar Aditya <59607654+kumaraditya303(a)users.noreply.github.com> committer: gvanrossum <gvanrossum(a)gmail.com> date: 2022-08-29T11:31:11-07:00 summary: GH-74116: Allow multiple drain waiters for asyncio.StreamWriter (GH-94705) files: A Misc/NEWS.d/next/Library/2022-07-09-08-55-04.gh-issue-74116.0XwYC1.rst M Lib/asyncio/streams.py M Lib/test/test_asyncio/test_streams.py diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 614b2cda6068..c4d837a11708 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -2,6 +2,7 @@ 'StreamReader', 'StreamWriter', 'StreamReaderProtocol', 'open_connection', 'start_server') +import collections import socket import sys import weakref @@ -128,7 +129,7 @@ def __init__(self, loop=None): else: self._loop = loop self._paused = False - self._drain_waiter = None + self._drain_waiters = collections.deque() self._connection_lost = False def pause_writing(self): @@ -143,38 +144,34 @@ def resume_writing(self): if self._loop.get_debug(): logger.debug("%r resumes writing", self) - waiter = self._drain_waiter - if waiter is not None: - self._drain_waiter = None + for waiter in self._drain_waiters: if not waiter.done(): waiter.set_result(None) def connection_lost(self, exc): self._connection_lost = True - # Wake up the writer if currently paused. + # Wake up the writer(s) if currently paused. if not self._paused: return - waiter = self._drain_waiter - if waiter is None: - return - self._drain_waiter = None - if waiter.done(): - return - if exc is None: - waiter.set_result(None) - else: - waiter.set_exception(exc) + + for waiter in self._drain_waiters: + if not waiter.done(): + if exc is None: + waiter.set_result(None) + else: + waiter.set_exception(exc) async def _drain_helper(self): if self._connection_lost: raise ConnectionResetError('Connection lost') if not self._paused: return - waiter = self._drain_waiter - assert waiter is None or waiter.cancelled() waiter = self._loop.create_future() - self._drain_waiter = waiter - await waiter + self._drain_waiters.append(waiter) + try: + await waiter + finally: + self._drain_waiters.remove(waiter) def _get_close_waiter(self, stream): raise NotImplementedError diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 098a0da344d0..0c49099bc499 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -864,6 +864,25 @@ def test_streamreaderprotocol_constructor_use_global_loop(self): self.assertEqual(cm.filename, __file__) self.assertIs(protocol._loop, self.loop) + def test_multiple_drain(self): + # See https://github.com/python/cpython/issues/74116 + drained = 0 + + async def drainer(stream): + nonlocal drained + await stream._drain_helper() + drained += 1 + + async def main(): + loop = asyncio.get_running_loop() + stream = asyncio.streams.FlowControlMixin(loop) + stream.pause_writing() + loop.call_later(0.1, stream.resume_writing) + await asyncio.gather(*[drainer(stream) for _ in range(10)]) + self.assertEqual(drained, 10) + + self.loop.run_until_complete(main()) + def test_drain_raises(self): # See http://bugs.python.org/issue25441 diff --git a/Misc/NEWS.d/next/Library/2022-07-09-08-55-04.gh-issue-74116.0XwYC1.rst b/Misc/NEWS.d/next/Library/2022-07-09-08-55-04.gh-issue-74116.0XwYC1.rst new file mode 100644 index 000000000000..33782598745b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-07-09-08-55-04.gh-issue-74116.0XwYC1.rst @@ -0,0 +1 @@ +Allow :meth:`asyncio.StreamWriter.drain` to be awaited concurrently by multiple tasks. Patch by Kumar Aditya.

1 0

[3.10] gh-94682: Build and test with OpenSSL 1.1.1q (gh-94683) (gh-96391)
by tiran Aug. 29, 2022

Aug. 29, 2022

https://github.com/python/cpython/commit/32a45011e72f9c5c0a61438ec295d2793a… commit: 32a45011e72f9c5c0a61438ec295d2793af51ba9 branch: 3.10 author: Christian Heimes <christian(a)python.org> committer: tiran <christian(a)python.org> date: 2022-08-29T20:02:00+02:00 summary: [3.10] gh-94682: Build and test with OpenSSL 1.1.1q (gh-94683) (gh-96391) Co-authored-by: Christian Heimes <christian(a)python.org> files: A Misc/NEWS.d/next/Build/2022-07-08-10-28-23.gh-issue-94682.ZtGt_0.rst M .azure-pipelines/ci.yml M .azure-pipelines/pr.yml M .github/workflows/build.yml M Mac/BuildScript/build-installer.py M PCbuild/get_externals.bat M PCbuild/python.props M PCbuild/readme.txt M Tools/ssl/multissltests.py diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index d0efa77f9305..8dd79eeccdb0 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -57,7 +57,7 @@ jobs: variables: testRunTitle: '$(build.sourceBranchName)-linux' testRunPlatform: linux - openssl_version: 1.1.1n + openssl_version: 1.1.1q steps: - template: ./posix-steps.yml @@ -83,7 +83,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1n + openssl_version: 1.1.1q steps: - template: ./posix-steps.yml diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index a4f32460c7ea..e1b282be129d 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -57,7 +57,7 @@ jobs: variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.1n + openssl_version: 1.1.1q steps: - template: ./posix-steps.yml @@ -83,7 +83,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1n + openssl_version: 1.1.1q steps: - template: ./posix-steps.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b98514aaf945..1545ae87dca0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -190,7 +190,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1n + OPENSSL_VER: 1.1.1q PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v3 @@ -234,7 +234,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1n, 3.0.2] + openssl_ver: [1.1.1q, 3.0.5] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 4308a2003ce7..668667c00729 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.1.1n", - url="https://www.openssl.org/source/openssl-1.1.1n.tar.gz", - checksum='2aad5635f9bb338bc2c6b7d19cbc9676', + name="OpenSSL 1.1.1q", + url="https://www.openssl.org/source/openssl-1.1.1q.tar.gz", + checksum='d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca', buildrecipe=build_universal_openssl, configure=None, install=None, @@ -793,10 +793,16 @@ def verifyThirdPartyFile(url, checksum, fname): print("Downloading %s"%(name,)) downloadURL(url, fname) print("Archive for %s stored as %s"%(name, fname)) + if len(checksum) == 32: + algo = 'md5' + elif len(checksum) == 64: + algo = 'sha256' + else: + raise ValueError(checksum) if os.system( - 'MD5=$(openssl md5 %s) ; test "${MD5##*= }" = "%s"' - % (shellQuote(fname), checksum) ): - fatal('MD5 checksum mismatch for file %s' % fname) + 'CHECKSUM=$(openssl %s %s) ; test "${CHECKSUM##*= }" = "%s"' + % (algo, shellQuote(fname), checksum) ): + fatal('%s checksum mismatch for file %s' % (algo, fname)) def build_universal_openssl(basedir, archList): """ diff --git a/Misc/NEWS.d/next/Build/2022-07-08-10-28-23.gh-issue-94682.ZtGt_0.rst b/Misc/NEWS.d/next/Build/2022-07-08-10-28-23.gh-issue-94682.ZtGt_0.rst new file mode 100644 index 000000000000..60717a15bcc2 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-07-08-10-28-23.gh-issue-94682.ZtGt_0.rst @@ -0,0 +1 @@ +Build and test with OpenSSL 1.1.1q diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index ee79addd44d0..208b573562dc 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -53,7 +53,7 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.3.0 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1n +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1q set libraries=%libraries% sqlite-3.37.2.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.12.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.12.0 @@ -77,7 +77,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.3.0 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1n +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.1q if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.12.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 diff --git a/PCbuild/python.props b/PCbuild/python.props index 859f9db2a8fc..b3604b19a10d 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -63,8 +63,8 @@ <libffiDir>$(ExternalsDir)libffi-3.3.0\</libffiDir> <libffiOutDir>$(ExternalsDir)libffi-3.3.0\$(ArchName)\</libffiOutDir> <libffiIncludeDir>$(libffiOutDir)include</libffiIncludeDir> - <opensslDir>$(ExternalsDir)openssl-1.1.1n\</opensslDir> - <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.1n\$(ArchName)\</opensslOutDir> + <opensslDir>$(ExternalsDir)openssl-1.1.1q\</opensslDir> + <opensslOutDir>$(ExternalsDir)openssl-bin-1.1.1q\$(ArchName)\</opensslOutDir> <opensslIncludeDir>$(opensslOutDir)include</opensslIncludeDir> <nasmDir>$(ExternalsDir)\nasm-2.11.06\</nasmDir> <zlibDir>$(ExternalsDir)\zlib-1.2.12\</zlibDir> diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 76f421a9aaee..3af4c5c7da18 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -167,7 +167,7 @@ _lzma Homepage: https://tukaani.org/xz/ _ssl - Python wrapper for version 1.1.1k of the OpenSSL secure sockets + Python wrapper for version 1.1.1q of the OpenSSL secure sockets library, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index 82076808bfd3..91d6f558bc51 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -47,8 +47,8 @@ ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1n", - "3.0.2" + "1.1.1q", + "3.0.5" ] LIBRESSL_OLD_VERSIONS = [

1 0

Improve accuracy for Spearman's rank correlation coefficient. (#96392)
by rhettinger Aug. 29, 2022

Aug. 29, 2022

https://github.com/python/cpython/commit/3d180e3ab21c5d41d1c46e3ef349b30ba4… commit: 3d180e3ab21c5d41d1c46e3ef349b30ba409f300 branch: main author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: rhettinger <rhettinger(a)users.noreply.github.com> date: 2022-08-29T12:19:48-05:00 summary: Improve accuracy for Spearman's rank correlation coefficient. (#96392) files: M Lib/statistics.py diff --git a/Lib/statistics.py b/Lib/statistics.py index b4676fed5e2..b4adabd3f05 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -379,7 +379,7 @@ def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[ [2.0, 1.0, 3.0] Ranks are conventionally numbered starting from one; however, - setting *start* to zero allow the ranks to be used as array indices: + setting *start* to zero allows the ranks to be used as array indices: >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] >>> scores = [8.1, 7.3, 9.4, 8.3] @@ -1073,8 +1073,9 @@ def correlation(x, y, /, *, method='linear'): if method not in {'linear', 'ranked'}: raise ValueError(f'Unknown method: {method!r}') if method == 'ranked': - x = _rank(x) - y = _rank(y) + start = (n - 1) / -2 # Center rankings around zero + x = _rank(x, start=start) + y = _rank(y, start=start) xbar = fsum(x) / n ybar = fsum(y) / n sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y))

1 0