Mailman 3 March 2019 - Python-checkins

bpo-36146: Refactor setup.py: Add PyBuildExt.srcdir (GH-12124)
by Victor Stinner March 1, 2019

March 1, 2019

https://github.com/python/cpython/commit/625dbf2567533e6001d57e5969fba75c1b… commit: 625dbf2567533e6001d57e5969fba75c1b6ece43 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2019-03-01T15:59:39+01:00 summary: bpo-36146: Refactor setup.py: Add PyBuildExt.srcdir (GH-12124) * Add PyBuildExt.srcdir atribute in setup.py: the source directory is now always absolute. * Add PyBuildExt.inc_dirs and PyBuildExt.lib_dirs attributes: replace 'inc_dirs' and 'lib_dirs' local variables of detect_modules(). * Replace "from distutils.errors import *" with "from distutils.errors import CCompilerError, DistutilsError" to be able to use static analyzers like pyflakes * Reorder imports. files: M setup.py diff --git a/setup.py b/setup.py index db77772da925..9001f62cd33e 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,23 @@ # Autodetecting setup.py script for building the Python extensions # -import sys, os, importlib.machinery, re, argparse -from glob import glob +import argparse import importlib._bootstrap +import importlib.machinery import importlib.util +import os +import re +import sys import sysconfig +from glob import glob from distutils import log -from distutils.errors import * -from distutils.core import Extension, setup from distutils.command.build_ext import build_ext +from distutils.command.build_scripts import build_scripts from distutils.command.install import install from distutils.command.install_lib import install_lib -from distutils.command.build_scripts import build_scripts +from distutils.core import Extension, setup +from distutils.errors import CCompilerError, DistutilsError from distutils.spawn import find_executable CROSS_COMPILING = "_PYTHON_HOST_PLATFORM" in os.environ @@ -227,6 +231,9 @@ class PyBuildExt(build_ext): def __init__(self, dist): build_ext.__init__(self, dist) + self.srcdir = None + self.lib_dirs = None + self.inc_dirs = None self.failed = [] self.failed_on_import = [] self.missing = [] @@ -237,6 +244,11 @@ def add(self, ext): self.extensions.append(ext) def build_extensions(self): + self.srcdir = sysconfig.get_config_var('srcdir') + if not self.srcdir: + # Maybe running on Windows but not using CYGWIN? + raise ValueError("No source directory; cannot proceed.") + self.srcdir = os.path.abspath(self.srcdir) # Detect which modules should be compiled self.detect_modules() @@ -253,15 +265,10 @@ def build_extensions(self): # Fix up the autodetected modules, prefixing all the source files # with Modules/. - srcdir = sysconfig.get_config_var('srcdir') - if not srcdir: - # Maybe running on Windows but not using CYGWIN? - raise ValueError("No source directory; cannot proceed.") - srcdir = os.path.abspath(srcdir) - moddirlist = [os.path.join(srcdir, 'Modules')] + moddirlist = [os.path.join(self.srcdir, 'Modules')] # Fix up the paths for scripts, too - self.distribution.scripts = [os.path.join(srcdir, filename) + self.distribution.scripts = [os.path.join(self.srcdir, filename) for filename in self.distribution.scripts] # Python header files @@ -398,7 +405,7 @@ def build_extension(self, ext): build_ext.build_extension(self, ext) except (CCompilerError, DistutilsError) as why: self.announce('WARNING: building of extension "%s" failed: %s' % - (ext.name, sys.exc_info()[1])) + (ext.name, why)) self.failed.append(ext.name) return @@ -527,8 +534,6 @@ def add_cross_compiling_paths(self): is_gcc = False is_clang = False in_incdirs = False - inc_dirs = [] - lib_dirs = [] try: if ret >> 8 == 0: with open(tmpfile) as fp: @@ -602,31 +607,29 @@ def detect_modules(self): # if a file is found in one of those directories, it can # be assumed that no additional -I,-L directives are needed. if not CROSS_COMPILING: - lib_dirs = self.compiler.library_dirs + system_lib_dirs - inc_dirs = self.compiler.include_dirs + system_include_dirs + self.lib_dirs = self.compiler.library_dirs + system_lib_dirs + self.inc_dirs = self.compiler.include_dirs + system_include_dirs else: # Add the sysroot paths. 'sysroot' is a compiler option used to # set the logical path of the standard system headers and # libraries. - lib_dirs = (self.compiler.library_dirs + - sysroot_paths(('LDFLAGS', 'CC'), system_lib_dirs)) - inc_dirs = (self.compiler.include_dirs + - sysroot_paths(('CPPFLAGS', 'CFLAGS', 'CC'), - system_include_dirs)) + self.lib_dirs = (self.compiler.library_dirs + + sysroot_paths(('LDFLAGS', 'CC'), system_lib_dirs)) + self.inc_dirs = (self.compiler.include_dirs + + sysroot_paths(('CPPFLAGS', 'CFLAGS', 'CC'), + system_include_dirs)) config_h = sysconfig.get_config_h_filename() with open(config_h) as file: config_h_vars = sysconfig.parse_config_h(file) - srcdir = sysconfig.get_config_var('srcdir') - # OSF/1 and Unixware have some stuff in /usr/ccs/lib (like -ldb) if HOST_PLATFORM in ['osf1', 'unixware7', 'openunix8']: - lib_dirs += ['/usr/ccs/lib'] + self.lib_dirs += ['/usr/ccs/lib'] # HP-UX11iv3 keeps files in lib/hpux folders. if HOST_PLATFORM == 'hp-ux11': - lib_dirs += ['/usr/lib/hpux64', '/usr/lib/hpux32'] + self.lib_dirs += ['/usr/lib/hpux64', '/usr/lib/hpux32'] if MACOS: # This should work on any unixy platform ;-) @@ -640,11 +643,11 @@ def detect_modules(self): 'CFLAGS', 'LDFLAGS') for item in cflags.split(): if item.startswith('-I'): - inc_dirs.append(item[2:]) + self.inc_dirs.append(item[2:]) for item in ldflags.split(): if item.startswith('-L'): - lib_dirs.append(item[2:]) + self.lib_dirs.append(item[2:]) # # The following modules are all pretty straightforward, and compile @@ -783,7 +786,7 @@ def detect_modules(self): libraries=['m'])) # readline - do_readline = self.compiler.find_library_file(lib_dirs, 'readline') + do_readline = self.compiler.find_library_file(self.lib_dirs, 'readline') readline_termcap_library = "" curses_library = "" # Cannot use os.popen here in py3k. @@ -818,11 +821,11 @@ def detect_modules(self): # use the same library for the readline and curses modules. if 'curses' in readline_termcap_library: curses_library = readline_termcap_library - elif self.compiler.find_library_file(lib_dirs, 'ncursesw'): + elif self.compiler.find_library_file(self.lib_dirs, 'ncursesw'): curses_library = 'ncursesw' - elif self.compiler.find_library_file(lib_dirs, 'ncurses'): + elif self.compiler.find_library_file(self.lib_dirs, 'ncurses'): curses_library = 'ncurses' - elif self.compiler.find_library_file(lib_dirs, 'curses'): + elif self.compiler.find_library_file(self.lib_dirs, 'curses'): curses_library = 'curses' if MACOS: @@ -836,7 +839,7 @@ def detect_modules(self): # MacOSX 10.4 has a broken readline. Don't try to build # the readline module unless the user has installed a fixed # readline package - if find_file('readline/rlconf.h', inc_dirs, []) is None: + if find_file('readline/rlconf.h', self.inc_dirs, []) is None: do_readline = False if do_readline: if MACOS and os_release < 9: @@ -854,7 +857,7 @@ def detect_modules(self): pass # Issue 7384: Already linked against curses or tinfo. elif curses_library: readline_libs.append(curses_library) - elif self.compiler.find_library_file(lib_dirs + + elif self.compiler.find_library_file(self.lib_dirs + ['/usr/lib/termcap'], 'termcap'): readline_libs.append('termcap') @@ -867,7 +870,7 @@ def detect_modules(self): # crypt module. - if self.compiler.find_library_file(lib_dirs, 'crypt'): + if self.compiler.find_library_file(self.lib_dirs, 'crypt'): libs = ['crypt'] else: libs = [] @@ -875,7 +878,7 @@ def detect_modules(self): if not VXWORKS: self.add(Extension('_crypt', ['_cryptmodule.c'], libraries=libs)) - elif self.compiler.find_library_file(lib_dirs, 'OPENSSL'): + elif self.compiler.find_library_file(self.lib_dirs, 'OPENSSL'): libs = ['OPENSSL'] self.add(Extension('_crypt', ['_cryptmodule.c'], libraries=libs)) @@ -890,14 +893,14 @@ def detect_modules(self): if not VXWORKS: self.add(Extension('_socket', ['socketmodule.c'], depends=['socketmodule.h'])) - elif self.compiler.find_library_file(lib_dirs, 'net'): + elif self.compiler.find_library_file(self.lib_dirs, 'net'): libs = ['net'] self.add(Extension('_socket', ['socketmodule.c'], depends=['socketmodule.h'], libraries=libs)) # Detect SSL support for the socket module (via _ssl) - self._detect_openssl(inc_dirs, lib_dirs) + self._detect_openssl() # We always compile these even when OpenSSL is available (issue #14693). # It's harmless and the object code is tiny (40-50 KiB per module, @@ -911,7 +914,7 @@ def detect_modules(self): self.add(Extension('_sha1', ['sha1module.c'], depends=['hashlib.h'])) - blake2_deps = glob(os.path.join(os.getcwd(), srcdir, + blake2_deps = glob(os.path.join(self.srcdir, 'Modules/_blake2/impl/*')) blake2_deps.append('hashlib.h') @@ -921,7 +924,7 @@ def detect_modules(self): '_blake2/blake2s_impl.c'], depends=blake2_deps)) - sha3_deps = glob(os.path.join(os.getcwd(), srcdir, + sha3_deps = glob(os.path.join(self.srcdir, 'Modules/_sha3/kcp/*')) sha3_deps.append('hashlib.h') self.add(Extension('_sha3', @@ -1008,7 +1011,7 @@ def gen_db_minor_ver_nums(major): # picked up when it is installed in a non-standard prefix and # the user has added that prefix into inc_dirs. std_variants = [] - for dn in inc_dirs: + for dn in self.inc_dirs: std_variants.append(os.path.join(dn, 'db3')) std_variants.append(os.path.join(dn, 'db4')) for x in gen_db_minor_ver_nums(4): @@ -1030,7 +1033,7 @@ class db_found(Exception): pass try: # See whether there is a Sleepycat header in the standard # search path. - for d in inc_dirs + db_inc_paths: + for d in self.inc_dirs + db_inc_paths: f = os.path.join(d, "db.h") if MACOS and is_macosx_sdk_path(d): f = os.path.join(sysroot, d[1:], "db.h") @@ -1108,7 +1111,7 @@ class db_found(Exception): pass ('db%d%d' % db_ver), ('db%d' % db_ver[0])): dblib_file = self.compiler.find_library_file( - db_dirs_to_check + lib_dirs, dblib ) + db_dirs_to_check + self.lib_dirs, dblib ) if dblib_file: dblib_dir = [ os.path.abspath(os.path.dirname(dblib_file)) ] raise db_found @@ -1123,11 +1126,11 @@ class db_found(Exception): pass # Only add the found library and include directories if they aren't # already being searched. This avoids an explicit runtime library # dependency. - if db_incdir in inc_dirs: + if db_incdir in self.inc_dirs: db_incs = None else: db_incs = [db_incdir] - if dblib_dir[0] in lib_dirs: + if dblib_dir[0] in self.lib_dirs: dblib_dir = None else: if db_setup_debug: print("db: no appropriate library found") @@ -1160,7 +1163,7 @@ class db_found(Exception): pass if MACOS: sysroot = macosx_sdk_root() - for d_ in inc_dirs + sqlite_inc_paths: + for d_ in self.inc_dirs + sqlite_inc_paths: d = d_ if MACOS and is_macosx_sdk_path(d): d = os.path.join(sysroot, d[1:]) @@ -1197,7 +1200,7 @@ class db_found(Exception): pass os.path.join(sqlite_incdir, '..', '..', 'lib'), ] sqlite_libfile = self.compiler.find_library_file( - sqlite_dirs_to_check + lib_dirs, 'sqlite3') + sqlite_dirs_to_check + self.lib_dirs, 'sqlite3') if sqlite_libfile: sqlite_libdir = [os.path.abspath(os.path.dirname(sqlite_libfile))] @@ -1240,7 +1243,7 @@ class db_found(Exception): pass if sqlite_incdir not in self.compiler.include_dirs: include_dirs.append(sqlite_incdir) # avoid a runtime library path for a system library dir - if sqlite_libdir and sqlite_libdir[0] in lib_dirs: + if sqlite_libdir and sqlite_libdir[0] in self.lib_dirs: sqlite_libdir = None self.add(Extension('_sqlite3', sqlite_srcs, define_macros=sqlite_defines, @@ -1266,13 +1269,13 @@ class db_found(Exception): pass dbmext = None for cand in dbm_order: if cand == "ndbm": - if find_file("ndbm.h", inc_dirs, []) is not None: + if find_file("ndbm.h", self.inc_dirs, []) is not None: # Some systems have -lndbm, others have -lgdbm_compat, # others don't have either - if self.compiler.find_library_file(lib_dirs, + if self.compiler.find_library_file(self.lib_dirs, 'ndbm'): ndbm_libs = ['ndbm'] - elif self.compiler.find_library_file(lib_dirs, + elif self.compiler.find_library_file(self.lib_dirs, 'gdbm_compat'): ndbm_libs = ['gdbm_compat'] else: @@ -1286,12 +1289,12 @@ class db_found(Exception): pass break elif cand == "gdbm": - if self.compiler.find_library_file(lib_dirs, 'gdbm'): + if self.compiler.find_library_file(self.lib_dirs, 'gdbm'): gdbm_libs = ['gdbm'] - if self.compiler.find_library_file(lib_dirs, + if self.compiler.find_library_file(self.lib_dirs, 'gdbm_compat'): gdbm_libs.append('gdbm_compat') - if find_file("gdbm/ndbm.h", inc_dirs, []) is not None: + if find_file("gdbm/ndbm.h", self.inc_dirs, []) is not None: if dbm_setup_debug: print("building dbm using gdbm") dbmext = Extension( '_dbm', ['_dbmmodule.c'], @@ -1300,7 +1303,7 @@ class db_found(Exception): pass ], libraries = gdbm_libs) break - if find_file("gdbm-ndbm.h", inc_dirs, []) is not None: + if find_file("gdbm-ndbm.h", self.inc_dirs, []) is not None: if dbm_setup_debug: print("building dbm using gdbm") dbmext = Extension( '_dbm', ['_dbmmodule.c'], @@ -1329,7 +1332,7 @@ class db_found(Exception): pass # Anthony Baxter's gdbm module. GNU dbm(3) will require -lgdbm: if ('gdbm' in dbm_order and - self.compiler.find_library_file(lib_dirs, 'gdbm')): + self.compiler.find_library_file(self.lib_dirs, 'gdbm')): self.add(Extension('_gdbm', ['_gdbmmodule.c'], libraries=['gdbm'])) else: @@ -1345,7 +1348,7 @@ class db_found(Exception): pass else: self.missing.extend(['resource', 'termios']) - self._detect_nis(inc_dirs, lib_dirs) + self._detect_nis() # Curses support, requiring the System V version of curses, often # provided by the ncurses library. @@ -1381,9 +1384,9 @@ class db_found(Exception): pass elif curses_library == 'curses' and not MACOS: # OSX has an old Berkeley curses, not good enough for # the _curses module. - if (self.compiler.find_library_file(lib_dirs, 'terminfo')): + if (self.compiler.find_library_file(self.lib_dirs, 'terminfo')): curses_libs = ['curses', 'terminfo'] - elif (self.compiler.find_library_file(lib_dirs, 'termcap')): + elif (self.compiler.find_library_file(self.lib_dirs, 'termcap')): curses_libs = ['curses', 'termcap'] else: curses_libs = ['curses'] @@ -1396,7 +1399,7 @@ class db_found(Exception): pass # If the curses module is enabled, check for the panel module if (module_enabled(self.extensions, '_curses') and - self.compiler.find_library_file(lib_dirs, panel_library)): + self.compiler.find_library_file(self.lib_dirs, panel_library)): self.add(Extension('_curses_panel', ['_curses_panel.c'], include_dirs=curses_includes, define_macros=curses_defines, @@ -1416,7 +1419,7 @@ class db_found(Exception): pass # # You can upgrade zlib to version 1.1.4 yourself by going to # http://www.gzip.org/zlib/ - zlib_inc = find_file('zlib.h', [], inc_dirs) + zlib_inc = find_file('zlib.h', [], self.inc_dirs) have_zlib = False if zlib_inc is not None: zlib_h = zlib_inc[0] + '/zlib.h' @@ -1433,7 +1436,7 @@ class db_found(Exception): pass version = line.split()[2] break if version >= version_req: - if (self.compiler.find_library_file(lib_dirs, 'z')): + if (self.compiler.find_library_file(self.lib_dirs, 'z')): if MACOS: zlib_extra_link_args = ('-Wl,-search_paths_first',) else: @@ -1465,7 +1468,7 @@ class db_found(Exception): pass extra_link_args=extra_link_args)) # Gustavo Niemeyer's bz2 module. - if (self.compiler.find_library_file(lib_dirs, 'bz2')): + if (self.compiler.find_library_file(self.lib_dirs, 'bz2')): if MACOS: bz2_extra_link_args = ('-Wl,-search_paths_first',) else: @@ -1477,7 +1480,7 @@ class db_found(Exception): pass self.missing.append('_bz2') # LZMA compression support. - if self.compiler.find_library_file(lib_dirs, 'lzma'): + if self.compiler.find_library_file(self.lib_dirs, 'lzma'): self.add(Extension('_lzma', ['_lzmamodule.c'], libraries=['lzma'])) else: @@ -1502,7 +1505,7 @@ class db_found(Exception): pass expat_sources = [] expat_depends = [] else: - expat_inc = [os.path.join(os.getcwd(), srcdir, 'Modules', 'expat')] + expat_inc = [os.path.join(self.srcdir, 'Modules', 'expat')] define_macros = [ ('HAVE_EXPAT_CONFIG_H', '1'), # bpo-30947: Python uses best available entropy sources to @@ -1544,7 +1547,7 @@ class db_found(Exception): pass # Fredrik Lundh's cElementTree module. Note that this also # uses expat (via the CAPI hook in pyexpat). - if os.path.isfile(os.path.join(srcdir, 'Modules', '_elementtree.c')): + if os.path.isfile(os.path.join(self.srcdir, 'Modules', '_elementtree.c')): define_macros.append(('USE_PYEXPAT_CAPI', None)) self.add(Extension('_elementtree', define_macros=define_macros, @@ -1567,7 +1570,7 @@ class db_found(Exception): pass self._detect_decimal() # Thomas Heller's _ctypes module - self.detect_ctypes(inc_dirs, lib_dirs) + self.detect_ctypes() # Richard Oudkerk's multiprocessing module if MS_WINDOWS: @@ -1636,15 +1639,15 @@ class db_found(Exception): pass '-framework', 'CoreFoundation'])) # Call the method for detecting whether _tkinter can be compiled - self.detect_tkinter(inc_dirs, lib_dirs) + self.detect_tkinter() if '_tkinter' not in [e.name for e in self.extensions]: self.missing.append('_tkinter') # Build the _uuid module if possible - uuid_incs = find_file("uuid.h", inc_dirs, ["/usr/include/uuid"]) + uuid_incs = find_file("uuid.h", self.inc_dirs, ["/usr/include/uuid"]) if uuid_incs is not None: - if self.compiler.find_library_file(lib_dirs, 'uuid'): + if self.compiler.find_library_file(self.lib_dirs, 'uuid'): uuid_libs = ['uuid'] else: uuid_libs = [] @@ -1698,7 +1701,7 @@ def detect_tkinter_explicitly(self): self.extensions.append(ext) return True - def detect_tkinter_darwin(self, inc_dirs, lib_dirs): + def detect_tkinter_darwin(self): # The _tkinter module, using frameworks. Since frameworks are quite # different the UNIX search logic is not sharable. from os.path import join, exists @@ -1787,7 +1790,7 @@ def detect_tkinter_darwin(self, inc_dirs, lib_dirs): self.extensions.append(ext) return True - def detect_tkinter(self, inc_dirs, lib_dirs): + def detect_tkinter(self): # The _tkinter module. # Check whether --with-tcltk-includes and --with-tcltk-libs were @@ -1801,7 +1804,7 @@ def detect_tkinter(self, inc_dirs, lib_dirs): # AquaTk is a separate method. Only one Tkinter will be built on # Darwin - either AquaTk, if it is found, or X11 based Tk. if (MACOS and - self.detect_tkinter_darwin(inc_dirs, lib_dirs)): + self.detect_tkinter_darwin()): return # Assume we haven't found any of the libraries or include files @@ -1810,9 +1813,9 @@ def detect_tkinter(self, inc_dirs, lib_dirs): tcllib = tklib = tcl_includes = tk_includes = None for version in ['8.6', '86', '8.5', '85', '8.4', '84', '8.3', '83', '8.2', '82', '8.1', '81', '8.0', '80']: - tklib = self.compiler.find_library_file(lib_dirs, + tklib = self.compiler.find_library_file(self.lib_dirs, 'tk' + version) - tcllib = self.compiler.find_library_file(lib_dirs, + tcllib = self.compiler.find_library_file(self.lib_dirs, 'tcl' + version) if tklib and tcllib: # Exit the loop when we've found the Tcl/Tk libraries @@ -1829,12 +1832,12 @@ def detect_tkinter(self, inc_dirs, lib_dirs): dotversion = dotversion[:-1] + '.' + dotversion[-1] tcl_include_sub = [] tk_include_sub = [] - for dir in inc_dirs: + for dir in self.inc_dirs: tcl_include_sub += [dir + os.sep + "tcl" + dotversion] tk_include_sub += [dir + os.sep + "tk" + dotversion] tk_include_sub += tcl_include_sub - tcl_includes = find_file('tcl.h', inc_dirs, tcl_include_sub) - tk_includes = find_file('tk.h', inc_dirs, tk_include_sub) + tcl_includes = find_file('tcl.h', self.inc_dirs, tcl_include_sub) + tk_includes = find_file('tk.h', self.inc_dirs, tk_include_sub) if (tcllib is None or tklib is None or tcl_includes is None or tk_includes is None): @@ -1871,11 +1874,11 @@ def detect_tkinter(self, inc_dirs, lib_dirs): return # Check for BLT extension - if self.compiler.find_library_file(lib_dirs + added_lib_dirs, + if self.compiler.find_library_file(self.lib_dirs + added_lib_dirs, 'BLT8.0'): defs.append( ('WITH_BLT', 1) ) libs.append('BLT8.0') - elif self.compiler.find_library_file(lib_dirs + added_lib_dirs, + elif self.compiler.find_library_file(self.lib_dirs + added_lib_dirs, 'BLT'): defs.append( ('WITH_BLT', 1) ) libs.append('BLT') @@ -1910,8 +1913,7 @@ def detect_tkinter(self, inc_dirs, lib_dirs): def configure_ctypes_darwin(self, ext): # Darwin (OS X) uses preconfigured files, in # the Modules/_ctypes/libffi_osx directory. - srcdir = sysconfig.get_config_var('srcdir') - ffi_srcdir = os.path.abspath(os.path.join(srcdir, 'Modules', + ffi_srcdir = os.path.abspath(os.path.join(self.srcdir, 'Modules', '_ctypes', 'libffi_osx')) sources = [os.path.join(ffi_srcdir, p) for p in ['ffi.c', @@ -1942,7 +1944,7 @@ def configure_ctypes(self, ext): return False return True - def detect_ctypes(self, inc_dirs, lib_dirs): + def detect_ctypes(self): self.use_system_libffi = False include_dirs = [] extra_compile_args = [] @@ -1989,7 +1991,7 @@ def detect_ctypes(self, inc_dirs, lib_dirs): libraries=['m']) self.extensions.extend([ext, ext_test]) - ffi_inc_dirs = inc_dirs.copy() + ffi_inc_dirs = self.inc_dirs.copy() if MACOS: if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"): return @@ -2008,7 +2010,7 @@ def detect_ctypes(self, inc_dirs, lib_dirs): ffi_lib = None if ffi_inc is not None: for lib_name in ('ffi', 'ffi_pic'): - if (self.compiler.find_library_file(lib_dirs, lib_name)): + if (self.compiler.find_library_file(self.lib_dirs, lib_name)): ffi_lib = lib_name break @@ -2030,8 +2032,7 @@ def _detect_decimal(self): sources = ['_decimal/_decimal.c'] depends = ['_decimal/docstrings.h'] else: - srcdir = sysconfig.get_config_var('srcdir') - include_dirs = [os.path.abspath(os.path.join(srcdir, + include_dirs = [os.path.abspath(os.path.join(self.srcdir, 'Modules', '_decimal', 'libmpdec'))] @@ -2137,7 +2138,7 @@ def _detect_decimal(self): sources=sources, depends=depends)) - def _detect_openssl(self, inc_dirs, lib_dirs): + def _detect_openssl(self): config_vars = sysconfig.get_config_vars() def split_var(name, sep): @@ -2160,14 +2161,14 @@ def split_var(name, sep): # Find OpenSSL includes ssl_incs = find_file( - 'openssl/ssl.h', inc_dirs, openssl_includes + 'openssl/ssl.h', self.inc_dirs, openssl_includes ) if ssl_incs is None: return None, None # OpenSSL 1.0.2 uses Kerberos for KRB5 ciphers krb5_h = find_file( - 'krb5.h', inc_dirs, + 'krb5.h', self.inc_dirs, ['/usr/kerberos/include'] ) if krb5_h: @@ -2188,7 +2189,7 @@ def split_var(name, sep): library_dirs=openssl_libdirs, libraries=openssl_libs)) - def _detect_nis(self, inc_dirs, lib_dirs): + def _detect_nis(self): if MS_WINDOWS or CYGWIN or HOST_PLATFORM == 'qnx6': self.missing.append('nis') return @@ -2201,12 +2202,12 @@ def _detect_nis(self, inc_dirs, lib_dirs): # moved headers and libraries to libtirpc and libnsl. The headers # are in tircp and nsl sub directories. rpcsvc_inc = find_file( - 'rpcsvc/yp_prot.h', inc_dirs, - [os.path.join(inc_dir, 'nsl') for inc_dir in inc_dirs] + 'rpcsvc/yp_prot.h', self.inc_dirs, + [os.path.join(inc_dir, 'nsl') for inc_dir in self.inc_dirs] ) rpc_inc = find_file( - 'rpc/rpc.h', inc_dirs, - [os.path.join(inc_dir, 'tirpc') for inc_dir in inc_dirs] + 'rpc/rpc.h', self.inc_dirs, + [os.path.join(inc_dir, 'tirpc') for inc_dir in self.inc_dirs] ) if rpcsvc_inc is None or rpc_inc is None: # not found @@ -2215,17 +2216,17 @@ def _detect_nis(self, inc_dirs, lib_dirs): includes_dirs.extend(rpcsvc_inc) includes_dirs.extend(rpc_inc) - if self.compiler.find_library_file(lib_dirs, 'nsl'): + if self.compiler.find_library_file(self.lib_dirs, 'nsl'): libs.append('nsl') else: # libnsl-devel: check for libnsl in nsl/ subdirectory - nsl_dirs = [os.path.join(lib_dir, 'nsl') for lib_dir in lib_dirs] + nsl_dirs = [os.path.join(lib_dir, 'nsl') for lib_dir in self.lib_dirs] libnsl = self.compiler.find_library_file(nsl_dirs, 'nsl') if libnsl is not None: library_dirs.append(os.path.dirname(libnsl)) libs.append('nsl') - if self.compiler.find_library_file(lib_dirs, 'tirpc'): + if self.compiler.find_library_file(self.lib_dirs, 'tirpc'): libs.append('tirpc') self.add(Extension('nis', ['nismodule.c'],

1 0

bpo-36146: Refactor setup.py: PyBuildExt.add() method (GH-12097)
by Victor Stinner March 1, 2019

March 1, 2019

https://github.com/python/cpython/commit/8058bdae3e5e1f77a202d9dc907b418940… commit: 8058bdae3e5e1f77a202d9dc907b4189409c9b03 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2019-03-01T15:31:45+01:00 summary: bpo-36146: Refactor setup.py: PyBuildExt.add() method (GH-12097) * Add PyBuildExt.add() which adds the extension directly to self.extensions, rather than using a temporary 'exts' local variable in detect_modules() and then add 'exts' to self.extensions * Convert 'missing' local variable from detect_modules() into PyBuildExt.missing attribute * _detect_openssl(), _decimal_ext() and _detect_nis() now call directly self.add(), rather than returning an extension (or None if not found). * Rename _decimal_ext() to _detect_decimal() for consistency with other methods. files: M setup.py diff --git a/setup.py b/setup.py index 56a1df327399..db77772da925 100644 --- a/setup.py +++ b/setup.py @@ -229,13 +229,17 @@ def __init__(self, dist): build_ext.__init__(self, dist) self.failed = [] self.failed_on_import = [] + self.missing = [] if '-j' in os.environ.get('MAKEFLAGS', ''): self.parallel = True + def add(self, ext): + self.extensions.append(ext) + def build_extensions(self): # Detect which modules should be compiled - missing = self.detect_modules() + self.detect_modules() # Remove modules that are present on the disabled list extensions = [ext for ext in self.extensions @@ -331,12 +335,12 @@ def print_three_column(lst): print("%-*s %-*s %-*s" % (longest, e, longest, f, longest, g)) - if missing: + if self.missing: print() print("Python build finished successfully!") print("The necessary bits to build these optional modules were not " "found:") - print_three_column(missing) + print_three_column(self.missing) print("To find the necessary bits, look in setup.py in" " detect_modules() for the module's name.") print() @@ -374,7 +378,7 @@ def print_three_column(lst): print() if any('_ssl' in l - for l in (missing, self.failed, self.failed_on_import)): + for l in (self.missing, self.failed, self.failed_on_import)): print() print("Could not build the ssl module!") print("Python requires an OpenSSL 1.0.2 or 1.1 compatible " @@ -609,8 +613,6 @@ def detect_modules(self): inc_dirs = (self.compiler.include_dirs + sysroot_paths(('CPPFLAGS', 'CFLAGS', 'CC'), system_include_dirs)) - exts = [] - missing = [] config_h = sysconfig.get_config_h_filename() with open(config_h) as file: @@ -650,22 +652,22 @@ def detect_modules(self): # # array objects - exts.append( Extension('array', ['arraymodule.c']) ) + self.add(Extension('array', ['arraymodule.c'])) # Context Variables - exts.append( Extension('_contextvars', ['_contextvarsmodule.c']) ) + self.add(Extension('_contextvars', ['_contextvarsmodule.c'])) shared_math = 'Modules/_math.o' # complex math library functions - exts.append( Extension('cmath', ['cmathmodule.c'], - extra_objects=[shared_math], - depends=['_math.h', shared_math], - libraries=['m']) ) + self.add(Extension('cmath', ['cmathmodule.c'], + extra_objects=[shared_math], + depends=['_math.h', shared_math], + libraries=['m'])) # math library functions, e.g. sin() - exts.append( Extension('math', ['mathmodule.c'], - extra_objects=[shared_math], - depends=['_math.h', shared_math], - libraries=['m']) ) + self.add(Extension('math', ['mathmodule.c'], + extra_objects=[shared_math], + depends=['_math.h', shared_math], + libraries=['m'])) # time libraries: librt may be needed for clock_gettime() time_libs = [] @@ -674,48 +676,48 @@ def detect_modules(self): time_libs.append(lib) # time operations and variables - exts.append( Extension('time', ['timemodule.c'], - libraries=time_libs) ) + self.add(Extension('time', ['timemodule.c'], + libraries=time_libs)) # libm is needed by delta_new() that uses round() and by accum() that # uses modf(). - exts.append( Extension('_datetime', ['_datetimemodule.c'], - libraries=['m']) ) + self.add(Extension('_datetime', ['_datetimemodule.c'], + libraries=['m'])) # random number generator implemented in C - exts.append( Extension("_random", ["_randommodule.c"]) ) + self.add(Extension("_random", ["_randommodule.c"])) # bisect - exts.append( Extension("_bisect", ["_bisectmodule.c"]) ) + self.add(Extension("_bisect", ["_bisectmodule.c"])) # heapq - exts.append( Extension("_heapq", ["_heapqmodule.c"]) ) + self.add(Extension("_heapq", ["_heapqmodule.c"])) # C-optimized pickle replacement - exts.append( Extension("_pickle", ["_pickle.c"]) ) + self.add(Extension("_pickle", ["_pickle.c"])) # atexit - exts.append( Extension("atexit", ["atexitmodule.c"]) ) + self.add(Extension("atexit", ["atexitmodule.c"])) # _json speedups - exts.append( Extension("_json", ["_json.c"], - # pycore_accu.h requires Py_BUILD_CORE_BUILTIN - extra_compile_args=['-DPy_BUILD_CORE_BUILTIN']) ) + self.add(Extension("_json", ["_json.c"], + # pycore_accu.h requires Py_BUILD_CORE_BUILTIN + extra_compile_args=['-DPy_BUILD_CORE_BUILTIN'])) # Python C API test module - exts.append( Extension('_testcapi', ['_testcapimodule.c'], - depends=['testcapi_long.h']) ) + self.add(Extension('_testcapi', ['_testcapimodule.c'], + depends=['testcapi_long.h'])) # Python PEP-3118 (buffer protocol) test module - exts.append( Extension('_testbuffer', ['_testbuffer.c']) ) + self.add(Extension('_testbuffer', ['_testbuffer.c'])) # Test loading multiple modules from one compiled file (http://bugs.python.org/issue16421) - exts.append( Extension('_testimportmultiple', ['_testimportmultiple.c']) ) + self.add(Extension('_testimportmultiple', ['_testimportmultiple.c'])) # Test multi-phase extension module init (PEP 489) - exts.append( Extension('_testmultiphase', ['_testmultiphase.c']) ) + self.add(Extension('_testmultiphase', ['_testmultiphase.c'])) # profiler (_lsprof is for cProfile.py) - exts.append( Extension('_lsprof', ['_lsprof.c', 'rotatingtree.c']) ) + self.add(Extension('_lsprof', ['_lsprof.c', 'rotatingtree.c'])) # static Unicode character database - exts.append( Extension('unicodedata', ['unicodedata.c'], - depends=['unicodedata_db.h', 'unicodename_db.h']) ) + self.add(Extension('unicodedata', ['unicodedata.c'], + depends=['unicodedata_db.h', 'unicodename_db.h'])) # _opcode module - exts.append( Extension('_opcode', ['_opcode.c']) ) + self.add(Extension('_opcode', ['_opcode.c'])) # asyncio speedups - exts.append( Extension("_asyncio", ["_asynciomodule.c"]) ) + self.add(Extension("_asyncio", ["_asynciomodule.c"])) # _abc speedups - exts.append( Extension("_abc", ["_abc.c"]) ) + self.add(Extension("_abc", ["_abc.c"])) # _queue module - exts.append( Extension("_queue", ["_queuemodule.c"]) ) + self.add(Extension("_queue", ["_queuemodule.c"])) # Modules with some UNIX dependencies -- on by default: # (If you have a really backward UNIX, select and socket may not be @@ -726,41 +728,42 @@ def detect_modules(self): if (config_h_vars.get('FLOCK_NEEDS_LIBBSD', False)): # May be necessary on AIX for flock function libs = ['bsd'] - exts.append( Extension('fcntl', ['fcntlmodule.c'], libraries=libs) ) + self.add(Extension('fcntl', ['fcntlmodule.c'], + libraries=libs)) # pwd(3) - exts.append( Extension('pwd', ['pwdmodule.c']) ) + self.add(Extension('pwd', ['pwdmodule.c'])) # grp(3) if not VXWORKS: - exts.append( Extension('grp', ['grpmodule.c']) ) + self.add(Extension('grp', ['grpmodule.c'])) # spwd, shadow passwords if (config_h_vars.get('HAVE_GETSPNAM', False) or config_h_vars.get('HAVE_GETSPENT', False)): - exts.append( Extension('spwd', ['spwdmodule.c']) ) + self.add(Extension('spwd', ['spwdmodule.c'])) else: - missing.append('spwd') + self.missing.append('spwd') # select(2); not on ancient System V - exts.append( Extension('select', ['selectmodule.c']) ) + self.add(Extension('select', ['selectmodule.c'])) # Fred Drake's interface to the Python parser - exts.append( Extension('parser', ['parsermodule.c']) ) + self.add(Extension('parser', ['parsermodule.c'])) # Memory-mapped files (also works on Win32). - exts.append( Extension('mmap', ['mmapmodule.c']) ) + self.add(Extension('mmap', ['mmapmodule.c'])) # Lance Ellinghaus's syslog module # syslog daemon interface - exts.append( Extension('syslog', ['syslogmodule.c']) ) + self.add(Extension('syslog', ['syslogmodule.c'])) # Fuzz tests. - exts.append( Extension( - '_xxtestfuzz', - ['_xxtestfuzz/_xxtestfuzz.c', '_xxtestfuzz/fuzzer.c']) - ) + self.add(Extension('_xxtestfuzz', + ['_xxtestfuzz/_xxtestfuzz.c', + '_xxtestfuzz/fuzzer.c'])) # Python interface to subinterpreter C-API. - exts.append(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'], - define_macros=[('Py_BUILD_CORE', '')])) + self.add(Extension('_xxsubinterpreters', + ['_xxsubinterpretersmodule.c'], + define_macros=[('Py_BUILD_CORE', '')])) # # Here ends the simple stuff. From here on, modules need certain @@ -776,8 +779,8 @@ def detect_modules(self): # 64-bit platforms. # # audioop needs libm for floor() in multiple functions. - exts.append( Extension('audioop', ['audioop.c'], - libraries=['m']) ) + self.add(Extension('audioop', ['audioop.c'], + libraries=['m'])) # readline do_readline = self.compiler.find_library_file(lib_dirs, 'readline') @@ -855,12 +858,12 @@ def detect_modules(self): ['/usr/lib/termcap'], 'termcap'): readline_libs.append('termcap') - exts.append( Extension('readline', ['readline.c'], - library_dirs=['/usr/lib/termcap'], - extra_link_args=readline_extra_link_args, - libraries=readline_libs) ) + self.add(Extension('readline', ['readline.c'], + library_dirs=['/usr/lib/termcap'], + extra_link_args=readline_extra_link_args, + libraries=readline_libs)) else: - missing.append('readline') + self.missing.append('readline') # crypt module. @@ -870,65 +873,60 @@ def detect_modules(self): libs = [] if not VXWORKS: - exts.append( Extension('_crypt', ['_cryptmodule.c'], libraries=libs) ) + self.add(Extension('_crypt', ['_cryptmodule.c'], + libraries=libs)) elif self.compiler.find_library_file(lib_dirs, 'OPENSSL'): libs = ['OPENSSL'] - exts.append( Extension('_crypt', ['_cryptmodule.c'], libraries=libs) ) + self.add(Extension('_crypt', ['_cryptmodule.c'], + libraries=libs)) # CSV files - exts.append( Extension('_csv', ['_csv.c']) ) + self.add(Extension('_csv', ['_csv.c'])) # POSIX subprocess module helper. - exts.append( Extension('_posixsubprocess', ['_posixsubprocess.c']) ) + self.add(Extension('_posixsubprocess', ['_posixsubprocess.c'])) # socket(2) if not VXWORKS: - exts.append( Extension('_socket', ['socketmodule.c'], - depends = ['socketmodule.h']) ) + self.add(Extension('_socket', ['socketmodule.c'], + depends=['socketmodule.h'])) elif self.compiler.find_library_file(lib_dirs, 'net'): libs = ['net'] - exts.append( Extension('_socket', ['socketmodule.c'], - depends = ['socketmodule.h'], libraries=libs) ) + self.add(Extension('_socket', ['socketmodule.c'], + depends=['socketmodule.h'], + libraries=libs)) # Detect SSL support for the socket module (via _ssl) - ssl_ext, hashlib_ext = self._detect_openssl(inc_dirs, lib_dirs) - if ssl_ext is not None: - exts.append(ssl_ext) - else: - missing.append('_ssl') - if hashlib_ext is not None: - exts.append(hashlib_ext) - else: - missing.append('_hashlib') + self._detect_openssl(inc_dirs, lib_dirs) # We always compile these even when OpenSSL is available (issue #14693). # It's harmless and the object code is tiny (40-50 KiB per module, # only loaded when actually used). - exts.append( Extension('_sha256', ['sha256module.c'], - depends=['hashlib.h']) ) - exts.append( Extension('_sha512', ['sha512module.c'], - depends=['hashlib.h']) ) - exts.append( Extension('_md5', ['md5module.c'], - depends=['hashlib.h']) ) - exts.append( Extension('_sha1', ['sha1module.c'], - depends=['hashlib.h']) ) + self.add(Extension('_sha256', ['sha256module.c'], + depends=['hashlib.h'])) + self.add(Extension('_sha512', ['sha512module.c'], + depends=['hashlib.h'])) + self.add(Extension('_md5', ['md5module.c'], + depends=['hashlib.h'])) + self.add(Extension('_sha1', ['sha1module.c'], + depends=['hashlib.h'])) blake2_deps = glob(os.path.join(os.getcwd(), srcdir, 'Modules/_blake2/impl/*')) blake2_deps.append('hashlib.h') - exts.append( Extension('_blake2', - ['_blake2/blake2module.c', - '_blake2/blake2b_impl.c', - '_blake2/blake2s_impl.c'], - depends=blake2_deps) ) + self.add(Extension('_blake2', + ['_blake2/blake2module.c', + '_blake2/blake2b_impl.c', + '_blake2/blake2s_impl.c'], + depends=blake2_deps)) sha3_deps = glob(os.path.join(os.getcwd(), srcdir, 'Modules/_sha3/kcp/*')) sha3_deps.append('hashlib.h') - exts.append( Extension('_sha3', - ['_sha3/sha3module.c'], - depends=sha3_deps)) + self.add(Extension('_sha3', + ['_sha3/sha3module.c'], + depends=sha3_deps)) # Modules that provide persistent dictionary-like semantics. You will # probably want to arrange for at least one of them to be available on @@ -1244,14 +1242,14 @@ class db_found(Exception): pass # avoid a runtime library path for a system library dir if sqlite_libdir and sqlite_libdir[0] in lib_dirs: sqlite_libdir = None - exts.append(Extension('_sqlite3', sqlite_srcs, - define_macros=sqlite_defines, - include_dirs=include_dirs, - library_dirs=sqlite_libdir, - extra_link_args=sqlite_extra_link_args, - libraries=["sqlite3",])) + self.add(Extension('_sqlite3', sqlite_srcs, + define_macros=sqlite_defines, + include_dirs=include_dirs, + library_dirs=sqlite_libdir, + extra_link_args=sqlite_extra_link_args, + libraries=["sqlite3",])) else: - missing.append('_sqlite3') + self.missing.append('_sqlite3') dbm_setup_debug = False # verbose debug prints from this script? dbm_order = ['gdbm'] @@ -1325,33 +1323,29 @@ class db_found(Exception): pass libraries=dblibs) break if dbmext is not None: - exts.append(dbmext) + self.add(dbmext) else: - missing.append('_dbm') + self.missing.append('_dbm') # Anthony Baxter's gdbm module. GNU dbm(3) will require -lgdbm: if ('gdbm' in dbm_order and self.compiler.find_library_file(lib_dirs, 'gdbm')): - exts.append( Extension('_gdbm', ['_gdbmmodule.c'], - libraries = ['gdbm'] ) ) + self.add(Extension('_gdbm', ['_gdbmmodule.c'], + libraries=['gdbm'])) else: - missing.append('_gdbm') + self.missing.append('_gdbm') # Unix-only modules if not MS_WINDOWS: if not VXWORKS: # Steen Lumholt's termios module - exts.append( Extension('termios', ['termios.c']) ) + self.add(Extension('termios', ['termios.c'])) # Jeremy Hylton's rlimit interface - exts.append( Extension('resource', ['resource.c']) ) + self.add(Extension('resource', ['resource.c'])) else: - missing.extend(['resource', 'termios']) + self.missing.extend(['resource', 'termios']) - nis = self._detect_nis(inc_dirs, lib_dirs) - if nis is not None: - exts.append(nis) - else: - missing.append('nis') + self._detect_nis(inc_dirs, lib_dirs) # Curses support, requiring the System V version of curses, often # provided by the ncurses library. @@ -1380,10 +1374,10 @@ class db_found(Exception): pass if curses_library.startswith('ncurses'): curses_libs = [curses_library] - exts.append( Extension('_curses', ['_cursesmodule.c'], - include_dirs=curses_includes, - define_macros=curses_defines, - libraries = curses_libs) ) + self.add(Extension('_curses', ['_cursesmodule.c'], + include_dirs=curses_includes, + define_macros=curses_defines, + libraries=curses_libs)) elif curses_library == 'curses' and not MACOS: # OSX has an old Berkeley curses, not good enough for # the _curses module. @@ -1394,21 +1388,21 @@ class db_found(Exception): pass else: curses_libs = ['curses'] - exts.append( Extension('_curses', ['_cursesmodule.c'], - define_macros=curses_defines, - libraries = curses_libs) ) + self.add(Extension('_curses', ['_cursesmodule.c'], + define_macros=curses_defines, + libraries=curses_libs)) else: - missing.append('_curses') + self.missing.append('_curses') # If the curses module is enabled, check for the panel module - if (module_enabled(exts, '_curses') and + if (module_enabled(self.extensions, '_curses') and self.compiler.find_library_file(lib_dirs, panel_library)): - exts.append( Extension('_curses_panel', ['_curses_panel.c'], - include_dirs=curses_includes, - define_macros=curses_defines, - libraries = [panel_library] + curses_libs) ) + self.add(Extension('_curses_panel', ['_curses_panel.c'], + include_dirs=curses_includes, + define_macros=curses_defines, + libraries=[panel_library, *curses_libs])) else: - missing.append('_curses_panel') + self.missing.append('_curses_panel') # Andrew Kuchling's zlib module. Note that some versions of zlib # 1.1.3 have security problems. See CERT Advisory CA-2002-07: @@ -1444,16 +1438,16 @@ class db_found(Exception): pass zlib_extra_link_args = ('-Wl,-search_paths_first',) else: zlib_extra_link_args = () - exts.append( Extension('zlib', ['zlibmodule.c'], - libraries = ['z'], - extra_link_args = zlib_extra_link_args)) + self.add(Extension('zlib', ['zlibmodule.c'], + libraries=['z'], + extra_link_args=zlib_extra_link_args)) have_zlib = True else: - missing.append('zlib') + self.missing.append('zlib') else: - missing.append('zlib') + self.missing.append('zlib') else: - missing.append('zlib') + self.missing.append('zlib') # Helper module for various ascii-encoders. Uses zlib for an optimized # crc32 if we have it. Otherwise binascii uses its own. @@ -1465,10 +1459,10 @@ class db_found(Exception): pass extra_compile_args = [] libraries = [] extra_link_args = [] - exts.append( Extension('binascii', ['binascii.c'], - extra_compile_args = extra_compile_args, - libraries = libraries, - extra_link_args = extra_link_args) ) + self.add(Extension('binascii', ['binascii.c'], + extra_compile_args=extra_compile_args, + libraries=libraries, + extra_link_args=extra_link_args)) # Gustavo Niemeyer's bz2 module. if (self.compiler.find_library_file(lib_dirs, 'bz2')): @@ -1476,18 +1470,18 @@ class db_found(Exception): pass bz2_extra_link_args = ('-Wl,-search_paths_first',) else: bz2_extra_link_args = () - exts.append( Extension('_bz2', ['_bz2module.c'], - libraries = ['bz2'], - extra_link_args = bz2_extra_link_args) ) + self.add(Extension('_bz2', ['_bz2module.c'], + libraries=['bz2'], + extra_link_args=bz2_extra_link_args)) else: - missing.append('_bz2') + self.missing.append('_bz2') # LZMA compression support. if self.compiler.find_library_file(lib_dirs, 'lzma'): - exts.append( Extension('_lzma', ['_lzmamodule.c'], - libraries = ['lzma']) ) + self.add(Extension('_lzma', ['_lzmamodule.c'], + libraries=['lzma'])) else: - missing.append('_lzma') + self.missing.append('_lzma') # Interface to the Expat XML parser # @@ -1539,40 +1533,38 @@ class db_found(Exception): pass if ret >> 8 == 0: extra_compile_args.append('-Wno-implicit-fallthrough') - exts.append(Extension('pyexpat', - define_macros = define_macros, - extra_compile_args = extra_compile_args, - include_dirs = expat_inc, - libraries = expat_lib, - sources = ['pyexpat.c'] + expat_sources, - depends = expat_depends, - )) + self.add(Extension('pyexpat', + define_macros=define_macros, + extra_compile_args=extra_compile_args, + include_dirs=expat_inc, + libraries=expat_lib, + sources=['pyexpat.c'] + expat_sources, + depends=expat_depends)) # Fredrik Lundh's cElementTree module. Note that this also # uses expat (via the CAPI hook in pyexpat). if os.path.isfile(os.path.join(srcdir, 'Modules', '_elementtree.c')): define_macros.append(('USE_PYEXPAT_CAPI', None)) - exts.append(Extension('_elementtree', - define_macros = define_macros, - include_dirs = expat_inc, - libraries = expat_lib, - sources = ['_elementtree.c'], - depends = ['pyexpat.c'] + expat_sources + - expat_depends, - )) + self.add(Extension('_elementtree', + define_macros=define_macros, + include_dirs=expat_inc, + libraries=expat_lib, + sources=['_elementtree.c'], + depends=['pyexpat.c', *expat_sources, + *expat_depends])) else: - missing.append('_elementtree') + self.missing.append('_elementtree') # Hye-Shik Chang's CJKCodecs modules. - exts.append(Extension('_multibytecodec', - ['cjkcodecs/multibytecodec.c'])) + self.add(Extension('_multibytecodec', + ['cjkcodecs/multibytecodec.c'])) for loc in ('kr', 'jp', 'cn', 'tw', 'hk', 'iso2022'): - exts.append(Extension('_codecs_%s' % loc, - ['cjkcodecs/_codecs_%s.c' % loc])) + self.add(Extension('_codecs_%s' % loc, + ['cjkcodecs/_codecs_%s.c' % loc])) # Stefan Krah's _decimal module - exts.append(self._decimal_ext()) + self._detect_decimal() # Thomas Heller's _ctypes module self.detect_ctypes(inc_dirs, lib_dirs) @@ -1621,37 +1613,33 @@ class db_found(Exception): pass if sysconfig.get_config_var('SHM_NEEDS_LIBRT'): # need to link with librt to get shm_open() libs.append('rt') - exts.append( Extension('_posixshmem', posixshmem_srcs, - define_macros={}, - libraries=libs, - include_dirs=["Modules/_multiprocessing"])) - - exts.append ( Extension('_multiprocessing', multiprocessing_srcs, - define_macros=list(macros.items()), - include_dirs=["Modules/_multiprocessing"])) + self.add(Extension('_posixshmem', posixshmem_srcs, + define_macros={}, + libraries=libs, + include_dirs=["Modules/_multiprocessing"])) + + self.add(Extension('_multiprocessing', multiprocessing_srcs, + define_macros=list(macros.items()), + include_dirs=["Modules/_multiprocessing"])) # End multiprocessing # Platform-specific libraries if HOST_PLATFORM.startswith(('linux', 'freebsd', 'gnukfreebsd')): - exts.append( Extension('ossaudiodev', ['ossaudiodev.c']) ) + self.add(Extension('ossaudiodev', ['ossaudiodev.c'])) else: - missing.append('ossaudiodev') + self.missing.append('ossaudiodev') if MACOS: - exts.append( - Extension('_scproxy', ['_scproxy.c'], - extra_link_args=[ - '-framework', 'SystemConfiguration', - '-framework', 'CoreFoundation', - ])) - - self.extensions.extend(exts) + self.add(Extension('_scproxy', ['_scproxy.c'], + extra_link_args=[ + '-framework', 'SystemConfiguration', + '-framework', 'CoreFoundation'])) # Call the method for detecting whether _tkinter can be compiled self.detect_tkinter(inc_dirs, lib_dirs) if '_tkinter' not in [e.name for e in self.extensions]: - missing.append('_tkinter') + self.missing.append('_tkinter') # Build the _uuid module if possible uuid_incs = find_file("uuid.h", inc_dirs, ["/usr/include/uuid"]) @@ -1664,7 +1652,7 @@ class db_found(Exception): pass libraries=uuid_libs, include_dirs=uuid_incs)) else: - missing.append('_uuid') + self.missing.append('_uuid') ## # Uncomment these lines if you want to play with xxmodule.c ## ext = Extension('xx', ['xxmodule.c']) @@ -1675,8 +1663,6 @@ class db_found(Exception): pass define_macros=[('Py_LIMITED_API', '0x03050000')]) self.extensions.append(ext) - return missing - def detect_tkinter_explicitly(self): # Build _tkinter using explicit locations for Tcl/Tk. # @@ -2035,7 +2021,7 @@ def detect_ctypes(self, inc_dirs, lib_dirs): # for dlopen, see bpo-32647 ext.libraries.append('dl') - def _decimal_ext(self): + def _detect_decimal(self): extra_compile_args = [] undef_macros = [] if '--with-system-libmpdec' in sysconfig.get_config_var("CONFIG_ARGS"): @@ -2142,17 +2128,14 @@ def _decimal_ext(self): # Uncomment for extra functionality: #define_macros.append(('EXTRA_FUNCTIONALITY', 1)) - ext = Extension ( - '_decimal', - include_dirs=include_dirs, - libraries=libraries, - define_macros=define_macros, - undef_macros=undef_macros, - extra_compile_args=extra_compile_args, - sources=sources, - depends=depends - ) - return ext + self.add(Extension('_decimal', + include_dirs=include_dirs, + libraries=libraries, + define_macros=define_macros, + undef_macros=undef_macros, + extra_compile_args=extra_compile_args, + sources=sources, + depends=depends)) def _detect_openssl(self, inc_dirs, lib_dirs): config_vars = sysconfig.get_config_vars() @@ -2191,29 +2174,24 @@ def split_var(name, sep): ssl_incs.extend(krb5_h) if config_vars.get("HAVE_X509_VERIFY_PARAM_SET1_HOST"): - ssl_ext = Extension( - '_ssl', ['_ssl.c'], - include_dirs=openssl_includes, - library_dirs=openssl_libdirs, - libraries=openssl_libs, - depends=['socketmodule.h'] - ) + self.add(Extension('_ssl', ['_ssl.c'], + include_dirs=openssl_includes, + library_dirs=openssl_libdirs, + libraries=openssl_libs, + depends=['socketmodule.h'])) else: - ssl_ext = None - - hashlib_ext = Extension( - '_hashlib', ['_hashopenssl.c'], - depends=['hashlib.h'], - include_dirs=openssl_includes, - library_dirs=openssl_libdirs, - libraries=openssl_libs, - ) + self.missing.append('_ssl') - return ssl_ext, hashlib_ext + self.add(Extension('_hashlib', ['_hashopenssl.c'], + depends=['hashlib.h'], + include_dirs=openssl_includes, + library_dirs=openssl_libdirs, + libraries=openssl_libs)) def _detect_nis(self, inc_dirs, lib_dirs): if MS_WINDOWS or CYGWIN or HOST_PLATFORM == 'qnx6': - return None + self.missing.append('nis') + return libs = [] library_dirs = [] @@ -2232,7 +2210,8 @@ def _detect_nis(self, inc_dirs, lib_dirs): ) if rpcsvc_inc is None or rpc_inc is None: # not found - return None + self.missing.append('nis') + return includes_dirs.extend(rpcsvc_inc) includes_dirs.extend(rpc_inc) @@ -2249,12 +2228,10 @@ def _detect_nis(self, inc_dirs, lib_dirs): if self.compiler.find_library_file(lib_dirs, 'tirpc'): libs.append('tirpc') - return Extension( - 'nis', ['nismodule.c'], - libraries=libs, - library_dirs=library_dirs, - include_dirs=includes_dirs - ) + self.add(Extension('nis', ['nismodule.c'], + libraries=libs, + library_dirs=library_dirs, + include_dirs=includes_dirs)) class PyBuildInstall(install):

1 0

bpo-36146: Fix inc_dirs in setup.py on macOS (GH-12098)
by Victor Stinner March 1, 2019

March 1, 2019

https://github.com/python/cpython/commit/96d81583be98cec9728636186ea32b662c… commit: 96d81583be98cec9728636186ea32b662cb091d5 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2019-03-01T13:53:46+01:00 summary: bpo-36146: Fix inc_dirs in setup.py on macOS (GH-12098) Fix setup.py on macOS: only add /usr/include/ffi to include directories of _ctypes, not for all extensions. files: A Misc/NEWS.d/next/Build/2019-02-28-18-09-01.bpo-36146.IwPJVT.rst M setup.py diff --git a/Misc/NEWS.d/next/Build/2019-02-28-18-09-01.bpo-36146.IwPJVT.rst b/Misc/NEWS.d/next/Build/2019-02-28-18-09-01.bpo-36146.IwPJVT.rst new file mode 100644 index 000000000000..93c1e1afac87 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2019-02-28-18-09-01.bpo-36146.IwPJVT.rst @@ -0,0 +1,2 @@ +Fix setup.py on macOS: only add ``/usr/include/ffi`` to include +directories of _ctypes, not for all extensions. diff --git a/setup.py b/setup.py index 2fef0ad468a4..56a1df327399 100644 --- a/setup.py +++ b/setup.py @@ -2003,16 +2003,17 @@ def detect_ctypes(self, inc_dirs, lib_dirs): libraries=['m']) self.extensions.extend([ext, ext_test]) + ffi_inc_dirs = inc_dirs.copy() if MACOS: if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"): return # OS X 10.5 comes with libffi.dylib; the include files are # in /usr/include/ffi - inc_dirs.append('/usr/include/ffi') + ffi_inc_dirs.append('/usr/include/ffi') ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")] if not ffi_inc or ffi_inc[0] == '': - ffi_inc = find_file('ffi.h', [], inc_dirs) + ffi_inc = find_file('ffi.h', [], ffi_inc_dirs) if ffi_inc is not None: ffi_h = ffi_inc[0] + '/ffi.h' if not os.path.exists(ffi_h):

1 0

bpo-36142: Remove _PyMain structure (GH-12120)
by Victor Stinner March 1, 2019

March 1, 2019

https://github.com/python/cpython/commit/62be763348d16ba90f96667aa024050326… commit: 62be763348d16ba90f96667aa0240503261393f0 branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2019-03-01T13:10:14+01:00 summary: bpo-36142: Remove _PyMain structure (GH-12120) * Move fields from _PyMain to _PyCoreConfig: * skip_first_line * run_command * run_module * run_filename * Replace _PyMain.stdin_is_interactive with a new stdin_is_interactive(config) function * Rename _PyMain to _PyArgv. Add "const _PyArgv *args" field to _PyCmdline. files: M Include/cpython/coreconfig.h M Lib/test/test_embed.py M Modules/main.c M Python/coreconfig.c diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h index 2ce99b2eab64..4985bf3a7598 100644 --- a/Include/cpython/coreconfig.h +++ b/Include/cpython/coreconfig.h @@ -276,7 +276,19 @@ typedef struct { int legacy_windows_stdio; #endif - /* --- Private fields -------- */ + /* --- Parameter only used by Py_Main() ---------- */ + + /* Skip the first line of the source ('run_filename' parameter), allowing use of non-Unix forms of + "#!cmd". This is intended for a DOS specific hack only. + + Set by the -x command line option. */ + int skip_source_first_line; + + wchar_t *run_command; /* -c command line argument */ + wchar_t *run_module; /* -m command line argument */ + wchar_t *run_filename; /* Trailing command line argument without -c or -m */ + + /* --- Private fields ---------------------------- */ /* Install importlib? If set to 0, importlib is not initialized at all. Needed by freeze_importlib. */ diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d35b9f4d5a2f..6c245ebcd194 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -322,6 +322,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'stdio_encoding': GET_DEFAULT_CONFIG, 'stdio_errors': GET_DEFAULT_CONFIG, + 'skip_source_first_line': 0, + 'run_command': None, + 'run_module': None, + 'run_filename': None, + '_install_importlib': 1, '_check_hash_pycs_mode': 'default', '_frozen': 0, diff --git a/Modules/main.c b/Modules/main.c index 0d76953a5cae..e7d75a79e787 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -312,6 +312,14 @@ pymain_run_command(wchar_t *command, PyCompilerFlags *cf) /* Main program */ typedef struct { + int argc; + int use_bytes_argv; + char **bytes_argv; + wchar_t **wchar_argv; +} _PyArgv; + +typedef struct { + const _PyArgv *args; wchar_t **argv; int nwarnoption; /* Number of -W command line options */ wchar_t **warnoptions; /* Command line -W options */ @@ -321,47 +329,33 @@ typedef struct { int print_version; /* -V option */ } _PyCmdline; -/* Structure used by Py_Main() to pass data to subfunctions */ -typedef struct { - /* Input arguments */ - int argc; - int use_bytes_argv; - char **bytes_argv; - wchar_t **wchar_argv; - - /* non-zero is stdin is a TTY or if -i option is used */ - int stdin_is_interactive; - int skip_first_line; /* -x option */ - wchar_t *filename; /* Trailing arg without -c or -m */ - wchar_t *command; /* -c argument */ - wchar_t *module; /* -m argument */ -} _PyMain; /* Non-zero if filename, command (-c) or module (-m) is set on the command line */ -#define RUN_CODE(pymain) \ - (pymain->command != NULL || pymain->filename != NULL \ - || pymain->module != NULL) +#define RUN_CODE(config) \ + (config->run_command != NULL || config->run_filename != NULL \ + || config->run_module != NULL) static _PyInitError -pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) +pymain_init_cmdline_argv(_PyCoreConfig *config, _PyCmdline *cmdline) { assert(cmdline->argv == NULL); - if (pymain->use_bytes_argv) { + const _PyArgv *args = cmdline->args; + + if (args->use_bytes_argv) { /* +1 for a the NULL terminator */ - size_t size = sizeof(wchar_t*) * (pymain->argc + 1); + size_t size = sizeof(wchar_t*) * (args->argc + 1); wchar_t** argv = (wchar_t **)PyMem_RawMalloc(size); if (argv == NULL) { return _Py_INIT_NO_MEMORY(); } - for (int i = 0; i < pymain->argc; i++) { + for (int i = 0; i < args->argc; i++) { size_t len; - wchar_t *arg = Py_DecodeLocale(pymain->bytes_argv[i], &len); + wchar_t *arg = Py_DecodeLocale(args->bytes_argv[i], &len); if (arg == NULL) { _Py_wstrlist_clear(i, argv); return DECODE_LOCALE_ERR("command line arguments", @@ -369,16 +363,16 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, } argv[i] = arg; } - argv[pymain->argc] = NULL; + argv[args->argc] = NULL; cmdline->argv = argv; } else { - cmdline->argv = pymain->wchar_argv; + cmdline->argv = args->wchar_argv; } wchar_t *program; - if (pymain->argc >= 1 && cmdline->argv != NULL) { + if (args->argc >= 1 && cmdline->argv != NULL) { program = cmdline->argv[0]; } else { @@ -394,7 +388,7 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, static void -pymain_clear_cmdline(_PyMain *pymain, _PyCmdline *cmdline) +pymain_clear_cmdline(_PyCmdline *cmdline) { PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -407,8 +401,8 @@ pymain_clear_cmdline(_PyMain *pymain, _PyCmdline *cmdline) cmdline->nenv_warnoption = 0; cmdline->env_warnoptions = NULL; - if (pymain->use_bytes_argv && cmdline->argv != NULL) { - _Py_wstrlist_clear(pymain->argc, cmdline->argv); + if (cmdline->args->use_bytes_argv && cmdline->argv != NULL) { + _Py_wstrlist_clear(cmdline->args->argc, cmdline->argv); } cmdline->argv = NULL; @@ -416,21 +410,6 @@ pymain_clear_cmdline(_PyMain *pymain, _PyCmdline *cmdline) } -static void -pymain_clear_pymain(_PyMain *pymain) -{ -#define CLEAR(ATTR) \ - do { \ - PyMem_RawFree(ATTR); \ - ATTR = NULL; \ - } while (0) - - CLEAR(pymain->filename); - CLEAR(pymain->command); - CLEAR(pymain->module); -#undef CLEAR -} - static void pymain_clear_config(_PyCoreConfig *config) { @@ -446,7 +425,7 @@ pymain_clear_config(_PyCoreConfig *config) static void -pymain_free(_PyMain *pymain) +pymain_free(void) { _PyImport_Fini2(); @@ -461,8 +440,6 @@ pymain_free(_PyMain *pymain) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - pymain_clear_pymain(pymain); - _Py_wstrlist_clear(orig_argc, orig_argv); orig_argc = 0; orig_argv = NULL; @@ -542,14 +519,14 @@ _Py_wstrlist_append(int *len, wchar_t ***list, const wchar_t *str) /* Parse the command line arguments */ static _PyInitError -pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline, int *need_usage) +pymain_parse_cmdline_impl(_PyCoreConfig *config, _PyCmdline *cmdline, + int *need_usage) { _PyInitError err; _PyOS_ResetGetOpt(); do { int longindex = -1; - int c = _PyOS_GetOpt(pymain->argc, cmdline->argv, PROGRAM_OPTS, + int c = _PyOS_GetOpt(cmdline->args->argc, cmdline->argv, PROGRAM_OPTS, longoptions, &longindex); if (c == EOF) { break; @@ -567,7 +544,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, memcpy(command, _PyOS_optarg, (len - 2) * sizeof(wchar_t)); command[len - 2] = '\n'; command[len - 1] = 0; - pymain->command = command; + config->run_command = command; break; } @@ -575,8 +552,8 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, /* -m is the last option; following arguments that look like options are left for the module to interpret. */ - pymain->module = _PyMem_RawWcsdup(_PyOS_optarg); - if (pymain->module == NULL) { + config->run_module = _PyMem_RawWcsdup(_PyOS_optarg); + if (config->run_module == NULL) { return _Py_INIT_NO_MEMORY(); } break; @@ -652,7 +629,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, break; case 'x': - pymain->skip_first_line = 1; + config->skip_source_first_line = 1; break; case 'h': @@ -699,23 +676,23 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, } } while (1); - if (pymain->command == NULL && pymain->module == NULL - && _PyOS_optind < pymain->argc + if (config->run_command == NULL && config->run_module == NULL + && _PyOS_optind < cmdline->args->argc && wcscmp(cmdline->argv[_PyOS_optind], L"-") != 0) { - pymain->filename = _PyMem_RawWcsdup(cmdline->argv[_PyOS_optind]); - if (pymain->filename == NULL) { + config->run_filename = _PyMem_RawWcsdup(cmdline->argv[_PyOS_optind]); + if (config->run_filename == NULL) { return _Py_INIT_NO_MEMORY(); } } - if (pymain->command != NULL || pymain->module != NULL) { + if (config->run_command != NULL || config->run_module != NULL) { /* Backup _PyOS_optind */ _PyOS_optind--; } /* -c and -m options are exclusive */ - assert(!(pymain->command != NULL && pymain->module != NULL)); + assert(!(config->run_command != NULL && config->run_module != NULL)); return _Py_INIT_OK(); } @@ -860,8 +837,7 @@ config_init_warnoptions(_PyCoreConfig *config, _PyCmdline *cmdline) /* Get warning options from PYTHONWARNINGS environment variable. */ static _PyInitError -cmdline_init_env_warnoptions(_PyMain *pymain, const _PyCoreConfig *config, - _PyCmdline *cmdline) +cmdline_init_env_warnoptions(const _PyCoreConfig *config, _PyCmdline *cmdline) { wchar_t *env; int res = _PyCoreConfig_GetEnvDup(config, &env, @@ -893,12 +869,17 @@ cmdline_init_env_warnoptions(_PyMain *pymain, const _PyCoreConfig *config, } -static void -pymain_init_stdio(_PyMain *pymain, _PyCoreConfig *config) +/* Return non-zero is stdin is a TTY or if -i command line option is used */ +static int +stdin_is_interactive(const _PyCoreConfig *config) { - pymain->stdin_is_interactive = (isatty(fileno(stdin)) - || config->interactive); + return (isatty(fileno(stdin)) || config->interactive); +} + +static void +pymain_init_stdio(_PyCoreConfig *config) +{ #if defined(MS_WINDOWS) || defined(__CYGWIN__) /* don't translate newlines (\r\n <=> \n) */ _setmode(fileno(stdin), O_BINARY); @@ -934,13 +915,13 @@ pymain_init_stdio(_PyMain *pymain, _PyCoreConfig *config) static void -pymain_header(_PyMain *pymain, const _PyCoreConfig *config) +pymain_header(const _PyCoreConfig *config) { if (config->quiet) { return; } - if (!config->verbose && (RUN_CODE(pymain) || !pymain->stdin_is_interactive)) { + if (!config->verbose && (RUN_CODE(config) || !stdin_is_interactive(config))) { return; } @@ -952,10 +933,10 @@ pymain_header(_PyMain *pymain, const _PyCoreConfig *config) static _PyInitError -pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) +pymain_init_core_argv(_PyCoreConfig *config, _PyCmdline *cmdline) { /* Copy argv to be able to modify it (to force -c/-m) */ - int argc = pymain->argc - _PyOS_optind; + int argc = cmdline->args->argc - _PyOS_optind; wchar_t **argv; if (argc <= 0 || cmdline->argv == NULL) { @@ -973,11 +954,11 @@ pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdlin } wchar_t *arg0 = NULL; - if (pymain->command != NULL) { + if (config->run_command != NULL) { /* Force sys.argv[0] = '-c' */ arg0 = L"-c"; } - else if (pymain->module != NULL) { + else if (config->run_module != NULL) { /* Force sys.argv[0] = '-m'*/ arg0 = L"-m"; } @@ -1022,12 +1003,12 @@ _Py_wstrlist_as_pylist(int len, wchar_t **list) static void -pymain_import_readline(_PyMain *pymain, const _PyCoreConfig *config) +pymain_import_readline(const _PyCoreConfig *config) { if (config->isolated) { return; } - if (!config->inspect && RUN_CODE(pymain)) { + if (!config->inspect && RUN_CODE(config)) { return; } if (!isatty(fileno(stdin))) { @@ -1045,7 +1026,7 @@ pymain_import_readline(_PyMain *pymain, const _PyCoreConfig *config) static void -pymain_run_startup(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +pymain_run_startup(_PyCoreConfig *config, PyCompilerFlags *cf) { const char *startup = _PyCoreConfig_GetEnv(config, "PYTHONSTARTUP"); if (startup == NULL) { @@ -1071,9 +1052,9 @@ pymain_run_startup(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) static int -pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +pymain_run_file(_PyCoreConfig *config, PyCompilerFlags *cf) { - const wchar_t *filename = pymain->filename; + const wchar_t *filename = config->run_filename; FILE *fp = _Py_wfopen(filename, L"r"); if (fp == NULL) { char *cfilename_buffer; @@ -1090,7 +1071,7 @@ pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) return 2; } - if (pymain->skip_first_line) { + if (config->skip_source_first_line) { int ch; /* Push back first newline so line numbers remain the same */ while ((ch = getc(fp)) != EOF) { @@ -1141,12 +1122,12 @@ pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) static int -pymain_run_stdin(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +pymain_run_stdin(_PyCoreConfig *config, PyCompilerFlags *cf) { - if (pymain->stdin_is_interactive) { + if (stdin_is_interactive(config)) { Py_InspectFlag = 0; /* do exit on SystemExit */ config->inspect = 0; - pymain_run_startup(pymain, config, cf); + pymain_run_startup(config, cf); pymain_run_interactive_hook(); } @@ -1162,7 +1143,7 @@ pymain_run_stdin(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) static void -pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf, int *exitcode) +pymain_repl(_PyCoreConfig *config, PyCompilerFlags *cf, int *exitcode) { /* Check this environment variable at the end, to give programs the opportunity to set it from Python. */ @@ -1171,7 +1152,7 @@ pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf, int *ex config->inspect = 1; } - if (!(Py_InspectFlag && pymain->stdin_is_interactive && RUN_CODE(pymain))) { + if (!(Py_InspectFlag && stdin_is_interactive(config) && RUN_CODE(config))) { return; } @@ -1187,12 +1168,11 @@ pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf, int *ex /* Parse the command line. Handle --version and --help options directly. */ static _PyInitError -pymain_parse_cmdline(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) +pymain_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline) { int need_usage = 0; _PyInitError err; - err = pymain_parse_cmdline_impl(pymain, config, cmdline, &need_usage); + err = pymain_parse_cmdline_impl(config, cmdline, &need_usage); if (_Py_INIT_FAILED(err)) { return err; } @@ -1208,17 +1188,16 @@ pymain_parse_cmdline(_PyMain *pymain, _PyCoreConfig *config, /* Parse command line options and environment variables. This code must not use Python runtime apart PyMem_Raw memory allocator. */ static _PyInitError -pymain_read_conf_impl(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) +pymain_read_conf_impl(_PyCoreConfig *config, _PyCmdline *cmdline) { _PyInitError err; - err = pymain_parse_cmdline(pymain, config, cmdline); + err = pymain_parse_cmdline(config, cmdline); if (_Py_INIT_FAILED(err)) { return err; } - err = pymain_init_core_argv(pymain, config, cmdline); + err = pymain_init_core_argv(config, cmdline); if (_Py_INIT_FAILED(err)) { return err; } @@ -1229,7 +1208,7 @@ pymain_read_conf_impl(_PyMain *pymain, _PyCoreConfig *config, } if (config->use_environment) { - err = cmdline_init_env_warnoptions(pymain, config, cmdline); + err = cmdline_init_env_warnoptions(config, cmdline); if (_Py_INIT_FAILED(err)) { return err; } @@ -1246,8 +1225,7 @@ pymain_read_conf_impl(_PyMain *pymain, _PyCoreConfig *config, /* Read the configuration and initialize the LC_CTYPE locale: enable UTF-8 mode (PEP 540) and/or coerce the C locale (PEP 538). */ static _PyInitError -pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) +pymain_read_conf(_PyCoreConfig *config, _PyCmdline *cmdline) { _PyInitError err; int init_utf8_mode = Py_UTF8Mode; @@ -1285,12 +1263,12 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding; #endif - err = pymain_init_cmdline_argv(pymain, config, cmdline); + err = pymain_init_cmdline_argv(config, cmdline); if (_Py_INIT_FAILED(err)) { goto done; } - err = pymain_read_conf_impl(pymain, config, cmdline); + err = pymain_read_conf_impl(config, cmdline); if (_Py_INIT_FAILED(err)) { goto done; } @@ -1335,9 +1313,10 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, err = _Py_INIT_NO_MEMORY(); goto done; } - pymain_clear_cmdline(pymain, cmdline); - pymain_clear_pymain(pymain); + pymain_clear_cmdline(cmdline); + const _PyArgv *args = cmdline->args; memset(cmdline, 0, sizeof(*cmdline)); + cmdline->args = args; config->utf8_mode = new_utf8_mode; config->coerce_c_locale = new_coerce_c_locale; @@ -1554,7 +1533,7 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config, static _PyInitError -pymain_init_python_main(_PyMain *pymain, PyInterpreterState *interp) +pymain_init_python_main(PyInterpreterState *interp) { _PyInitError err; @@ -1573,19 +1552,19 @@ pymain_init_python_main(_PyMain *pymain, PyInterpreterState *interp) static _PyInitError -pymain_run_python(_PyMain *pymain, PyInterpreterState *interp, int *exitcode) +pymain_run_python(PyInterpreterState *interp, int *exitcode) { _PyInitError err; _PyCoreConfig *config = &interp->core_config; PyObject *main_importer_path = NULL; - if (pymain->filename != NULL) { + if (config->run_filename != NULL) { /* If filename is a package (ex: directory or ZIP file) which contains __main__.py, main_importer_path is set to filename and will be prepended to sys.path. Otherwise, main_importer_path is set to NULL. */ - main_importer_path = pymain_get_importer(pymain->filename); + main_importer_path = pymain_get_importer(config->run_filename); } if (main_importer_path != NULL) { @@ -1612,27 +1591,27 @@ pymain_run_python(_PyMain *pymain, PyInterpreterState *interp, int *exitcode) PyCompilerFlags cf = {.cf_flags = 0}; - pymain_header(pymain, config); - pymain_import_readline(pymain, config); + pymain_header(config); + pymain_import_readline(config); - if (pymain->command) { - *exitcode = pymain_run_command(pymain->command, &cf); + if (config->run_command) { + *exitcode = pymain_run_command(config->run_command, &cf); } - else if (pymain->module) { - *exitcode = (pymain_run_module(pymain->module, 1) != 0); + else if (config->run_module) { + *exitcode = (pymain_run_module(config->run_module, 1) != 0); } else if (main_importer_path != NULL) { int sts = pymain_run_module(L"__main__", 0); *exitcode = (sts != 0); } - else if (pymain->filename != NULL) { - *exitcode = pymain_run_file(pymain, config, &cf); + else if (config->run_filename != NULL) { + *exitcode = pymain_run_file(config, &cf); } else { - *exitcode = pymain_run_stdin(pymain, config, &cf); + *exitcode = pymain_run_stdin(config, &cf); } - pymain_repl(pymain, config, &cf, exitcode); + pymain_repl(config, &cf, exitcode); err = _Py_INIT_OK(); done: @@ -1642,8 +1621,7 @@ pymain_run_python(_PyMain *pymain, PyInterpreterState *interp, int *exitcode) static _PyInitError -pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) +pymain_cmdline_impl(_PyCoreConfig *config, _PyCmdline *cmdline) { _PyInitError err; @@ -1652,7 +1630,7 @@ pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, return err; } - err = pymain_read_conf(pymain, config, cmdline); + err = pymain_read_conf(config, cmdline); if (_Py_INIT_FAILED(err)) { return err; } @@ -1669,11 +1647,11 @@ pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, } /* For Py_GetArgcArgv(). Cleared by pymain_free(). */ - orig_argv = _Py_wstrlist_copy(pymain->argc, cmdline->argv); + orig_argv = _Py_wstrlist_copy(cmdline->args->argc, cmdline->argv); if (orig_argv == NULL) { return _Py_INIT_NO_MEMORY(); } - orig_argc = pymain->argc; + orig_argc = cmdline->args->argc; return _Py_INIT_OK(); } @@ -1690,7 +1668,7 @@ pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline is a temporary structure used to prioritize these variables. */ static _PyInitError -pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) +pymain_cmdline(_PyArgv *args, _PyCoreConfig *config) { /* Force default allocator, since pymain_free() and pymain_clear_config() must use the same allocator than this function. */ @@ -1703,10 +1681,11 @@ pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) _PyCmdline cmdline; memset(&cmdline, 0, sizeof(cmdline)); + cmdline.args = args; - _PyInitError err = pymain_cmdline_impl(pymain, config, &cmdline); + _PyInitError err = pymain_cmdline_impl(config, &cmdline); - pymain_clear_cmdline(pymain, &cmdline); + pymain_clear_cmdline(&cmdline); #ifdef Py_DEBUG /* Make sure that PYMEM_DOMAIN_RAW has not been modified */ @@ -1720,7 +1699,7 @@ pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) static _PyInitError -pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) +pymain_init(_PyArgv *args, PyInterpreterState **interp_p) { _PyInitError err; @@ -1738,14 +1717,14 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) _PyCoreConfig_GetGlobalConfig(config); - err = pymain_cmdline(pymain, config); + err = pymain_cmdline(args, config); if (_Py_INIT_FAILED(err)) { goto done; } _PyCoreConfig_SetGlobalConfig(config); - pymain_init_stdio(pymain, config); + pymain_init_stdio(config); PyInterpreterState *interp; err = _Py_InitializeCore(&interp, config); @@ -1754,7 +1733,7 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) } *interp_p = interp; - err = pymain_init_python_main(pymain, interp); + err = pymain_init_python_main(interp); if (_Py_INIT_FAILED(err)) { goto done; } @@ -1768,18 +1747,18 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) static int -pymain_main(_PyMain *pymain) +pymain_main(_PyArgv *args) { _PyInitError err; PyInterpreterState *interp; - err = pymain_init(pymain, &interp); + err = pymain_init(args, &interp); if (_Py_INIT_FAILED(err)) { _Py_ExitInitError(err); } int exitcode = 0; - err = pymain_run_python(pymain, interp, &exitcode); + err = pymain_run_python(interp, &exitcode); if (_Py_INIT_FAILED(err)) { _Py_ExitInitError(err); } @@ -1790,7 +1769,7 @@ pymain_main(_PyMain *pymain) exitcode = 120; } - pymain_free(pymain); + pymain_free(); if (_Py_UnhandledKeyboardInterrupt) { /* https://bugs.python.org/issue1054041 - We need to exit via the @@ -1822,28 +1801,24 @@ pymain_main(_PyMain *pymain) int Py_Main(int argc, wchar_t **argv) { - _PyMain pymain; - memset(&pymain, 0, sizeof(pymain)); - - pymain.use_bytes_argv = 0; - pymain.argc = argc; - pymain.wchar_argv = argv; - - return pymain_main(&pymain); + _PyArgv args = { + .argc = argc, + .use_bytes_argv = 0, + .bytes_argv = NULL, + .wchar_argv = argv}; + return pymain_main(&args); } int _Py_UnixMain(int argc, char **argv) { - _PyMain pymain; - memset(&pymain, 0, sizeof(pymain)); - - pymain.use_bytes_argv = 1; - pymain.argc = argc; - pymain.bytes_argv = argv; - - return pymain_main(&pymain); + _PyArgv args = { + .argc = argc, + .use_bytes_argv = 1, + .bytes_argv = argv, + .wchar_argv = NULL}; + return pymain_main(&args); } diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 06dbada6dfb6..d9b30130c931 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -424,6 +424,10 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) COPY_ATTR(legacy_windows_fs_encoding); COPY_ATTR(legacy_windows_stdio); #endif + COPY_ATTR(skip_source_first_line); + COPY_WSTR_ATTR(run_command); + COPY_WSTR_ATTR(run_module); + COPY_WSTR_ATTR(run_filename); COPY_ATTR(_check_hash_pycs_mode); COPY_ATTR(_frozen); @@ -1559,6 +1563,10 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) SET_ITEM_INT(legacy_windows_fs_encoding); SET_ITEM_INT(legacy_windows_stdio); #endif + SET_ITEM_INT(skip_source_first_line); + SET_ITEM_WSTR(run_command); + SET_ITEM_WSTR(run_module); + SET_ITEM_WSTR(run_filename); SET_ITEM_INT(_install_importlib); SET_ITEM_STR(_check_hash_pycs_mode); SET_ITEM_INT(_frozen);

1 0

bpo-36142: Rework error reporting in pymain_main() (GH-12113)
by Victor Stinner March 1, 2019

March 1, 2019

https://github.com/python/cpython/commit/dfe884759d1f4441c889695f8985bc9feb… commit: dfe884759d1f4441c889695f8985bc9feb9f37eb branch: master author: Victor Stinner <vstinner(a)redhat.com> committer: GitHub <noreply(a)github.com> date: 2019-03-01T12:14:41+01:00 summary: bpo-36142: Rework error reporting in pymain_main() (GH-12113) Add a new _Py_INIT_EXIT() macro to be able to exit Python with an exitcode using _PyInitError API. Rewrite function calls by pymain_main() to use _PyInitError. Changes: * Remove _PyMain.err and _PyMain.status field * Add _Py_INIT_EXIT() macro and _PyInitError.exitcode field. * Rename _Py_FatalInitError() to _Py_ExitInitError(). files: M Include/cpython/coreconfig.h M Include/cpython/pylifecycle.h M Modules/main.c M Programs/_freeze_importlib.c M Programs/_testembed.c M Python/frozenmain.c M Python/pathconfig.c M Python/pylifecycle.c diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h index f7ac3f9cff09..2ce99b2eab64 100644 --- a/Include/cpython/coreconfig.h +++ b/Include/cpython/coreconfig.h @@ -11,6 +11,7 @@ typedef struct { const char *prefix; const char *msg; int user_err; + int exitcode; } _PyInitError; /* Almost all errors causing Python initialization to fail */ @@ -22,16 +23,18 @@ typedef struct { #endif #define _Py_INIT_OK() \ - (_PyInitError){.prefix = NULL, .msg = NULL, .user_err = 0} + (_PyInitError){.prefix = NULL, .msg = NULL, .user_err = 0, .exitcode = -1} #define _Py_INIT_ERR(MSG) \ - (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 0} + (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 0, .exitcode = -1} /* Error that can be fixed by the user like invalid input parameter. Don't abort() the process on such error. */ #define _Py_INIT_USER_ERR(MSG) \ - (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1} + (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1, .exitcode = -1} #define _Py_INIT_NO_MEMORY() _Py_INIT_USER_ERR("memory allocation failed") +#define _Py_INIT_EXIT(EXITCODE) \ + (_PyInitError){.prefix = NULL, .msg = NULL, .user_err = 0, .exitcode = (EXITCODE)} #define _Py_INIT_FAILED(err) \ - (err.msg != NULL) + (err.msg != NULL || err.exitcode != -1) /* _PyCoreConfig */ diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index a3fdeefde01b..3bffc80c9376 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -39,7 +39,7 @@ PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter( PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig( const _PyCoreConfig *config); -PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalInitError(_PyInitError err); +PyAPI_FUNC(void) _Py_NO_RETURN _Py_ExitInitError(_PyInitError err); /* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level * exit functions. diff --git a/Modules/main.c b/Modules/main.c index f373dab20890..0d76953a5cae 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -329,11 +329,6 @@ typedef struct { char **bytes_argv; wchar_t **wchar_argv; - /* Exit status or "exit code": result of pymain_main() */ - int status; - /* Error message if a function failed */ - _PyInitError err; - /* non-zero is stdin is a TTY or if -i option is used */ int stdin_is_interactive; int skip_first_line; /* -x option */ @@ -342,9 +337,6 @@ typedef struct { wchar_t *module; /* -m argument */ } _PyMain; -#define _PyMain_INIT {.err = _Py_INIT_OK()} -/* Note: _PyMain_INIT sets other fields to 0/NULL */ - /* Non-zero if filename, command (-c) or module (-m) is set on the command line */ @@ -353,19 +345,7 @@ typedef struct { || pymain->module != NULL) -static wchar_t* -pymain_wstrdup(_PyMain *pymain, const wchar_t *str) -{ - wchar_t *str2 = _PyMem_RawWcsdup(str); - if (str2 == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return NULL; - } - return str2; -} - - -static int +static _PyInitError pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) { @@ -376,8 +356,7 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, size_t size = sizeof(wchar_t*) * (pymain->argc + 1); wchar_t** argv = (wchar_t **)PyMem_RawMalloc(size); if (argv == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; + return _Py_INIT_NO_MEMORY(); } for (int i = 0; i < pymain->argc; i++) { @@ -385,9 +364,8 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, wchar_t *arg = Py_DecodeLocale(pymain->bytes_argv[i], &len); if (arg == NULL) { _Py_wstrlist_clear(i, argv); - pymain->err = DECODE_LOCALE_ERR("command line arguments", - (Py_ssize_t)len); - return -1; + return DECODE_LOCALE_ERR("command line arguments", + (Py_ssize_t)len); } argv[i] = arg; } @@ -406,12 +384,12 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, else { program = L""; } - config->program = pymain_wstrdup(pymain, program); + config->program = _PyMem_RawWcsdup(program); if (config->program == NULL) { - return -1; + return _Py_INIT_NO_MEMORY(); } - return 0; + return _Py_INIT_OK(); } @@ -562,26 +540,12 @@ _Py_wstrlist_append(int *len, wchar_t ***list, const wchar_t *str) } -static int -pymain_wstrlist_append(_PyMain *pymain, int *len, wchar_t ***list, const wchar_t *str) -{ - _PyInitError err = _Py_wstrlist_append(len, list, str); - if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; - } - return 0; -} - - -/* Parse the command line arguments - Return 0 on success. - Return 1 if parsing failed. - Set pymain->err and return -1 on other errors. */ -static int +/* Parse the command line arguments */ +static _PyInitError pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, - _PyCmdline *cmdline) + _PyCmdline *cmdline, int *need_usage) { + _PyInitError err; _PyOS_ResetGetOpt(); do { int longindex = -1; @@ -598,8 +562,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, size_t len = wcslen(_PyOS_optarg) + 1 + 1; wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); if (command == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; + return _Py_INIT_NO_MEMORY(); } memcpy(command, _PyOS_optarg, (len - 2) * sizeof(wchar_t)); command[len - 2] = '\n'; @@ -612,9 +575,9 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, /* -m is the last option; following arguments that look like options are left for the module to interpret. */ - pymain->module = pymain_wstrdup(pymain, _PyOS_optarg); + pymain->module = _PyMem_RawWcsdup(_PyOS_optarg); if (pymain->module == NULL) { - return -1; + return _Py_INIT_NO_MEMORY(); } break; } @@ -632,7 +595,8 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, } else { fprintf(stderr, "--check-hash-based-pycs must be one of " "'default', 'always', or 'never'\n"); - return 1; + *need_usage = 1; + return _Py_INIT_OK(); } break; @@ -701,20 +665,20 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, break; case 'W': - if (pymain_wstrlist_append(pymain, - &cmdline->nwarnoption, - &cmdline->warnoptions, - _PyOS_optarg) < 0) { - return -1; + err = _Py_wstrlist_append(&cmdline->nwarnoption, + &cmdline->warnoptions, + _PyOS_optarg); + if (_Py_INIT_FAILED(err)) { + return err; } break; case 'X': - if (pymain_wstrlist_append(pymain, - &config->nxoption, - &config->xoptions, - _PyOS_optarg) < 0) { - return -1; + err = _Py_wstrlist_append(&config->nxoption, + &config->xoptions, + _PyOS_optarg); + if (_Py_INIT_FAILED(err)) { + return err; } break; @@ -730,7 +694,8 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, default: /* unknown argument: parsing failed */ - return 1; + *need_usage = 1; + return _Py_INIT_OK(); } } while (1); @@ -738,16 +703,21 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, && _PyOS_optind < pymain->argc && wcscmp(cmdline->argv[_PyOS_optind], L"-") != 0) { - pymain->filename = pymain_wstrdup(pymain, cmdline->argv[_PyOS_optind]); + pymain->filename = _PyMem_RawWcsdup(cmdline->argv[_PyOS_optind]); if (pymain->filename == NULL) { - return -1; + return _Py_INIT_NO_MEMORY(); } } + if (pymain->command != NULL || pymain->module != NULL) { + /* Backup _PyOS_optind */ + _PyOS_optind--; + } + /* -c and -m options are exclusive */ assert(!(pymain->command != NULL && pymain->module != NULL)); - return 0; + return _Py_INIT_OK(); } @@ -888,9 +858,7 @@ config_init_warnoptions(_PyCoreConfig *config, _PyCmdline *cmdline) } -/* Get warning options from PYTHONWARNINGS environment variable. - Return 0 on success. - Set pymain->err and return -1 on error. */ +/* Get warning options from PYTHONWARNINGS environment variable. */ static _PyInitError cmdline_init_env_warnoptions(_PyMain *pymain, const _PyCoreConfig *config, _PyCmdline *cmdline) @@ -983,7 +951,7 @@ pymain_header(_PyMain *pymain, const _PyCoreConfig *config) } -static int +static _PyInitError pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) { /* Copy argv to be able to modify it (to force -c/-m) */ @@ -1001,8 +969,7 @@ pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdlin } if (argv == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; + return _Py_INIT_NO_MEMORY(); } wchar_t *arg0 = NULL; @@ -1018,8 +985,7 @@ pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdlin arg0 = _PyMem_RawWcsdup(arg0); if (arg0 == NULL) { _Py_wstrlist_clear(argc, argv); - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; + return _Py_INIT_NO_MEMORY(); } assert(argc >= 1); @@ -1029,7 +995,7 @@ pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdlin config->argc = argc; config->argv = argv; - return 0; + return _Py_INIT_OK(); } @@ -1104,7 +1070,7 @@ pymain_run_startup(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) } -static void +static int pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) { const wchar_t *filename = pymain->filename; @@ -1121,8 +1087,7 @@ pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) fprintf(stderr, "%ls: can't open file '%s': [Errno %d] %s\n", config->program, cfilename, err, strerror(err)); PyMem_RawFree(cfilename_buffer); - pymain->status = 2; - return; + return 2; } if (pymain->skip_first_line) { @@ -1141,17 +1106,15 @@ pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) fprintf(stderr, "%ls: '%ls' is a directory, cannot continue\n", config->program, filename); - pymain->status = 1; fclose(fp); - return; + return 1; } /* call pending calls like signal handlers (SIGINT) */ if (Py_MakePendingCalls() == -1) { PyErr_Print(); - pymain->status = 1; fclose(fp); - return; + return 1; } PyObject *unicode, *bytes = NULL; @@ -1173,11 +1136,11 @@ pymain_run_file(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) /* PyRun_AnyFileExFlags(closeit=1) calls fclose(fp) before running code */ int run = PyRun_AnyFileExFlags(fp, filename_str, 1, cf); Py_XDECREF(bytes); - pymain->status = (run != 0); + return (run != 0); } -static void +static int pymain_run_stdin(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) { if (pymain->stdin_is_interactive) { @@ -1190,17 +1153,16 @@ pymain_run_stdin(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) /* call pending calls like signal handlers (SIGINT) */ if (Py_MakePendingCalls() == -1) { PyErr_Print(); - pymain->status = 1; - return; + return 1; } int run = PyRun_AnyFileExFlags(stdin, "<stdin>", 0, cf); - pymain->status = (run != 0); + return (run != 0); } static void -pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) +pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf, int *exitcode) { /* Check this environment variable at the end, to give programs the opportunity to set it from Python. */ @@ -1218,100 +1180,86 @@ pymain_repl(_PyMain *pymain, _PyCoreConfig *config, PyCompilerFlags *cf) pymain_run_interactive_hook(); int res = PyRun_AnyFileFlags(stdin, "<stdin>", cf); - pymain->status = (res != 0); + *exitcode = (res != 0); } /* Parse the command line. - Handle --version and --help options directly. - - Return 1 if Python must exit. - Return 0 on success. - Set pymain->err and return -1 on failure. */ -static int + Handle --version and --help options directly. */ +static _PyInitError pymain_parse_cmdline(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) { - int res = pymain_parse_cmdline_impl(pymain, config, cmdline); - if (res < 0) { - return -1; - } - if (res) { - pymain_usage(1, config->program); - pymain->status = 2; - return 1; + int need_usage = 0; + _PyInitError err; + err = pymain_parse_cmdline_impl(pymain, config, cmdline, &need_usage); + if (_Py_INIT_FAILED(err)) { + return err; } - if (pymain->command != NULL || pymain->module != NULL) { - /* Backup _PyOS_optind */ - _PyOS_optind--; + if (need_usage) { + pymain_usage(1, config->program); + return _Py_INIT_EXIT(2); } - - return 0; + return _Py_INIT_OK(); } /* Parse command line options and environment variables. - This code must not use Python runtime apart PyMem_Raw memory allocator. - - Return 0 on success. - Return 1 if Python is done and must exit. - Set pymain->err and return -1 on error. */ -static int + This code must not use Python runtime apart PyMem_Raw memory allocator. */ +static _PyInitError pymain_read_conf_impl(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) { _PyInitError err; - int res = pymain_parse_cmdline(pymain, config, cmdline); - if (res != 0) { - return res; + err = pymain_parse_cmdline(pymain, config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; } - if (pymain_init_core_argv(pymain, config, cmdline) < 0) { - return -1; + err = pymain_init_core_argv(pymain, config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; } err = _PyCoreConfig_Read(config); if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; + return err; } if (config->use_environment) { err = cmdline_init_env_warnoptions(pymain, config, cmdline); if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; + return err; } } err = config_init_warnoptions(config, cmdline); if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; + return err; } - return 0; + return _Py_INIT_OK(); } /* Read the configuration and initialize the LC_CTYPE locale: enable UTF-8 mode (PEP 540) and/or coerce the C locale (PEP 538). */ -static int +static _PyInitError pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) { + _PyInitError err; int init_utf8_mode = Py_UTF8Mode; #ifdef MS_WINDOWS int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag; #endif _PyCoreConfig save_config = _PyCoreConfig_INIT; - int res = -1; int locale_coerced = 0; int loops = 0; if (_PyCoreConfig_Copy(&save_config, config) < 0) { - pymain->err = _Py_INIT_NO_MEMORY(); + err = _Py_INIT_NO_MEMORY(); goto done; } @@ -1325,8 +1273,8 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, /* Watchdog to prevent an infinite loop */ loops++; if (loops == 3) { - pymain->err = _Py_INIT_ERR("Encoding changed twice while " - "reading the configuration"); + err = _Py_INIT_ERR("Encoding changed twice while " + "reading the configuration"); goto done; } @@ -1337,13 +1285,13 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding; #endif - if (pymain_init_cmdline_argv(pymain, config, cmdline) < 0) { + err = pymain_init_cmdline_argv(pymain, config, cmdline); + if (_Py_INIT_FAILED(err)) { goto done; } - int conf_res = pymain_read_conf_impl(pymain, config, cmdline); - if (conf_res != 0) { - res = conf_res; + err = pymain_read_conf_impl(pymain, config, cmdline); + if (_Py_INIT_FAILED(err)) { goto done; } @@ -1384,7 +1332,7 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, int new_utf8_mode = config->utf8_mode; int new_coerce_c_locale = config->coerce_c_locale; if (_PyCoreConfig_Copy(config, &save_config) < 0) { - pymain->err = _Py_INIT_NO_MEMORY(); + err = _Py_INIT_NO_MEMORY(); goto done; } pymain_clear_cmdline(pymain, cmdline); @@ -1396,7 +1344,7 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, /* The encoding changed: read again the configuration with the new encoding */ } - res = 0; + err = _Py_INIT_OK(); done: _PyCoreConfig_Clear(&save_config); @@ -1404,7 +1352,7 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, #ifdef MS_WINDOWS Py_LegacyWindowsFSEncodingFlag = init_legacy_encoding; #endif - return res; + return err; } @@ -1605,31 +1553,29 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config, } -static int -pymain_init_python_main(_PyMain *pymain, _PyCoreConfig *config, - PyInterpreterState *interp) +static _PyInitError +pymain_init_python_main(_PyMain *pymain, PyInterpreterState *interp) { _PyInitError err; _PyMainInterpreterConfig main_config = _PyMainInterpreterConfig_INIT; - err = _PyMainInterpreterConfig_Read(&main_config, config); + err = _PyMainInterpreterConfig_Read(&main_config, &interp->core_config); if (!_Py_INIT_FAILED(err)) { err = _Py_InitializeMainInterpreter(interp, &main_config); } _PyMainInterpreterConfig_Clear(&main_config); if (_Py_INIT_FAILED(err)) { - pymain->err = err; - return -1; + return err; } - return 0; + return _Py_INIT_OK(); } -static int -pymain_run_python(_PyMain *pymain, PyInterpreterState *interp) +static _PyInitError +pymain_run_python(_PyMain *pymain, PyInterpreterState *interp, int *exitcode) { - int res = 0; + _PyInitError err; _PyCoreConfig *config = &interp->core_config; PyObject *main_importer_path = NULL; @@ -1644,7 +1590,7 @@ pymain_run_python(_PyMain *pymain, PyInterpreterState *interp) if (main_importer_path != NULL) { if (pymain_sys_path_add_path0(interp, main_importer_path) < 0) { - pymain->status = 1; + err = _Py_INIT_EXIT(1); goto done; } } @@ -1652,14 +1598,13 @@ pymain_run_python(_PyMain *pymain, PyInterpreterState *interp) PyObject *path0 = _PyPathConfig_ComputeArgv0(config->argc, config->argv); if (path0 == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - res = -1; + err = _Py_INIT_NO_MEMORY(); goto done; } if (pymain_sys_path_add_path0(interp, path0) < 0) { Py_DECREF(path0); - pymain->status = 1; + err = _Py_INIT_EXIT(1); goto done; } Py_DECREF(path0); @@ -1671,67 +1616,65 @@ pymain_run_python(_PyMain *pymain, PyInterpreterState *interp) pymain_import_readline(pymain, config); if (pymain->command) { - pymain->status = pymain_run_command(pymain->command, &cf); + *exitcode = pymain_run_command(pymain->command, &cf); } else if (pymain->module) { - pymain->status = (pymain_run_module(pymain->module, 1) != 0); + *exitcode = (pymain_run_module(pymain->module, 1) != 0); } else if (main_importer_path != NULL) { int sts = pymain_run_module(L"__main__", 0); - pymain->status = (sts != 0); + *exitcode = (sts != 0); } else if (pymain->filename != NULL) { - pymain_run_file(pymain, config, &cf); + *exitcode = pymain_run_file(pymain, config, &cf); } else { - pymain_run_stdin(pymain, config, &cf); + *exitcode = pymain_run_stdin(pymain, config, &cf); } - pymain_repl(pymain, config, &cf); + pymain_repl(pymain, config, &cf, exitcode); + err = _Py_INIT_OK(); done: Py_XDECREF(main_importer_path); - return res; + return err; } -static int +static _PyInitError pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) { - pymain->err = _PyRuntime_Initialize(); - if (_Py_INIT_FAILED(pymain->err)) { - return -1; - } + _PyInitError err; - int res = pymain_read_conf(pymain, config, cmdline); - if (res < 0) { - return -1; + err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + return err; } - if (res > 0) { - /* --help or --version command: we are done */ - return 1; + + err = pymain_read_conf(pymain, config, cmdline); + if (_Py_INIT_FAILED(err)) { + return err; } if (cmdline->print_help) { pymain_usage(0, config->program); - return 1; + return _Py_INIT_EXIT(0); } if (cmdline->print_version) { printf("Python %s\n", (cmdline->print_version >= 2) ? Py_GetVersion() : PY_VERSION); - return 1; + return _Py_INIT_EXIT(0); } /* For Py_GetArgcArgv(). Cleared by pymain_free(). */ orig_argv = _Py_wstrlist_copy(pymain->argc, cmdline->argv); if (orig_argv == NULL) { - pymain->err = _Py_INIT_NO_MEMORY(); - return -1; + return _Py_INIT_NO_MEMORY(); } orig_argc = pymain->argc; - return 0; + return _Py_INIT_OK(); } @@ -1746,7 +1689,7 @@ pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline is a temporary structure used to prioritize these variables. */ -static int +static _PyInitError pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) { /* Force default allocator, since pymain_free() and pymain_clear_config() @@ -1761,7 +1704,7 @@ pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) _PyCmdline cmdline; memset(&cmdline, 0, sizeof(cmdline)); - int res = pymain_cmdline_impl(pymain, config, &cmdline); + _PyInitError err = pymain_cmdline_impl(pymain, config, &cmdline); pymain_clear_cmdline(pymain, &cmdline); @@ -1772,13 +1715,15 @@ pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) assert(memcmp(&cur_alloc, &default_alloc, sizeof(cur_alloc)) == 0); #endif PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - return res; + return err; } -static int +static _PyInitError pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) { + _PyInitError err; + /* 754 requires that FP exceptions run in "no stop" mode by default, * and until C vendors implement C99's ways to control FP exceptions, * Python requires non-stop mode. Alas, some platforms enable FP @@ -1793,13 +1738,9 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) _PyCoreConfig_GetGlobalConfig(config); - int cmd_res = pymain_cmdline(pymain, config); - if (cmd_res < 0) { - _Py_FatalInitError(pymain->err); - } - if (cmd_res == 1) { - pymain_clear_config(config); - return 1; + err = pymain_cmdline(pymain, config); + if (_Py_INIT_FAILED(err)) { + goto done; } _PyCoreConfig_SetGlobalConfig(config); @@ -1807,37 +1748,46 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) pymain_init_stdio(pymain, config); PyInterpreterState *interp; - pymain->err = _Py_InitializeCore(&interp, config); - if (_Py_INIT_FAILED(pymain->err)) { - _Py_FatalInitError(pymain->err); + err = _Py_InitializeCore(&interp, config); + if (_Py_INIT_FAILED(err)) { + goto done; } *interp_p = interp; - pymain_clear_config(config); - config = &interp->core_config; - - if (pymain_init_python_main(pymain, config, interp) < 0) { - _Py_FatalInitError(pymain->err); + err = pymain_init_python_main(pymain, interp); + if (_Py_INIT_FAILED(err)) { + goto done; } - return 0; + + err = _Py_INIT_OK(); + +done: + pymain_clear_config(config); + return err; } static int pymain_main(_PyMain *pymain) { + _PyInitError err; + PyInterpreterState *interp; - int res = pymain_init(pymain, &interp); - if (res != 1) { - if (pymain_run_python(pymain, interp) < 0) { - _Py_FatalInitError(pymain->err); - } + err = pymain_init(pymain, &interp); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } - if (Py_FinalizeEx() < 0) { - /* Value unlikely to be confused with a non-error exit status or - other special meaning */ - pymain->status = 120; - } + int exitcode = 0; + err = pymain_run_python(pymain, interp, &exitcode); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } + + if (Py_FinalizeEx() < 0) { + /* Value unlikely to be confused with a non-error exit status or + other special meaning */ + exitcode = 120; } pymain_free(pymain); @@ -1859,20 +1809,22 @@ pymain_main(_PyMain *pymain) #ifdef MS_WINDOWS /* cmd.exe detects this, prints ^C, and offers to terminate. */ /* https://msdn.microsoft.com/en-us/library/cc704588.aspx */ - pymain->status = STATUS_CONTROL_C_EXIT; + exitcode = STATUS_CONTROL_C_EXIT; #else - pymain->status = SIGINT + 128; + exitcode = SIGINT + 128; #endif /* !MS_WINDOWS */ } - return pymain->status; + return exitcode; } int Py_Main(int argc, wchar_t **argv) { - _PyMain pymain = _PyMain_INIT; + _PyMain pymain; + memset(&pymain, 0, sizeof(pymain)); + pymain.use_bytes_argv = 0; pymain.argc = argc; pymain.wchar_argv = argv; @@ -1884,7 +1836,9 @@ Py_Main(int argc, wchar_t **argv) int _Py_UnixMain(int argc, char **argv) { - _PyMain pymain = _PyMain_INIT; + _PyMain pymain; + memset(&pymain, 0, sizeof(pymain)); + pymain.use_bytes_argv = 1; pymain.argc = argc; pymain.bytes_argv = argv; diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 3f43757ecb1f..d0e40c96a7a8 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -89,7 +89,7 @@ main(int argc, char *argv[]) /* No need to call _PyCoreConfig_Clear() since we didn't allocate any memory: program_name is a constant string. */ if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } sprintf(buf, "<frozen %s>", name); diff --git a/Programs/_testembed.c b/Programs/_testembed.c index e28d94c175e6..6b5311be27ac 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -553,7 +553,7 @@ static int test_init_from_config(void) _PyInitError err = _Py_InitializeFromConfig(&config); /* Don't call _PyCoreConfig_Clear() since all strings are static */ if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } dump_config(); Py_Finalize(); @@ -618,7 +618,7 @@ static int test_init_isolated(void) test_init_env_putenvs(); _PyInitError err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } dump_config(); Py_Finalize(); @@ -635,7 +635,7 @@ static int test_init_dev_mode(void) config.program_name = L"./_testembed"; _PyInitError err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } dump_config(); Py_Finalize(); diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 616090965b13..6554aa75b038 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -86,7 +86,7 @@ Py_FrozenMain(int argc, char **argv) /* No need to call _PyCoreConfig_Clear() since we didn't allocate any memory: program_name is a constant string. */ if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } #ifdef MS_WINDOWS diff --git a/Python/pathconfig.c b/Python/pathconfig.c index eadc09baa9f1..f96f0153e210 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -408,7 +408,7 @@ pathconfig_global_init(void) error: _PyCoreConfig_Clear(&config); - _Py_FatalInitError(err); + _Py_ExitInitError(err); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 1d65d3b86f72..088e7aa9313d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -960,7 +960,7 @@ Py_InitializeEx(int install_sigs) _PyCoreConfig_Clear(&config); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } } @@ -1432,7 +1432,7 @@ Py_NewInterpreter(void) PyThreadState *tstate; _PyInitError err = new_interpreter(&tstate); if (_Py_INIT_FAILED(err)) { - _Py_FatalInitError(err); + _Py_ExitInitError(err); } return tstate; @@ -2073,12 +2073,17 @@ Py_FatalError(const char *msg) } void _Py_NO_RETURN -_Py_FatalInitError(_PyInitError err) +_Py_ExitInitError(_PyInitError err) { - /* On "user" error: exit with status 1. - For all other errors, call abort(). */ - int status = err.user_err ? 1 : -1; - fatal_error(err.prefix, err.msg, status); + if (err.exitcode >= 0) { + exit(err.exitcode); + } + else { + /* On "user" error: exit with status 1. + For all other errors, call abort(). */ + int status = err.user_err ? 1 : -1; + fatal_error(err.prefix, err.msg, status); + } } /* Clean up and exit */

1 0

bpo-36152: IDLE: Remove unused parameter from colorizer (GH-12109)
by Miss Islington (bot) March 1, 2019

March 1, 2019

https://github.com/python/cpython/commit/70852b1eb6fbcc41fe9cad042e9ca61c51… commit: 70852b1eb6fbcc41fe9cad042e9ca61c5148fbda branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2019-03-01T02:43:43-08:00 summary: bpo-36152: IDLE: Remove unused parameter from colorizer (GH-12109) Remove colorizer.ColorDelegator.close_when_done and the corresponding argument of .close(). In IDLE, both have always been None or False since 2007. (cherry picked from commit b9f0354efce95b7557bc43ea193c4b652cd28392) Co-authored-by: Cheryl Sabella <cheryl.sabella(a)gmail.com> files: A Misc/NEWS.d/next/IDLE/2019-02-28-18-52-40.bpo-36152.9pkHIU.rst M Lib/idlelib/colorizer.py M Lib/idlelib/editor.py diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index fd13281fb295..facbef8bb189 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -66,8 +66,6 @@ class ColorDelegator(Delegator): colorizing: Boolean flag when colorizing is in process. stop_colorizing: Boolean flag to end an active colorizing process. - close_when_done: Widget to destroy after colorizing process - completes (doesn't seem to be used by IDLE). """ def __init__(self): @@ -157,9 +155,7 @@ def notify_range(self, index1, index2=None): self.after_id = self.after(1, self.recolorize) return - close_when_done = None # Window to be closed when done colorizing. - - def close(self, close_when_done=None): + def close(self): if self.after_id: after_id = self.after_id self.after_id = None @@ -167,11 +163,6 @@ def close(self, close_when_done=None): self.after_cancel(after_id) self.allow_colorizing = False self.stop_colorizing = True - if close_when_done: - if not self.colorizing: - close_when_done.destroy() - else: - self.close_when_done = close_when_done def toggle_colorize_event(self, event=None): """Toggle colorizing on and off. @@ -205,9 +196,7 @@ def recolorize(self): process is not already running. After colorizing is complete, some cleanup is done to - make sure that all the text has been colorized and to close - the window if the close event had been called while the - process was running. + make sure that all the text has been colorized. """ self.after_id = None if not self.delegate: @@ -232,10 +221,6 @@ def recolorize(self): if self.allow_colorizing and self.tag_nextrange("TODO", "1.0"): if DEBUG: print("reschedule colorizing") self.after_id = self.after(1, self.recolorize) - if self.close_when_done: - top = self.close_when_done - self.close_when_done = None - top.destroy() def recolorize_main(self): "Evaluate text and apply colorizing tags." diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index e05b52a96dcc..83260329640e 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -1033,7 +1033,7 @@ def _close(self): self.io = None self.undo = None if self.color: - self.color.close(False) + self.color.close() self.color = None self.text = None self.tkinter_vars = None diff --git a/Misc/NEWS.d/next/IDLE/2019-02-28-18-52-40.bpo-36152.9pkHIU.rst b/Misc/NEWS.d/next/IDLE/2019-02-28-18-52-40.bpo-36152.9pkHIU.rst new file mode 100644 index 000000000000..b75ae89ef30a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2019-02-28-18-52-40.bpo-36152.9pkHIU.rst @@ -0,0 +1,3 @@ +Remove colorizer.ColorDelegator.close_when_done and the +corresponding argument of .close(). In IDLE, both have +always been None or False since 2007.

1 0

bpo-36152: IDLE: Remove unused parameter from colorizer (GH-12109)
by Cheryl Sabella March 1, 2019

March 1, 2019

https://github.com/python/cpython/commit/b9f0354efce95b7557bc43ea193c4b652c… commit: b9f0354efce95b7557bc43ea193c4b652cd28392 branch: master author: Cheryl Sabella <cheryl.sabella(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-03-01T05:19:40-05:00 summary: bpo-36152: IDLE: Remove unused parameter from colorizer (GH-12109) Remove colorizer.ColorDelegator.close_when_done and the corresponding argument of .close(). In IDLE, both have always been None or False since 2007. files: A Misc/NEWS.d/next/IDLE/2019-02-28-18-52-40.bpo-36152.9pkHIU.rst M Lib/idlelib/colorizer.py M Lib/idlelib/editor.py diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index fd13281fb295..facbef8bb189 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -66,8 +66,6 @@ class ColorDelegator(Delegator): colorizing: Boolean flag when colorizing is in process. stop_colorizing: Boolean flag to end an active colorizing process. - close_when_done: Widget to destroy after colorizing process - completes (doesn't seem to be used by IDLE). """ def __init__(self): @@ -157,9 +155,7 @@ def notify_range(self, index1, index2=None): self.after_id = self.after(1, self.recolorize) return - close_when_done = None # Window to be closed when done colorizing. - - def close(self, close_when_done=None): + def close(self): if self.after_id: after_id = self.after_id self.after_id = None @@ -167,11 +163,6 @@ def close(self, close_when_done=None): self.after_cancel(after_id) self.allow_colorizing = False self.stop_colorizing = True - if close_when_done: - if not self.colorizing: - close_when_done.destroy() - else: - self.close_when_done = close_when_done def toggle_colorize_event(self, event=None): """Toggle colorizing on and off. @@ -205,9 +196,7 @@ def recolorize(self): process is not already running. After colorizing is complete, some cleanup is done to - make sure that all the text has been colorized and to close - the window if the close event had been called while the - process was running. + make sure that all the text has been colorized. """ self.after_id = None if not self.delegate: @@ -232,10 +221,6 @@ def recolorize(self): if self.allow_colorizing and self.tag_nextrange("TODO", "1.0"): if DEBUG: print("reschedule colorizing") self.after_id = self.after(1, self.recolorize) - if self.close_when_done: - top = self.close_when_done - self.close_when_done = None - top.destroy() def recolorize_main(self): "Evaluate text and apply colorizing tags." diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index e05b52a96dcc..83260329640e 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -1033,7 +1033,7 @@ def _close(self): self.io = None self.undo = None if self.color: - self.color.close(False) + self.color.close() self.color = None self.text = None self.tkinter_vars = None diff --git a/Misc/NEWS.d/next/IDLE/2019-02-28-18-52-40.bpo-36152.9pkHIU.rst b/Misc/NEWS.d/next/IDLE/2019-02-28-18-52-40.bpo-36152.9pkHIU.rst new file mode 100644 index 000000000000..b75ae89ef30a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2019-02-28-18-52-40.bpo-36152.9pkHIU.rst @@ -0,0 +1,3 @@ +Remove colorizer.ColorDelegator.close_when_done and the +corresponding argument of .close(). In IDLE, both have +always been None or False since 2007.

1 0

bpo-36155: Check for identity on test_gc.test_get_objects (GH-12116)
by Pablo Galindo March 1, 2019

March 1, 2019

https://github.com/python/cpython/commit/d60a79a1015aa26ff7ee3166820ec43558… commit: d60a79a1015aa26ff7ee3166820ec43558911b60 branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2019-03-01T01:12:27-08:00 summary: bpo-36155: Check for identity on test_gc.test_get_objects (GH-12116) files: M Lib/test/test_gc.py diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 65e74d4759ce..2dab53004452 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -770,21 +770,45 @@ def test_get_objects(self): gc.collect() l = [] l.append(l) - self.assertIn(l, gc.get_objects(generation=0)) - self.assertNotIn(l, gc.get_objects(generation=1)) - self.assertNotIn(l, gc.get_objects(generation=2)) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=2)) + ) gc.collect(generation=0) - self.assertNotIn(l, gc.get_objects(generation=0)) - self.assertIn(l, gc.get_objects(generation=1)) - self.assertNotIn(l, gc.get_objects(generation=2)) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=2)) + ) gc.collect(generation=1) - self.assertNotIn(l, gc.get_objects(generation=0)) - self.assertNotIn(l, gc.get_objects(generation=1)) - self.assertIn(l, gc.get_objects(generation=2)) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=2)) + ) gc.collect(generation=2) - self.assertNotIn(l, gc.get_objects(generation=0)) - self.assertNotIn(l, gc.get_objects(generation=1)) - self.assertIn(l, gc.get_objects(generation=2)) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=2)) + ) del l gc.collect()

1 0

[3.5] bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) (#10994)
by larryhastings March 1, 2019

March 1, 2019

https://github.com/python/cpython/commit/56f8783e3e32ddc0cb84a96711e3861aea… commit: 56f8783e3e32ddc0cb84a96711e3861aea9895ac branch: 3.5 author: Alex Viscreanu <alexviscreanu(a)gmail.com> committer: larryhastings <larry(a)hastings.org> date: 2019-02-28T23:36:00-08:00 summary: [3.5] bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) (#10994) * bpo-33127: Compatibility patch for LibreSSL 2.7.0 (GH-6210) LibreSSL 2.7 introduced OpenSSL 1.1.0 API. The ssl module now detects LibreSSL 2.7 and only provides API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7. Documentation updates and fixes for failing tests will be provided in another patch set. Signed-off-by: Christian Heimes <christian(a)python.org> files: A Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst M Modules/_ssl.c diff --git a/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst new file mode 100644 index 000000000000..635aabbde031 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-03-24-15-08-24.bpo-33127.olJmHv.rst @@ -0,0 +1 @@ +The ssl module now compiles with LibreSSL 2.7.1. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index f721391d9db9..b9369b817df3 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -101,6 +101,12 @@ struct py_ssl_library_code { #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(LIBRESSL_VERSION_NUMBER) # define OPENSSL_VERSION_1_1 1 +# define PY_OPENSSL_1_1_API 1 +#endif + +/* LibreSSL 2.7.0 provides necessary OpenSSL 1.1.0 APIs */ +#if defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER >= 0x2070000fL +# define PY_OPENSSL_1_1_API 1 #endif /* Openssl comes with TLSv1.1 and TLSv1.2 between 1.0.0h and 1.0.1 @@ -129,16 +135,18 @@ struct py_ssl_library_code { #define INVALID_SOCKET (-1) #endif -#ifdef OPENSSL_VERSION_1_1 -/* OpenSSL 1.1.0+ */ -#ifndef OPENSSL_NO_SSL2 -#define OPENSSL_NO_SSL2 -#endif -#else /* OpenSSL < 1.1.0 */ -#if defined(WITH_THREAD) +/* OpenSSL 1.0.2 and LibreSSL needs extra code for locking */ +#ifndef OPENSSL_VERSION_1_1 #define HAVE_OPENSSL_CRYPTO_LOCK #endif +#if defined(OPENSSL_VERSION_1_1) && !defined(OPENSSL_NO_SSL2) +#define OPENSSL_NO_SSL2 +#endif + +#ifndef PY_OPENSSL_1_1_API +/* OpenSSL 1.1 API shims for OpenSSL < 1.1.0 and LibreSSL < 2.7.0 */ + #define TLS_method SSLv23_method static int X509_NAME_ENTRY_set(const X509_NAME_ENTRY *ne) @@ -187,7 +195,8 @@ static X509_VERIFY_PARAM *X509_STORE_get0_param(X509_STORE *store) { return store->param; } -#endif /* OpenSSL < 1.1.0 or LibreSSL */ + +#endif /* OpenSSL < 1.1.0 or LibreSSL < 2.7.0 */ enum py_ssl_error {

1 0

bpo-36018: Add documentation link to "random variable" (GH-12114)
by Miss Islington (bot) March 1, 2019

March 1, 2019

https://github.com/python/cpython/commit/9add4b3317629933d88cf206a24b15e922… commit: 9add4b3317629933d88cf206a24b15e922fa8941 branch: master author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> date: 2019-02-28T21:47:26-08:00 summary: bpo-36018: Add documentation link to "random variable" (GH-12114) https://bugs.python.org/issue36018 files: M Doc/library/statistics.rst diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 8d961b7ca5b1..8f8c0098f84a 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -471,9 +471,11 @@ A single exception is defined: :class:`NormalDist` objects =========================== -A :class:`NormalDist` is a a composite class that treats the mean and standard -deviation of data measurements as a single entity. It is a tool for creating -and manipulating normal distributions of a random variable. +:class:`NormalDist` is a tool for creating and manipulating normal +distributions of a `random variable +<http://www.stat.yale.edu/Courses/1997-98/101/ranvar.htm>`_. It is a +composite class that treats the mean and standard deviation of data +measurements as a single entity. Normal distributions arise from the `Central Limit Theorem <https://en.wikipedia.org/wiki/Central_limit_theorem>`_ and have a wide range @@ -530,7 +532,7 @@ of applications in statistics, including simulations and hypothesis testing. Using a `probability density function (pdf) <https://en.wikipedia.org/wiki/Probability_density_function>`_, - compute the relative likelihood that a random sample *X* will be near + compute the relative likelihood that a random variable *X* will be near the given value *x*. Mathematically, it is the ratio ``P(x <= X < x+dx) / dx``. @@ -544,7 +546,7 @@ of applications in statistics, including simulations and hypothesis testing. Using a `cumulative distribution function (cdf) <https://en.wikipedia.org/wiki/Cumulative_distribution_function>`_, - compute the probability that a random sample *X* will be less than or + compute the probability that a random variable *X* will be less than or equal to *x*. Mathematically, it is written ``P(X <= x)``. Instances of :class:`NormalDist` support addition, subtraction,

1 0