[Python-checkins] cpython: Issue #5309: distutils' build and build_ext commands now accept a ``-j``
antoine.pitrou
python-checkins at python.org
Fri Sep 26 23:32:14 CEST 2014
https://hg.python.org/cpython/rev/bbe57429eba0
changeset: 92592:bbe57429eba0
user: Antoine Pitrou <solipsis at pitrou.net>
date: Fri Sep 26 23:31:59 2014 +0200
summary:
Issue #5309: distutils' build and build_ext commands now accept a ``-j``
option to enable parallel building of extension modules.
files:
Lib/distutils/command/build.py | 9 ++
Lib/distutils/command/build_ext.py | 57 +++++++++++++-
Lib/distutils/tests/test_build_ext.py | 56 +++++++++-----
Misc/NEWS | 3 +
Modules/Setup.dist | 1 +
setup.py | 19 +++++
6 files changed, 117 insertions(+), 28 deletions(-)
diff --git a/Lib/distutils/command/build.py b/Lib/distutils/command/build.py
--- a/Lib/distutils/command/build.py
+++ b/Lib/distutils/command/build.py
@@ -36,6 +36,8 @@
"(default: %s)" % get_platform()),
('compiler=', 'c',
"specify the compiler type"),
+ ('parallel=', 'j',
+ "number of parallel build jobs"),
('debug', 'g',
"compile extensions and libraries with debugging information"),
('force', 'f',
@@ -65,6 +67,7 @@
self.debug = None
self.force = 0
self.executable = None
+ self.parallel = None
def finalize_options(self):
if self.plat_name is None:
@@ -116,6 +119,12 @@
if self.executable is None:
self.executable = os.path.normpath(sys.executable)
+ if isinstance(self.parallel, str):
+ try:
+ self.parallel = int(self.parallel)
+ except ValueError:
+ raise DistutilsOptionError("parallel should be an integer")
+
def run(self):
# Run all relevant sub-commands. This will be some subset of:
# - build_py - pure Python modules
diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py
--- a/Lib/distutils/command/build_ext.py
+++ b/Lib/distutils/command/build_ext.py
@@ -4,7 +4,10 @@
modules (currently limited to C extensions, should accommodate C++
extensions ASAP)."""
-import sys, os, re
+import contextlib
+import os
+import re
+import sys
from distutils.core import Command
from distutils.errors import *
from distutils.sysconfig import customize_compiler, get_python_version
@@ -85,6 +88,8 @@
"forcibly build everything (ignore file timestamps)"),
('compiler=', 'c',
"specify the compiler type"),
+ ('parallel=', 'j',
+ "number of parallel build jobs"),
('swig-cpp', None,
"make SWIG create C++ files (default is C)"),
('swig-opts=', None,
@@ -124,6 +129,7 @@
self.swig_cpp = None
self.swig_opts = None
self.user = None
+ self.parallel = None
def finalize_options(self):
from distutils import sysconfig
@@ -134,6 +140,7 @@
('compiler', 'compiler'),
('debug', 'debug'),
('force', 'force'),
+ ('parallel', 'parallel'),
('plat_name', 'plat_name'),
)
@@ -274,6 +281,12 @@
self.library_dirs.append(user_lib)
self.rpath.append(user_lib)
+ if isinstance(self.parallel, str):
+ try:
+ self.parallel = int(self.parallel)
+ except ValueError:
+ raise DistutilsOptionError("parallel should be an integer")
+
def run(self):
from distutils.ccompiler import new_compiler
@@ -442,15 +455,45 @@
def build_extensions(self):
# First, sanity-check the 'extensions' list
self.check_extensions_list(self.extensions)
+ if self.parallel:
+ self._build_extensions_parallel()
+ else:
+ self._build_extensions_serial()
+ def _build_extensions_parallel(self):
+ workers = self.parallel
+ if self.parallel is True:
+ workers = os.cpu_count() # may return None
+ try:
+ from concurrent.futures import ThreadPoolExecutor
+ except ImportError:
+ workers = None
+
+ if workers is None:
+ self._build_extensions_serial()
+ return
+
+ with ThreadPoolExecutor(max_workers=workers) as executor:
+ futures = [executor.submit(self.build_extension, ext)
+ for ext in self.extensions]
+ for ext, fut in zip(self.extensions, futures):
+ with self._filter_build_errors(ext):
+ fut.result()
+
+ def _build_extensions_serial(self):
for ext in self.extensions:
- try:
+ with self._filter_build_errors(ext):
self.build_extension(ext)
- except (CCompilerError, DistutilsError, CompileError) as e:
- if not ext.optional:
- raise
- self.warn('building extension "%s" failed: %s' %
- (ext.name, e))
+
+ @contextlib.contextmanager
+ def _filter_build_errors(self, ext):
+ try:
+ yield
+ except (CCompilerError, DistutilsError, CompileError) as e:
+ if not ext.optional:
+ raise
+ self.warn('building extension "%s" failed: %s' %
+ (ext.name, e))
def build_extension(self, ext):
sources = ext.sources
diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py
--- a/Lib/distutils/tests/test_build_ext.py
+++ b/Lib/distutils/tests/test_build_ext.py
@@ -37,6 +37,9 @@
from distutils.command import build_ext
build_ext.USER_BASE = site.USER_BASE
+ def build_ext(self, *args, **kwargs):
+ return build_ext(*args, **kwargs)
+
def test_build_ext(self):
global ALREADY_TESTED
copy_xxmodule_c(self.tmp_dir)
@@ -44,7 +47,7 @@
xx_ext = Extension('xx', [xx_c])
dist = Distribution({'name': 'xx', 'ext_modules': [xx_ext]})
dist.package_dir = self.tmp_dir
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
fixup_build_ext(cmd)
cmd.build_lib = self.tmp_dir
cmd.build_temp = self.tmp_dir
@@ -91,7 +94,7 @@
def test_solaris_enable_shared(self):
dist = Distribution({'name': 'xx'})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
old = sys.platform
sys.platform = 'sunos' # fooling finalize_options
@@ -113,7 +116,7 @@
def test_user_site(self):
import site
dist = Distribution({'name': 'xx'})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
# making sure the user option is there
options = [name for name, short, lable in
@@ -144,14 +147,14 @@
# with the optional argument.
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.ensure_finalized()
self.assertRaises((UnknownFileError, CompileError),
cmd.run) # should raise an error
modules = [Extension('foo', ['xxx'], optional=True)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.ensure_finalized()
cmd.run() # should pass
@@ -160,7 +163,7 @@
# etc.) are in the include search path.
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.finalize_options()
from distutils import sysconfig
@@ -172,14 +175,14 @@
# make sure cmd.libraries is turned into a list
# if it's a string
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.libraries = 'my_lib, other_lib lastlib'
cmd.finalize_options()
self.assertEqual(cmd.libraries, ['my_lib', 'other_lib', 'lastlib'])
# make sure cmd.library_dirs is turned into a list
# if it's a string
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.library_dirs = 'my_lib_dir%sother_lib_dir' % os.pathsep
cmd.finalize_options()
self.assertIn('my_lib_dir', cmd.library_dirs)
@@ -187,7 +190,7 @@
# make sure rpath is turned into a list
# if it's a string
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.rpath = 'one%stwo' % os.pathsep
cmd.finalize_options()
self.assertEqual(cmd.rpath, ['one', 'two'])
@@ -196,32 +199,32 @@
# make sure define is turned into 2-tuples
# strings if they are ','-separated strings
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.define = 'one,two'
cmd.finalize_options()
self.assertEqual(cmd.define, [('one', '1'), ('two', '1')])
# make sure undef is turned into a list of
# strings if they are ','-separated strings
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.undef = 'one,two'
cmd.finalize_options()
self.assertEqual(cmd.undef, ['one', 'two'])
# make sure swig_opts is turned into a list
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.swig_opts = None
cmd.finalize_options()
self.assertEqual(cmd.swig_opts, [])
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.swig_opts = '1 2'
cmd.finalize_options()
self.assertEqual(cmd.swig_opts, ['1', '2'])
def test_check_extensions_list(self):
dist = Distribution()
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.finalize_options()
#'extensions' option must be a list of Extension instances
@@ -270,7 +273,7 @@
def test_get_source_files(self):
modules = [Extension('foo', ['xxx'], optional=False)]
dist = Distribution({'name': 'xx', 'ext_modules': modules})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.ensure_finalized()
self.assertEqual(cmd.get_source_files(), ['xxx'])
@@ -279,7 +282,7 @@
# should not be overriden by a compiler instance
# when the command is run
dist = Distribution()
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.compiler = 'unix'
cmd.ensure_finalized()
cmd.run()
@@ -292,7 +295,7 @@
ext = Extension('foo', [c_file], optional=False)
dist = Distribution({'name': 'xx',
'ext_modules': [ext]})
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
fixup_build_ext(cmd)
cmd.ensure_finalized()
self.assertEqual(len(cmd.get_outputs()), 1)
@@ -355,7 +358,7 @@
#etree_ext = Extension('lxml.etree', [etree_c])
#dist = Distribution({'name': 'lxml', 'ext_modules': [etree_ext]})
dist = Distribution()
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.inplace = 1
cmd.distribution.package_dir = {'': 'src'}
cmd.distribution.packages = ['lxml', 'lxml.html']
@@ -462,7 +465,7 @@
'ext_modules': [deptarget_ext]
})
dist.package_dir = self.tmp_dir
- cmd = build_ext(dist)
+ cmd = self.build_ext(dist)
cmd.build_lib = self.tmp_dir
cmd.build_temp = self.tmp_dir
@@ -481,8 +484,19 @@
self.fail("Wrong deployment target during compilation")
+class ParallelBuildExtTestCase(BuildExtTestCase):
+
+ def build_ext(self, *args, **kwargs):
+ build_ext = super().build_ext(*args, **kwargs)
+ build_ext.parallel = True
+ return build_ext
+
+
def test_suite():
- return unittest.makeSuite(BuildExtTestCase)
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(BuildExtTestCase))
+ suite.addTest(unittest.makeSuite(ParallelBuildExtTestCase))
+ return suite
if __name__ == '__main__':
- support.run_unittest(test_suite())
+ support.run_unittest(__name__)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -139,6 +139,9 @@
Library
-------
+- Issue #5309: distutils' build and build_ext commands now accept a ``-j``
+ option to enable parallel building of extension modules.
+
- Issue #22448: Improve canceled timer handles cleanup to prevent
unbound memory usage. Patch by Joshua Moore-Oliva.
diff --git a/Modules/Setup.dist b/Modules/Setup.dist
--- a/Modules/Setup.dist
+++ b/Modules/Setup.dist
@@ -118,6 +118,7 @@
itertools itertoolsmodule.c # Functions creating iterators for efficient looping
atexit atexitmodule.c # Register functions to be run at interpreter-shutdown
_stat _stat.c # stat.h interface
+time timemodule.c # time module
# access to ISO C locale support
_locale _localemodule.c # -lintl
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -25,6 +25,11 @@
py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST')
sysconfig.get_config_vars()['CFLAGS'] = cflags + ' ' + py_cflags_nodist
+class Dummy:
+ """Hack for parallel build"""
+ ProcessPoolExecutor = None
+sys.modules['concurrent.futures.process'] = Dummy
+
def get_platform():
# cross build
if "_PYTHON_HOST_PLATFORM" in os.environ:
@@ -174,6 +179,8 @@
build_ext.__init__(self, dist)
self.failed = []
self.failed_on_import = []
+ if '-j' in os.environ.get('MAKEFLAGS', ''):
+ self.parallel = True
def build_extensions(self):
@@ -253,6 +260,9 @@
build_ext.build_extensions(self)
+ for ext in self.extensions:
+ self.check_extension_import(ext)
+
longest = max([len(e.name) for e in self.extensions])
if self.failed or self.failed_on_import:
all_failed = self.failed + self.failed_on_import
@@ -305,6 +315,15 @@
(ext.name, sys.exc_info()[1]))
self.failed.append(ext.name)
return
+
+ def check_extension_import(self, ext):
+ # Don't try to import an extension that has failed to compile
+ if ext.name in self.failed:
+ self.announce(
+ 'WARNING: skipping import check for failed build "%s"' %
+ ext.name, level=1)
+ return
+
# Workaround for Mac OS X: The Carbon-based modules cannot be
# reliably imported into a command-line Python
if 'Carbon' in ext.extra_link_args:
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list