[Python-checkins] bpo-36786: Run compileall in parallel during "make install" (GH-13078)

Antoine Pitrou webhook-mailer at python.org
Wed May 15 17:45:22 EDT 2019 

https://github.com/python/cpython/commit/1a2dd82f56bd813aacc570e172cefe55a8a41504
commit: 1a2dd82f56bd813aacc570e172cefe55a8a41504
branch: master
author: Antoine Pitrou <antoine at python.org>
committer: GitHub <noreply at github.com>
date: 2019-05-15T23:45:18+02:00
summary:

bpo-36786: Run compileall in parallel during "make install" (GH-13078)

files:
A Misc/NEWS.d/next/Build/2019-05-03-21-08-06.bpo-36786.gOLFbD.rst
M Doc/library/compileall.rst
M Lib/compileall.py
M Lib/test/test_compileall.py
M Makefile.pre.in

diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst
index 5e08616e9347..bb5000a0736c 100644
--- a/Doc/library/compileall.rst
+++ b/Doc/library/compileall.rst
@@ -158,7 +158,8 @@ Public functions
    The argument *workers* specifies how many workers are used to
    compile files in parallel. The default is to not use multiple workers.
    If the platform can't use multiple workers and *workers* argument is given,
-   then sequential compilation will be used as a fallback.  If *workers* is
+   then sequential compilation will be used as a fallback.  If *workers*
+   is 0, the number of cores in the system is used.  If *workers* is
    lower than ``0``, a :exc:`ValueError` will be raised.
 
    *invalidation_mode* should be a member of the
@@ -184,6 +185,9 @@ Public functions
    .. versionchanged:: 3.7
       The *invalidation_mode* parameter was added.
 
+   .. versionchanged:: 3.8
+      Setting *workers* to 0 now chooses the optimal number of cores.
+
 .. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP)
 
    Compile the file with path *fullname*. Return a true value if the file
diff --git a/Lib/compileall.py b/Lib/compileall.py
index aa65c6b904e7..49306d9dabbc 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -67,20 +67,20 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
     invalidation_mode: how the up-to-dateness of the pyc will be checked
     """
     ProcessPoolExecutor = None
-    if workers is not None:
-        if workers < 0:
-            raise ValueError('workers must be greater or equal to 0')
-        elif workers != 1:
-            try:
-                # Only import when needed, as low resource platforms may
-                # fail to import it
-                from concurrent.futures import ProcessPoolExecutor
-            except ImportError:
-                workers = 1
+    if workers < 0:
+        raise ValueError('workers must be greater or equal to 0')
+    if workers != 1:
+        try:
+            # Only import when needed, as low resource platforms may
+            # fail to import it
+            from concurrent.futures import ProcessPoolExecutor
+        except ImportError:
+            workers = 1
     files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
                       ddir=ddir)
     success = True
-    if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
+    if workers != 1 and ProcessPoolExecutor is not None:
+        # If workers == 0, let ProcessPoolExecutor choose
         workers = workers or None
         with ProcessPoolExecutor(max_workers=workers) as executor:
             results = executor.map(partial(compile_file,
@@ -290,9 +290,6 @@ def main():
                 print("Error reading file list {}".format(args.flist))
             return False
 
-    if args.workers is not None:
-        args.workers = args.workers or None
-
     if args.invalidation_mode:
         ivl_mode = args.invalidation_mode.replace('-', '_').upper()
         invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 8e584602de36..04f6e1e049dd 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -575,7 +575,7 @@ def test_workers_available_cores(self, compile_dir):
                         new=[sys.executable, self.directory, "-j0"]):
             compileall.main()
             self.assertTrue(compile_dir.called)
-            self.assertEqual(compile_dir.call_args[-1]['workers'], None)
+            self.assertEqual(compile_dir.call_args[-1]['workers'], 0)
 
 
 class CommmandLineTestsWithSourceEpoch(CommandLineTestsBase,
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 75eb66be3c01..4924dedc357c 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1432,30 +1432,30 @@ libinstall:	build_all $(srcdir)/Modules/xxmodule.c
 	fi
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST)  $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST) -f \
+		-j0 -d $(LIBDEST) -f \
 		-x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
 		$(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST) -f \
+		-j0 -d $(LIBDEST) -f \
 		-x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
 		$(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi -OO $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST) -f \
+		-j0 -d $(LIBDEST) -f \
 		-x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
 		$(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST)/site-packages -f \
+		-j0 -d $(LIBDEST)/site-packages -f \
 		-x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST)/site-packages -f \
+		-j0 -d $(LIBDEST)/site-packages -f \
 		-x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -Wi -OO $(DESTDIR)$(LIBDEST)/compileall.py \
-		-d $(LIBDEST)/site-packages -f \
+		-j0 -d $(LIBDEST)/site-packages -f \
 		-x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
 		$(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/Grammar.txt
diff --git a/Misc/NEWS.d/next/Build/2019-05-03-21-08-06.bpo-36786.gOLFbD.rst b/Misc/NEWS.d/next/Build/2019-05-03-21-08-06.bpo-36786.gOLFbD.rst
new file mode 100644
index 000000000000..0fcda434a799
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-05-03-21-08-06.bpo-36786.gOLFbD.rst
@@ -0,0 +1 @@
+"make install" now runs compileall in parallel.

More information about the Python-checkins mailing list