[Python-checkins] [3.6] bpo-31536: Avoid wholesale rebuild after `make regen-all` (GH-3678) (#3797)
Antoine Pitrou
webhook-mailer at python.org
Thu Sep 28 07:21:35 EDT 2017
https://github.com/python/cpython/commit/ec3d34c5b2feb10cb4d7606a10d81c178c3afce3
commit: ec3d34c5b2feb10cb4d7606a10d81c178c3afce3
branch: 3.6
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Antoine Pitrou <pitrou at free.fr>
date: 2017-09-28T13:21:32+02:00
summary:
[3.6] bpo-31536: Avoid wholesale rebuild after `make regen-all` (GH-3678) (#3797)
bpo-31536: Avoid wholesale rebuild after `make regen-all`
files:
A Misc/NEWS.d/next/Build/2017-09-20-21-32-21.bpo-31536.KUDjno.rst
A Tools/scripts/update_file.py
M Makefile.pre.in
M Parser/asdl_c.py
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 6cb0c63e5d6..6a0118f7871 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -230,6 +230,7 @@ PYTHON= python$(EXE)
BUILDPYTHON= python$(BUILDEXE)
PYTHON_FOR_REGEN=@PYTHON_FOR_REGEN@
+UPDATE_FILE=@PYTHON_FOR_REGEN@ $(srcdir)/Tools/scripts/update_file.py
PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
_PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
BUILD_GNU_TYPE= @build@
@@ -696,12 +697,14 @@ regen-importlib: Programs/_freeze_importlib
# from Lib/importlib/_bootstrap_external.py using _freeze_importlib
./Programs/_freeze_importlib \
$(srcdir)/Lib/importlib/_bootstrap_external.py \
- $(srcdir)/Python/importlib_external.h
+ $(srcdir)/Python/importlib_external.h.new
+ $(UPDATE_FILE) $(srcdir)/Python/importlib_external.h $(srcdir)/Python/importlib_external.h.new
# Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py
# using _freeze_importlib
./Programs/_freeze_importlib \
$(srcdir)/Lib/importlib/_bootstrap.py \
- $(srcdir)/Python/importlib.h
+ $(srcdir)/Python/importlib.h.new
+ $(UPDATE_FILE) $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib.h.new
############################################################################
@@ -775,8 +778,10 @@ regen-grammar: $(PGEN)
# from Grammar/Grammar using pgen
@$(MKDIR_P) Include
$(PGEN) $(srcdir)/Grammar/Grammar \
- $(srcdir)/Include/graminit.h \
- $(srcdir)/Python/graminit.c
+ $(srcdir)/Include/graminit.h.new \
+ $(srcdir)/Python/graminit.c.new
+ $(UPDATE_FILE) $(srcdir)/Include/graminit.h $(srcdir)/Include/graminit.h.new
+ $(UPDATE_FILE) $(srcdir)/Python/graminit.c $(srcdir)/Python/graminit.c.new
Parser/grammar.o: $(srcdir)/Parser/grammar.c \
$(srcdir)/Include/token.h \
@@ -794,13 +799,15 @@ regen-ast:
# Regenerate Include/Python-ast.h using Parser/asdl_c.py -h
$(MKDIR_P) $(srcdir)/Include
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
- -h $(srcdir)/Include \
+ -h $(srcdir)/Include/Python-ast.h.new \
$(srcdir)/Parser/Python.asdl
+ $(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new
# Regenerate Python/Python-ast.c using Parser/asdl_c.py -c
$(MKDIR_P) $(srcdir)/Python
$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
- -c $(srcdir)/Python \
+ -c $(srcdir)/Python/Python-ast.c.new \
$(srcdir)/Parser/Python.asdl
+ $(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new
.PHONY: regen-opcode
regen-opcode:
@@ -808,7 +815,8 @@ regen-opcode:
# using Tools/scripts/generate_opcode_h.py
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \
$(srcdir)/Lib/opcode.py \
- $(srcdir)/Include/opcode.h
+ $(srcdir)/Include/opcode.h.new
+ $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new
Python/compile.o Python/symtable.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
@@ -865,7 +873,8 @@ regen-opcode-targets:
# Regenerate Python/opcode_targets.h from Lib/opcode.py
# using Python/makeopcodetargets.py
$(PYTHON_FOR_REGEN) $(srcdir)/Python/makeopcodetargets.py \
- $(srcdir)/Python/opcode_targets.h
+ $(srcdir)/Python/opcode_targets.h.new
+ $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new
Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h
@@ -892,7 +901,8 @@ regen-typeslots:
# using Objects/typeslots.py
$(PYTHON_FOR_REGEN) $(srcdir)/Objects/typeslots.py \
< $(srcdir)/Include/typeslots.h \
- $(srcdir)/Objects/typeslots.inc
+ $(srcdir)/Objects/typeslots.inc.new
+ $(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new
############################################################################
# Header files
diff --git a/Misc/NEWS.d/next/Build/2017-09-20-21-32-21.bpo-31536.KUDjno.rst b/Misc/NEWS.d/next/Build/2017-09-20-21-32-21.bpo-31536.KUDjno.rst
new file mode 100644
index 00000000000..414f1a45eab
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2017-09-20-21-32-21.bpo-31536.KUDjno.rst
@@ -0,0 +1 @@
+Avoid wholesale rebuild after `make regen-all` if nothing changed.
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 1e5f4d9a2a5..6302af6fa3e 100644
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -1281,59 +1281,55 @@ def main(srcfile, dump_module=False):
print(mod)
if not asdl.check(mod):
sys.exit(1)
- if INC_DIR:
- p = "%s/%s-ast.h" % (INC_DIR, mod.name)
- f = open(p, "w")
- f.write(auto_gen_msg)
- f.write('#include "asdl.h"\n\n')
- c = ChainOfVisitors(TypeDefVisitor(f),
- StructVisitor(f),
- PrototypeVisitor(f),
- )
- c.visit(mod)
- f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
- f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
- f.write("int PyAST_Check(PyObject* obj);\n")
- f.close()
-
- if SRC_DIR:
- p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
- f = open(p, "w")
- f.write(auto_gen_msg)
- f.write('#include <stddef.h>\n')
- f.write('\n')
- f.write('#include "Python.h"\n')
- f.write('#include "%s-ast.h"\n' % mod.name)
- f.write('\n')
- f.write("static PyTypeObject AST_type;\n")
- v = ChainOfVisitors(
- PyTypesDeclareVisitor(f),
- PyTypesVisitor(f),
- Obj2ModPrototypeVisitor(f),
- FunctionVisitor(f),
- ObjVisitor(f),
- Obj2ModVisitor(f),
- ASTModuleVisitor(f),
- PartingShots(f),
- )
- v.visit(mod)
- f.close()
+ if H_FILE:
+ with open(H_FILE, "w") as f:
+ f.write(auto_gen_msg)
+ f.write('#include "asdl.h"\n\n')
+ c = ChainOfVisitors(TypeDefVisitor(f),
+ StructVisitor(f),
+ PrototypeVisitor(f),
+ )
+ c.visit(mod)
+ f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
+ f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
+ f.write("int PyAST_Check(PyObject* obj);\n")
+
+ if C_FILE:
+ with open(C_FILE, "w") as f:
+ f.write(auto_gen_msg)
+ f.write('#include <stddef.h>\n')
+ f.write('\n')
+ f.write('#include "Python.h"\n')
+ f.write('#include "%s-ast.h"\n' % mod.name)
+ f.write('\n')
+ f.write("static PyTypeObject AST_type;\n")
+ v = ChainOfVisitors(
+ PyTypesDeclareVisitor(f),
+ PyTypesVisitor(f),
+ Obj2ModPrototypeVisitor(f),
+ FunctionVisitor(f),
+ ObjVisitor(f),
+ Obj2ModVisitor(f),
+ ASTModuleVisitor(f),
+ PartingShots(f),
+ )
+ v.visit(mod)
if __name__ == "__main__":
import getopt
- INC_DIR = ''
- SRC_DIR = ''
+ H_FILE = ''
+ C_FILE = ''
dump_module = False
opts, args = getopt.getopt(sys.argv[1:], "dh:c:")
for o, v in opts:
if o == '-h':
- INC_DIR = v
+ H_FILE = v
if o == '-c':
- SRC_DIR = v
+ C_FILE = v
if o == '-d':
dump_module = True
- if INC_DIR and SRC_DIR:
+ if H_FILE and C_FILE:
print('Must specify exactly one output file')
sys.exit(1)
elif len(args) != 1:
diff --git a/Tools/scripts/update_file.py b/Tools/scripts/update_file.py
new file mode 100644
index 00000000000..224585c69bb
--- /dev/null
+++ b/Tools/scripts/update_file.py
@@ -0,0 +1,28 @@
+"""
+A script that replaces an old file with a new one, only if the contents
+actually changed. If not, the new file is simply deleted.
+
+This avoids wholesale rebuilds when a code (re)generation phase does not
+actually change the in-tree generated code.
+"""
+
+import os
+import sys
+
+
+def main(old_path, new_path):
+ with open(old_path, 'rb') as f:
+ old_contents = f.read()
+ with open(new_path, 'rb') as f:
+ new_contents = f.read()
+ if old_contents != new_contents:
+ os.replace(new_path, old_path)
+ else:
+ os.unlink(new_path)
+
+
+if __name__ == '__main__':
+ if len(sys.argv) != 3:
+ print("Usage: %s <path to be updated> <path with new contents>" % (sys.argv[0],))
+ sys.exit(1)
+ main(sys.argv[1], sys.argv[2])
More information about the Python-checkins
mailing list