[Python-checkins] cpython (2.7): Issue #24915: Make PGO builds support Clang and use the test suite for
brett.cannon
python-checkins at python.org
Sat Sep 19 00:11:34 CEST 2015
https://hg.python.org/cpython/rev/0f4e6c303531
changeset: 98059:0f4e6c303531
branch: 2.7
parent: 98034:f6125114b55f
user: Brett Cannon <brett at python.org>
date: Fri Sep 18 15:09:42 2015 -0700
summary:
Issue #24915: Make PGO builds support Clang and use the test suite for
profile data.
Thanks to Alecsandru Patrascu of Intel for the initial patch.
files:
.gitignore | 3 +
.hgignore | 3 +
Makefile.pre.in | 31 ++++++++++---
Misc/NEWS | 3 +
README | 29 +++++++++++++
configure | 82 +++++++++++++++++++++++++++++++++++++
configure.ac | 45 ++++++++++++++++++++
7 files changed, 189 insertions(+), 7 deletions(-)
diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,9 @@
*.pyo
*.rej
*~
+*.gc??
+*.profclang?
+*.profraw
Doc/build/
Doc/tools/docutils/
Doc/tools/jinja2/
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -45,6 +45,9 @@
*.pyd
*.cover
*~
+*.gc??
+*.profclang?
+*.profraw
Lib/distutils/command/*.pdb
Lib/lib2to3/*.pickle
Lib/test/data/*
diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -42,6 +42,11 @@
HGVERSION= @HGVERSION@
HGTAG= @HGTAG@
HGBRANCH= @HGBRANCH@
+PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@
+PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@
+LLVM_PROF_MERGER=@LLVM_PROF_MERGER@
+LLVM_PROF_FILE=@LLVM_PROF_FILE@
+LLVM_PROF_ERR=@LLVM_PROF_ERR@
GNULD= @GNULD@
@@ -204,8 +209,7 @@
TCLTK_LIBS= @TCLTK_LIBS@
# The task to run while instrument when building the profile-opt target
-PROFILE_TASK= $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck
-#PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py
+PROFILE_TASK=-m test.regrtest >/dev/null 2>&1
# === Definitions added by makesetup ===
@@ -420,27 +424,38 @@
all: build_all
build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks
-# Compile a binary with gcc profile guided optimization.
+# Compile a binary with profile guided optimization.
profile-opt:
+ @if [ $(LLVM_PROF_ERR) == yes ]; then \
+ echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\
+ echo "Please add it to PATH and run ./configure again" ;\
+ exit 1;\
+ fi
@echo "Building with support for profile generation:"
$(MAKE) clean
+ $(MAKE) profile-removal
$(MAKE) build_all_generate_profile
- @echo "Running benchmark to generate profile data:"
$(MAKE) profile-removal
+ @echo "Running code to generate profile data (this can take a while):"
$(MAKE) run_profile_task
+ $(MAKE) build_all_merge_profile
@echo "Rebuilding with profile guided optimizations:"
$(MAKE) clean
$(MAKE) build_all_use_profile
+ $(MAKE) profile-removal
build_all_generate_profile:
- $(MAKE) all CFLAGS="$(CFLAGS) -fprofile-generate" LIBS="$(LIBS) -lgcov"
+ $(MAKE) all CFLAGS="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)"
run_profile_task:
: # FIXME: can't run for a cross build
- ./$(BUILDPYTHON) $(PROFILE_TASK)
+ $(LLVM_PROF_FILE) ./$(BUILDPYTHON) $(PROFILE_TASK) || true
+
+build_all_merge_profile:
+ $(LLVM_PROF_MERGER)
build_all_use_profile:
- $(MAKE) all CFLAGS="$(CFLAGS) -fprofile-use"
+ $(MAKE) all CFLAGS="$(CFLAGS) $(PGO_PROF_USE_FLAG)"
coverage:
@echo "Building with support for coverage checking:"
@@ -1330,9 +1345,11 @@
find build -name 'fficonfig.h' -exec rm -f {} ';' || true
find build -name 'fficonfig.py' -exec rm -f {} ';' || true
-rm -f Lib/lib2to3/*Grammar*.pickle
+ -rm -rf build
profile-removal:
find . -name '*.gc??' -exec rm -f {} ';'
+ find . -name '*.profclang?' -exec rm -f {} ';'
clobber: clean profile-removal
-rm -f $(BUILDPYTHON) $(PGEN) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -150,6 +150,9 @@
Build
-----
+- Issue #24915: When doing a PGO build, the test suite is now used instead of
+ pybench; Clang support was also added as part off this work.
+
- Issue #24986: It is now possible to build Python on Windows without errors
when external libraries are not available.
diff --git a/README b/README
--- a/README
+++ b/README
@@ -173,6 +173,11 @@
build your desired target. The interpreter executable is built in the
top level directory.
+If you need an optimized version of Python, you type "make profile-opt"
+in the top level directory. This will rebuild the interpreter executable
+using Profile Guided Optimization (PGO). For more details, see the
+section below.
+
Once you have built a Python interpreter, see the subsections below on
testing and installation. If you run into trouble, see the next
section.
@@ -185,6 +190,30 @@
interpreter has been built.
+Profile Guided Optimization
+---------------------------
+
+PGO takes advantage of recent versions of the GCC or Clang compilers.
+If ran, the "profile-opt" rule will do several steps.
+
+First, the entire Python directory is cleaned of temporary files that
+may resulted in a previous compilation.
+
+Then, an instrumented version of the interpreter is built, using suitable
+compiler flags for each flavour. Note that this is just an intermediary
+step and the binary resulted after this step is not good for real life
+workloads, as it has profiling instructions embedded inside.
+
+After this instrumented version of the interpreter is built, the Makefile
+will automatically run a training workload. This is necessary in order to
+profile the interpreter execution. Note also that any output, both stdout
+and stderr, that may appear at this step is supressed.
+
+Finally, the last step is to rebuild the interpreter, using the information
+collected in the previous one. The end result will be a the Python binary
+that is optimized and suitable for distribution or production installation.
+
+
Troubleshooting
---------------
diff --git a/configure b/configure
--- a/configure
+++ b/configure
@@ -661,6 +661,12 @@
SO
LIBTOOL_CRUFT
OTHER_LIBTOOL_OPT
+LLVM_PROF_FOUND
+LLVM_PROF_ERR
+LLVM_PROF_FILE
+LLVM_PROF_MERGER
+PGO_PROF_USE_FLAG
+PGO_PROF_GEN_FLAG
UNIVERSAL_ARCH_FLAGS
BASECFLAGS
OPT
@@ -6326,6 +6332,82 @@
CFLAGS=$save_CFLAGS
fi
+
+# Enable PGO flags.
+# Extract the first word of "llvm-profdata", so it can be a program name with args.
+set dummy llvm-profdata; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$LLVM_PROF_FOUND"; then
+ ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+ ac_cv_prog_LLVM_PROF_FOUND="found"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found"
+fi
+fi
+LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND
+if test -n "$LLVM_PROF_FOUND"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5
+$as_echo "$LLVM_PROF_FOUND" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+LLVM_PROF_ERR=no
+case $CC in
+ *clang*)
+ # Any changes made here should be reflected in the GCC+Darwin case below
+ PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+ PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+ LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+ LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+ if test $LLVM_PROF_FOUND = not-found
+ then
+ LLVM_PROF_ERR=yes
+ fi
+ ;;
+ *gcc*)
+ case $ac_sys_system in
+ Darwin*)
+ PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+ PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+ LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+ LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+ if test $LLVM_PROF_FOUND = not-found
+ then
+ LLVM_PROF_ERR=yes
+ fi
+ ;;
+ *)
+ PGO_PROF_GEN_FLAG="-fprofile-generate"
+ PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
+ LLVM_PROF_MERGER="true"
+ LLVM_PROF_FILE=""
+ ;;
+ esac
+ ;;
+esac
+
+
# On some compilers, pthreads are available without further options
# (e.g. MacOS X). On some of these systems, the compiler will not
# complain if unaccepted options are passed (e.g. gcc on Mac OS X).
diff --git a/configure.ac b/configure.ac
--- a/configure.ac
+++ b/configure.ac
@@ -1352,6 +1352,51 @@
CFLAGS=$save_CFLAGS
fi
+
+# Enable PGO flags.
+AC_SUBST(PGO_PROF_GEN_FLAG)
+AC_SUBST(PGO_PROF_USE_FLAG)
+AC_SUBST(LLVM_PROF_MERGER)
+AC_SUBST(LLVM_PROF_FILE)
+AC_SUBST(LLVM_PROF_ERR)
+AC_SUBST(LLVM_PROF_FOUND)
+AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found)
+LLVM_PROF_ERR=no
+case $CC in
+ *clang*)
+ # Any changes made here should be reflected in the GCC+Darwin case below
+ PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+ PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+ LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+ LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+ if test $LLVM_PROF_FOUND = not-found
+ then
+ LLVM_PROF_ERR=yes
+ fi
+ ;;
+ *gcc*)
+ case $ac_sys_system in
+ Darwin*)
+ PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+ PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+ LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+ LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+ if test $LLVM_PROF_FOUND = not-found
+ then
+ LLVM_PROF_ERR=yes
+ fi
+ ;;
+ *)
+ PGO_PROF_GEN_FLAG="-fprofile-generate"
+ PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
+ LLVM_PROF_MERGER="true"
+ LLVM_PROF_FILE=""
+ ;;
+ esac
+ ;;
+esac
+
+
# On some compilers, pthreads are available without further options
# (e.g. MacOS X). On some of these systems, the compiler will not
# complain if unaccepted options are passed (e.g. gcc on Mac OS X).
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list