Python-checkins
Threads by month
- ----- 2024 -----
- July
- June
- May
- April
- March
- February
- January
- ----- 2023 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2022 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2021 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2020 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2019 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2018 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2017 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2016 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2015 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2014 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2013 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2012 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2011 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2010 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2009 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2008 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2007 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2006 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2005 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2004 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2003 -----
- December
- November
- October
- September
- August
November 2021
- 1 participants
- 444 discussions
https://github.com/python/cpython/commit/b9e9292d75fdea621e05e39b8629e6935d…
commit: b9e9292d75fdea621e05e39b8629e6935d282d0d
branch: main
author: Christian Heimes <christian(a)python.org>
committer: tiran <christian(a)python.org>
date: 2021-11-23T22:58:13+01:00
summary:
bpo-45847: Port _ssl and _hashlib to PY_STDLIB_MOD (GH-29727)
files:
M Modules/Setup.stdlib.in
M configure
M configure.ac
M setup.py
diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in
index 7422ddb59e309..9f66d531fef0f 100644
--- a/Modules/Setup.stdlib.in
+++ b/Modules/Setup.stdlib.in
@@ -123,6 +123,11 @@
#
@MODULE__SQLITE3_TRUE@_sqlite3 _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c
+# needs -lssl and -lcrypt
+@MODULE__SSL_TRUE@_ssl _ssl.c
+# needs -lcrypt
+@MODULE__HASHLIB_TRUE@_hashlib _hashopenssl.c
+
############################################################################
# macOS specific modules
diff --git a/configure b/configure
index 01aa80091cd49..ac0a1f320b2a7 100755
--- a/configure
+++ b/configure
@@ -642,6 +642,10 @@ MODULE__TESTINTERNALCAPI_FALSE
MODULE__TESTINTERNALCAPI_TRUE
MODULE__TESTCAPI_FALSE
MODULE__TESTCAPI_TRUE
+MODULE__HASHLIB_FALSE
+MODULE__HASHLIB_TRUE
+MODULE__SSL_FALSE
+MODULE__SSL_TRUE
MODULE__LZMA_FALSE
MODULE__LZMA_TRUE
MODULE__BZ2_FALSE
@@ -20297,6 +20301,16 @@ rm -f core conftest.err conftest.$ac_objext \
# rpath to libssl and libcrypto
+if test "x$GNULD" = xyes; then :
+
+ rpath_arg="-Wl,--enable-new-dtags,-rpath="
+
+else
+
+ rpath_arg="-Wl,-rpath="
+
+fi
+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-openssl-rpath" >&5
$as_echo_n "checking for --with-openssl-rpath... " >&6; }
@@ -20310,12 +20324,26 @@ fi
case $with_openssl_rpath in #(
auto|yes) :
- OPENSSL_RPATH=auto ;; #(
+
+ OPENSSL_RPATH=auto
+ for arg in "$OPENSSL_LDFLAGS"; do
+ case $arg in #(
+ -L*) :
+ OPENSSL_LDFLAGS_RPATH="$OPENSSL_LDFLAGS_RPATH ${rpath_arg}$(echo $arg | cut -c3-)"
+ ;; #(
+ *) :
+ ;;
+esac
+ done
+ ;; #(
no) :
OPENSSL_RPATH= ;; #(
*) :
if test -d "$with_openssl_rpath"; then :
- OPENSSL_RPATH="$with_openssl_rpath"
+
+ OPENSSL_RPATH="$with_openssl_rpath"
+ OPENSSL_LDFLAGS_RPATH="${rpath_arg}$with_openssl_rpath"
+
else
as_fn_error $? "--with-openssl-rpath \"$with_openssl_rpath\" is not a directory" "$LINENO" 5
fi
@@ -20326,71 +20354,163 @@ esac
$as_echo "$OPENSSL_RPATH" >&6; }
+# This static linking is NOT OFFICIALLY SUPPORTED and not advertised.
+# Requires static OpenSSL build with position-independent code. Some features
+# like DSO engines or external OSSL providers don't work. Only tested with GCC
+# and clang on X86_64.
+if test "x$PY_UNSUPPORTED_OPENSSL_BUILD" = xstatic; then :
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for unsupported static openssl build" >&5
+$as_echo_n "checking for unsupported static openssl build... " >&6; }
+ new_OPENSSL_LIBS=
+ for arg in $OPENSSL_LIBS; do
+ case $arg in #(
+ -l*) :
+
+ libname=$(echo $arg | cut -c3-)
+ new_OPENSSL_LIBS="$new_OPENSSL_LIBS -l:lib${libname}.a -Wl,--exclude-libs,lib${libname}.a"
+ ;; #(
+ *) :
+ new_OPENSSL_LIBS="$new_OPENSSL_LIBS $arg"
+ ;;
+esac
+ done
+ OPENSSL_LIBS="$new_OPENSSL_LIBS $ZLIB_LIBS"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $OPENSSL_LIBS" >&5
+$as_echo "$OPENSSL_LIBS" >&6; }
+
+fi
+
+LIBCRYPTO_LIBS=
+for arg in $OPENSSL_LIBS; do
+ case $arg in #(
+ -l*ssl*|-Wl*ssl*) :
+ ;; #(
+ *) :
+ LIBCRYPTO_LIBS="$LIBCRYPTO_LIBS $arg"
+ ;;
+esac
+done
+
# check if OpenSSL libraries work as expected
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether OpenSSL provides required APIs" >&5
-$as_echo_n "checking whether OpenSSL provides required APIs... " >&6; }
-if ${ac_cv_working_openssl+:} false; then :
+save_CFLAGS=$CFLAGS
+save_CPPFLAGS=$CPPFLAGS
+save_LDFLAGS=$LDFLAGS
+save_LIBS=$LIBS
+
+
+ LIBS="$LIBS $OPENSSL_LIBS"
+ CFLAGS="$CFLAGS $OPENSSL_INCLUDES"
+ LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH"
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether OpenSSL provides required ssl module APIs" >&5
+$as_echo_n "checking whether OpenSSL provides required ssl module APIs... " >&6; }
+if ${ac_cv_working_openssl_ssl+:} false; then :
$as_echo_n "(cached) " >&6
else
-save_LIBS="$LIBS"
-save_CFLAGS="$CFLAGS"
-save_LDFLAGS="$LDFLAGS"
-LIBS="$LIBS $OPENSSL_LIBS"
-CFLAGS="$CFLAGS_NODIST $OPENSSL_INCLUDES"
-LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS"
-
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-#include <openssl/opensslv.h>
-#include <openssl/evp.h>
-#include <openssl/ssl.h>
+ #include <openssl/opensslv.h>
+ #include <openssl/ssl.h>
+ #if OPENSSL_VERSION_NUMBER < 0x10101000L
+ #error "OpenSSL >= 1.1.1 is required"
+ #endif
+ static void keylog_cb(const SSL *ssl, const char *line) {}
-#if OPENSSL_VERSION_NUMBER < 0x10101000L
-#error "OpenSSL >= 1.1.1 is required"
-#endif
+int
+main ()
+{
+
+ SSL_CTX *ctx = SSL_CTX_new(TLS_client_method());
+ SSL_CTX_set_keylog_callback(ctx, keylog_cb);
+ SSL *ssl = SSL_new(ctx);
+ X509_VERIFY_PARAM *param = SSL_get0_param(ssl);
+ X509_VERIFY_PARAM_set1_host(param, "python.org", 0);
+ SSL_free(ssl);
+ SSL_CTX_free(ctx);
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_working_openssl_ssl=yes
+else
+ ac_cv_working_openssl_ssl=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_working_openssl_ssl" >&5
+$as_echo "$ac_cv_working_openssl_ssl" >&6; }
+
+CFLAGS=$save_CFLAGS
+CPPFLAGS=$save_CPPFLAGS
+LDFLAGS=$save_LDFLAGS
+LIBS=$save_LIBS
+
+
+
+save_CFLAGS=$CFLAGS
+save_CPPFLAGS=$CPPFLAGS
+save_LDFLAGS=$LDFLAGS
+save_LIBS=$LIBS
+
+
+ LIBS="$LIBS $LIBCRYPTO_LIBS"
+ CFLAGS="$CFLAGS $OPENSSL_INCLUDES"
+ LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH"
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether OpenSSL provides required hashlib module APIs" >&5
+$as_echo_n "checking whether OpenSSL provides required hashlib module APIs... " >&6; }
+if ${ac_cv_working_openssl_hashlib+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
-static void keylog_cb(const SSL *ssl, const char *line) {}
+ #include <openssl/opensslv.h>
+ #include <openssl/evp.h>
+ #if OPENSSL_VERSION_NUMBER < 0x10101000L
+ #error "OpenSSL >= 1.1.1 is required"
+ #endif
int
main ()
{
-/* SSL APIs */
-SSL_CTX *ctx = SSL_CTX_new(TLS_client_method());
-SSL_CTX_set_keylog_callback(ctx, keylog_cb);
-SSL *ssl = SSL_new(ctx);
-X509_VERIFY_PARAM *param = SSL_get0_param(ssl);
-X509_VERIFY_PARAM_set1_host(param, "python.org", 0);
-SSL_free(ssl);
-SSL_CTX_free(ctx);
-
-/* hashlib APIs */
-OBJ_nid2sn(NID_md5);
-OBJ_nid2sn(NID_sha1);
-OBJ_nid2sn(NID_sha3_512);
-OBJ_nid2sn(NID_blake2b512);
-EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0);
+ OBJ_nid2sn(NID_md5);
+ OBJ_nid2sn(NID_sha1);
+ OBJ_nid2sn(NID_sha3_512);
+ OBJ_nid2sn(NID_blake2b512);
+ EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0);
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_working_openssl=yes
+ ac_cv_working_openssl_hashlib=yes
else
- ac_cv_working_openssl=no
+ ac_cv_working_openssl_hashlib=no
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
-LIBS="$save_LIBS"
-CFLAGS="$save_CFLAGS"
-LDFLAGS="$save_LDFLAGS"
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_working_openssl" >&5
-$as_echo "$ac_cv_working_openssl" >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_working_openssl_hashlib" >&5
+$as_echo "$ac_cv_working_openssl_hashlib" >&6; }
+
+CFLAGS=$save_CFLAGS
+CPPFLAGS=$save_CPPFLAGS
+LDFLAGS=$save_LDFLAGS
+LIBS=$save_LIBS
+
+
# ssl module default cipher suite string
@@ -21800,6 +21920,79 @@ $as_echo "$py_cv_module__lzma" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _ssl" >&5
+$as_echo_n "checking for stdlib extension module _ssl... " >&6; }
+ case $py_stdlib_not_available in #(
+ *_ssl*) :
+ py_cv_module__ssl=n/a ;; #(
+ *) :
+ if true; then :
+ if test "$ac_cv_working_openssl_ssl" = yes; then :
+ py_cv_module__ssl=yes
+else
+ py_cv_module__ssl=missing
+fi
+else
+ py_cv_module__ssl=disabled
+fi
+ ;;
+esac
+ as_fn_append MODULE_BLOCK "MODULE__SSL=$py_cv_module__ssl$as_nl"
+ if test "x$py_cv_module__ssl" = xyes; then :
+
+ as_fn_append MODULE_BLOCK "MODULE__SSL_CFLAGS=$OPENSSL_INCLUDES$as_nl"
+ as_fn_append MODULE_BLOCK "MODULE__SSL_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS$as_nl"
+
+fi
+ if test "$py_cv_module__ssl" = yes; then
+ MODULE__SSL_TRUE=
+ MODULE__SSL_FALSE='#'
+else
+ MODULE__SSL_TRUE='#'
+ MODULE__SSL_FALSE=
+fi
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__ssl" >&5
+$as_echo "$py_cv_module__ssl" >&6; }
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _hashlib" >&5
+$as_echo_n "checking for stdlib extension module _hashlib... " >&6; }
+ case $py_stdlib_not_available in #(
+ *_hashlib*) :
+ py_cv_module__hashlib=n/a ;; #(
+ *) :
+ if true; then :
+ if test "$ac_cv_working_openssl_hashlib" = yes; then :
+ py_cv_module__hashlib=yes
+else
+ py_cv_module__hashlib=missing
+fi
+else
+ py_cv_module__hashlib=disabled
+fi
+ ;;
+esac
+ as_fn_append MODULE_BLOCK "MODULE__HASHLIB=$py_cv_module__hashlib$as_nl"
+ if test "x$py_cv_module__hashlib" = xyes; then :
+
+ as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES$as_nl"
+ as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS$as_nl"
+
+fi
+ if test "$py_cv_module__hashlib" = yes; then
+ MODULE__HASHLIB_TRUE=
+ MODULE__HASHLIB_FALSE='#'
+else
+ MODULE__HASHLIB_TRUE='#'
+ MODULE__HASHLIB_FALSE=
+fi
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__hashlib" >&5
+$as_echo "$py_cv_module__hashlib" >&6; }
+
+
+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testcapi" >&5
$as_echo_n "checking for stdlib extension module _testcapi... " >&6; }
case $py_stdlib_not_available in #(
@@ -22481,6 +22674,14 @@ if test -z "${MODULE__LZMA_TRUE}" && test -z "${MODULE__LZMA_FALSE}"; then
as_fn_error $? "conditional \"MODULE__LZMA\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
fi
+if test -z "${MODULE__SSL_TRUE}" && test -z "${MODULE__SSL_FALSE}"; then
+ as_fn_error $? "conditional \"MODULE__SSL\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${MODULE__HASHLIB_TRUE}" && test -z "${MODULE__HASHLIB_FALSE}"; then
+ as_fn_error $? "conditional \"MODULE__HASHLIB\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
if test -z "${MODULE__TESTCAPI_TRUE}" && test -z "${MODULE__TESTCAPI_FALSE}"; then
as_fn_error $? "conditional \"MODULE__TESTCAPI\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
diff --git a/configure.ac b/configure.ac
index 0008e8a746c7b..92afdf39fefc0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -5891,6 +5891,12 @@ ac_includes_default="$save_includes_default"
AX_CHECK_OPENSSL([have_openssl=yes],[have_openssl=no])
# rpath to libssl and libcrypto
+AS_VAR_IF([GNULD], [yes], [
+ rpath_arg="-Wl,--enable-new-dtags,-rpath="
+], [
+ rpath_arg="-Wl,-rpath="
+])
+
AC_MSG_CHECKING(for --with-openssl-rpath)
AC_ARG_WITH(openssl-rpath,
AS_HELP_STRING([--with-openssl-rpath=@<:@DIR|auto|no@:>@],
@@ -5903,58 +5909,104 @@ AC_ARG_WITH(openssl-rpath,
[with_openssl_rpath=no]
)
AS_CASE($with_openssl_rpath,
- [auto|yes],[OPENSSL_RPATH=auto],
- [no],[OPENSSL_RPATH=],
+ [auto|yes], [
+ OPENSSL_RPATH=auto
+ dnl look for linker directories
+ for arg in "$OPENSSL_LDFLAGS"; do
+ AS_CASE([$arg],
+ [-L*], [OPENSSL_LDFLAGS_RPATH="$OPENSSL_LDFLAGS_RPATH ${rpath_arg}$(echo $arg | cut -c3-)"]
+ )
+ done
+ ],
+ [no], [OPENSSL_RPATH=],
[AS_IF(
[test -d "$with_openssl_rpath"],
- [OPENSSL_RPATH="$with_openssl_rpath"],
+ [
+ OPENSSL_RPATH="$with_openssl_rpath"
+ OPENSSL_LDFLAGS_RPATH="${rpath_arg}$with_openssl_rpath"
+ ],
AC_MSG_ERROR([--with-openssl-rpath "$with_openssl_rpath" is not a directory]))
]
)
AC_MSG_RESULT($OPENSSL_RPATH)
AC_SUBST([OPENSSL_RPATH])
+# This static linking is NOT OFFICIALLY SUPPORTED and not advertised.
+# Requires static OpenSSL build with position-independent code. Some features
+# like DSO engines or external OSSL providers don't work. Only tested with GCC
+# and clang on X86_64.
+AS_VAR_IF([PY_UNSUPPORTED_OPENSSL_BUILD], [static], [
+ AC_MSG_CHECKING([for unsupported static openssl build])
+ new_OPENSSL_LIBS=
+ for arg in $OPENSSL_LIBS; do
+ AS_CASE([$arg],
+ [-l*], [
+ libname=$(echo $arg | cut -c3-)
+ new_OPENSSL_LIBS="$new_OPENSSL_LIBS -l:lib${libname}.a -Wl,--exclude-libs,lib${libname}.a"
+ ],
+ [new_OPENSSL_LIBS="$new_OPENSSL_LIBS $arg"]
+ )
+ done
+ dnl include libz for OpenSSL build flavors with compression support
+ OPENSSL_LIBS="$new_OPENSSL_LIBS $ZLIB_LIBS"
+ AC_MSG_RESULT([$OPENSSL_LIBS])
+])
+
+dnl AX_CHECK_OPENSSL does not export libcrypto-only libs
+LIBCRYPTO_LIBS=
+for arg in $OPENSSL_LIBS; do
+ AS_CASE([$arg],
+ [-l*ssl*|-Wl*ssl*], [],
+ [LIBCRYPTO_LIBS="$LIBCRYPTO_LIBS $arg"]
+ )
+done
+
# check if OpenSSL libraries work as expected
-AC_CACHE_CHECK([whether OpenSSL provides required APIs], [ac_cv_working_openssl], [
-save_LIBS="$LIBS"
-save_CFLAGS="$CFLAGS"
-save_LDFLAGS="$LDFLAGS"
-LIBS="$LIBS $OPENSSL_LIBS"
-CFLAGS="$CFLAGS_NODIST $OPENSSL_INCLUDES"
-LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS"
+WITH_SAVE_ENV([
+ LIBS="$LIBS $OPENSSL_LIBS"
+ CFLAGS="$CFLAGS $OPENSSL_INCLUDES"
+ LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH"
-AC_LINK_IFELSE([AC_LANG_PROGRAM([[
-#include <openssl/opensslv.h>
-#include <openssl/evp.h>
-#include <openssl/ssl.h>
+ AC_CACHE_CHECK([whether OpenSSL provides required ssl module APIs], [ac_cv_working_openssl_ssl], [
+ AC_LINK_IFELSE([AC_LANG_PROGRAM([
+ #include <openssl/opensslv.h>
+ #include <openssl/ssl.h>
+ #if OPENSSL_VERSION_NUMBER < 0x10101000L
+ #error "OpenSSL >= 1.1.1 is required"
+ #endif
+ static void keylog_cb(const SSL *ssl, const char *line) {}
+ ], [
+ SSL_CTX *ctx = SSL_CTX_new(TLS_client_method());
+ SSL_CTX_set_keylog_callback(ctx, keylog_cb);
+ SSL *ssl = SSL_new(ctx);
+ X509_VERIFY_PARAM *param = SSL_get0_param(ssl);
+ X509_VERIFY_PARAM_set1_host(param, "python.org", 0);
+ SSL_free(ssl);
+ SSL_CTX_free(ctx);
+ ])], [ac_cv_working_openssl_ssl=yes], [ac_cv_working_openssl_ssl=no])
+ ])
+])
-#if OPENSSL_VERSION_NUMBER < 0x10101000L
-#error "OpenSSL >= 1.1.1 is required"
-#endif
+WITH_SAVE_ENV([
+ LIBS="$LIBS $LIBCRYPTO_LIBS"
+ CFLAGS="$CFLAGS $OPENSSL_INCLUDES"
+ LDFLAGS="$LDFLAGS $OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH"
-static void keylog_cb(const SSL *ssl, const char *line) {}
-]], [[
-/* SSL APIs */
-SSL_CTX *ctx = SSL_CTX_new(TLS_client_method());
-SSL_CTX_set_keylog_callback(ctx, keylog_cb);
-SSL *ssl = SSL_new(ctx);
-X509_VERIFY_PARAM *param = SSL_get0_param(ssl);
-X509_VERIFY_PARAM_set1_host(param, "python.org", 0);
-SSL_free(ssl);
-SSL_CTX_free(ctx);
-
-/* hashlib APIs */
-OBJ_nid2sn(NID_md5);
-OBJ_nid2sn(NID_sha1);
-OBJ_nid2sn(NID_sha3_512);
-OBJ_nid2sn(NID_blake2b512);
-EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0);
-]])],
- [ac_cv_working_openssl=yes],
- [ac_cv_working_openssl=no])
-LIBS="$save_LIBS"
-CFLAGS="$save_CFLAGS"
-LDFLAGS="$save_LDFLAGS"
+ AC_CACHE_CHECK([whether OpenSSL provides required hashlib module APIs], [ac_cv_working_openssl_hashlib], [
+ AC_LINK_IFELSE([AC_LANG_PROGRAM([
+ #include <openssl/opensslv.h>
+ #include <openssl/evp.h>
+ #if OPENSSL_VERSION_NUMBER < 0x10101000L
+ #error "OpenSSL >= 1.1.1 is required"
+ #endif
+ ], [
+ OBJ_nid2sn(NID_md5);
+ OBJ_nid2sn(NID_sha1);
+ OBJ_nid2sn(NID_sha3_512);
+ OBJ_nid2sn(NID_blake2b512);
+ EVP_PBE_scrypt(NULL, 0, NULL, 0, 2, 8, 1, 0, NULL, 0);
+ ])], [ac_cv_working_openssl_hashlib=yes], [ac_cv_working_openssl_hashlib=no])
+ ])
])
# ssl module default cipher suite string
@@ -6245,6 +6297,12 @@ PY_STDLIB_MOD([_bz2], [], [test "$have_bzip2" = yes],
PY_STDLIB_MOD([_lzma], [], [test "$have_liblzma" = yes],
[$LIBLZMA_CFLAGS], [$LIBLZMA_LIBS])
+dnl OpenSSL bindings
+PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes],
+ [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS])
+PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes],
+ [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS])
+
dnl test modules
PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes])
PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes])
diff --git a/setup.py b/setup.py
index 721103e7fceb7..7972bac46c106 100644
--- a/setup.py
+++ b/setup.py
@@ -1883,81 +1883,8 @@ def detect_decimal(self):
)
def detect_openssl_hashlib(self):
- # Detect SSL support for the socket module (via _ssl)
- config_vars = sysconfig.get_config_vars()
-
- def split_var(name, sep):
- # poor man's shlex, the re module is not available yet.
- value = config_vars.get(name)
- if not value:
- return ()
- # This trick works because ax_check_openssl uses --libs-only-L,
- # --libs-only-l, and --cflags-only-I.
- value = ' ' + value
- sep = ' ' + sep
- return [v.strip() for v in value.split(sep) if v.strip()]
-
- openssl_includes = split_var('OPENSSL_INCLUDES', '-I')
- openssl_libdirs = split_var('OPENSSL_LDFLAGS', '-L')
- openssl_libs = split_var('OPENSSL_LIBS', '-l')
- openssl_rpath = config_vars.get('OPENSSL_RPATH')
- if not openssl_libs:
- # libssl and libcrypto not found
- self.missing.extend(['_ssl', '_hashlib'])
- return None, None
-
- # Find OpenSSL includes
- ssl_incs = find_file(
- 'openssl/ssl.h', self.inc_dirs, openssl_includes
- )
- if ssl_incs is None:
- self.missing.extend(['_ssl', '_hashlib'])
- return None, None
-
- if openssl_rpath == 'auto':
- runtime_library_dirs = openssl_libdirs[:]
- elif not openssl_rpath:
- runtime_library_dirs = []
- else:
- runtime_library_dirs = [openssl_rpath]
-
- openssl_extension_kwargs = dict(
- include_dirs=openssl_includes,
- library_dirs=openssl_libdirs,
- libraries=openssl_libs,
- runtime_library_dirs=runtime_library_dirs,
- )
-
- # This static linking is NOT OFFICIALLY SUPPORTED.
- # Requires static OpenSSL build with position-independent code. Some
- # features like DSO engines or external OSSL providers don't work.
- # Only tested on GCC and clang on X86_64.
- if os.environ.get("PY_UNSUPPORTED_OPENSSL_BUILD") == "static":
- extra_linker_args = []
- for lib in openssl_extension_kwargs["libraries"]:
- # link statically
- extra_linker_args.append(f"-l:lib{lib}.a")
- # don't export symbols
- extra_linker_args.append(f"-Wl,--exclude-libs,lib{lib}.a")
- openssl_extension_kwargs["extra_link_args"] = extra_linker_args
- # don't link OpenSSL shared libraries.
- # include libz for OpenSSL build flavors with compression support
- openssl_extension_kwargs["libraries"] = ["z"]
-
- self.add(
- Extension(
- '_ssl',
- ['_ssl.c'],
- **openssl_extension_kwargs
- )
- )
- self.add(
- Extension(
- '_hashlib',
- ['_hashopenssl.c'],
- **openssl_extension_kwargs,
- )
- )
+ self.addext(Extension('_ssl', ['_ssl.c']))
+ self.addext(Extension('_hashlib', ['_hashopenssl.c']))
def detect_hash_builtins(self):
# By default we always compile these even when OpenSSL is available
1
0
https://github.com/python/cpython/commit/095bc8f0d6845dded8f67fbc6eca20dfac…
commit: 095bc8f0d6845dded8f67fbc6eca20dfac8b3929
branch: main
author: Christian Heimes <christian(a)python.org>
committer: tiran <christian(a)python.org>
date: 2021-11-23T22:26:50+01:00
summary:
bpo-45847: Port _crypt to PY_STDLIB_MOD (GH-29725)
files:
M Modules/Setup.stdlib.in
M configure
M configure.ac
M setup.py
diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in
index 338de84f09ebc..7422ddb59e309 100644
--- a/Modules/Setup.stdlib.in
+++ b/Modules/Setup.stdlib.in
@@ -98,6 +98,8 @@
# Modules with some UNIX dependencies
#
+# needs -lcrypt on some systems
+@MODULE__CRYPT_TRUE@_crypt _cryptmodule.c
@MODULE_FCNTL_TRUE@fcntl fcntlmodule.c
@MODULE_GRP_TRUE@grp grpmodule.c
@MODULE_MMAP_TRUE@mmap mmapmodule.c
diff --git a/configure b/configure
index 711b6d2805f16..01aa80091cd49 100755
--- a/configure
+++ b/configure
@@ -656,6 +656,8 @@ MODULE_NIS_FALSE
MODULE_NIS_TRUE
MODULE__DECIMAL_FALSE
MODULE__DECIMAL_TRUE
+MODULE__CRYPT_FALSE
+MODULE__CRYPT_TRUE
MODULE__BLAKE2_FALSE
MODULE__BLAKE2_TRUE
MODULE__SHA3_FALSE
@@ -784,6 +786,8 @@ HAVE_GETHOSTBYNAME_R
HAVE_GETHOSTBYNAME_R_3_ARG
HAVE_GETHOSTBYNAME_R_5_ARG
HAVE_GETHOSTBYNAME_R_6_ARG
+LIBCRYPT_LIBS
+LIBCRYPT_CFLAGS
LIBOBJS
LIBLZMA_LIBS
LIBLZMA_CFLAGS
@@ -1041,7 +1045,9 @@ ZLIB_LIBS
BZIP2_CFLAGS
BZIP2_LIBS
LIBLZMA_CFLAGS
-LIBLZMA_LIBS'
+LIBLZMA_LIBS
+LIBCRYPT_CFLAGS
+LIBCRYPT_LIBS'
# Initialize some variables set by options.
@@ -1831,6 +1837,10 @@ Some influential environment variables:
C compiler flags for LIBLZMA, overriding pkg-config
LIBLZMA_LIBS
linker flags for LIBLZMA, overriding pkg-config
+ LIBCRYPT_CFLAGS
+ C compiler flags for LIBCRYPT, overriding pkg-config
+ LIBCRYPT_LIBS
+ linker flags for LIBCRYPT, overriding pkg-config
Use these variables to override the choices made by `configure' or to help
it to find libraries and programs with nonstandard names/locations.
@@ -15251,11 +15261,75 @@ fi
done
-# We search for both crypt and crypt_r as one or the other may be defined
-# This gets us our -lcrypt in LIBS when required on the target platform.
-# Save/restore LIBS to avoid linking libpython with libcrypt.
-LIBS_SAVE=$LIBS
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing crypt_r" >&5
+
+
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBCRYPT" >&5
+$as_echo_n "checking for LIBCRYPT... " >&6; }
+
+if test -n "$LIBCRYPT_CFLAGS"; then
+ pkg_cv_LIBCRYPT_CFLAGS="$LIBCRYPT_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libxcrypt >= 3.1.1\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "libxcrypt >= 3.1.1") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LIBCRYPT_CFLAGS=`$PKG_CONFIG --cflags "libxcrypt >= 3.1.1" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LIBCRYPT_LIBS"; then
+ pkg_cv_LIBCRYPT_LIBS="$LIBCRYPT_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libxcrypt >= 3.1.1\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "libxcrypt >= 3.1.1") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LIBCRYPT_LIBS=`$PKG_CONFIG --libs "libxcrypt >= 3.1.1" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LIBCRYPT_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libxcrypt >= 3.1.1" 2>&1`
+ else
+ LIBCRYPT_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libxcrypt >= 3.1.1" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LIBCRYPT_PKG_ERRORS" >&5
+
+
+ save_CFLAGS=$CFLAGS
+save_CPPFLAGS=$CPPFLAGS
+save_LDFLAGS=$LDFLAGS
+save_LIBS=$LIBS
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing crypt_r" >&5
$as_echo_n "checking for library containing crypt_r... " >&6; }
if ${ac_cv_search_crypt_r+:} false; then :
$as_echo_n "(cached) " >&6
@@ -15309,12 +15383,37 @@ ac_res=$ac_cv_search_crypt_r
if test "$ac_res" != no; then :
test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+ $as_echo "#define HAVE_CRYPT_R 1" >>confdefs.h
+
+ if test "$ac_cv_search_crypt_r" = "none required"; then
+ LIBCRYPT_LIBS=
+ else
+ LIBCRYPT_LIBS="$ac_cv_search_crypt_r"
+ fi
+
fi
-LIBS="$LIBS_SAVE"
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing crypt" >&5
-$as_echo_n "checking for library containing crypt... " >&6; }
-if ${ac_cv_search_crypt+:} false; then :
+
+CFLAGS=$save_CFLAGS
+CPPFLAGS=$save_CPPFLAGS
+LDFLAGS=$save_LDFLAGS
+LIBS=$save_LIBS
+
+
+
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+ save_CFLAGS=$CFLAGS
+save_CPPFLAGS=$CPPFLAGS
+save_LDFLAGS=$LDFLAGS
+save_LIBS=$LIBS
+
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing crypt_r" >&5
+$as_echo_n "checking for library containing crypt_r... " >&6; }
+if ${ac_cv_search_crypt_r+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_func_search_save_LIBS=$LIBS
@@ -15327,11 +15426,11 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
#ifdef __cplusplus
extern "C"
#endif
-char crypt ();
+char crypt_r ();
int
main ()
{
-return crypt ();
+return crypt_r ();
;
return 0;
}
@@ -15344,59 +15443,112 @@ for ac_lib in '' crypt; do
LIBS="-l$ac_lib $ac_func_search_save_LIBS"
fi
if ac_fn_c_try_link "$LINENO"; then :
- ac_cv_search_crypt=$ac_res
+ ac_cv_search_crypt_r=$ac_res
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext
- if ${ac_cv_search_crypt+:} false; then :
+ if ${ac_cv_search_crypt_r+:} false; then :
break
fi
done
-if ${ac_cv_search_crypt+:} false; then :
+if ${ac_cv_search_crypt_r+:} false; then :
else
- ac_cv_search_crypt=no
+ ac_cv_search_crypt_r=no
fi
rm conftest.$ac_ext
LIBS=$ac_func_search_save_LIBS
fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_crypt" >&5
-$as_echo "$ac_cv_search_crypt" >&6; }
-ac_res=$ac_cv_search_crypt
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_crypt_r" >&5
+$as_echo "$ac_cv_search_crypt_r" >&6; }
+ac_res=$ac_cv_search_crypt_r
if test "$ac_res" != no; then :
test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
+ $as_echo "#define HAVE_CRYPT_R 1" >>confdefs.h
+
+ if test "$ac_cv_search_crypt_r" = "none required"; then
+ LIBCRYPT_LIBS=
+ else
+ LIBCRYPT_LIBS="$ac_cv_search_crypt_r"
+ fi
+
fi
-ac_fn_c_check_func "$LINENO" "crypt_r" "ac_cv_func_crypt_r"
-if test "x$ac_cv_func_crypt_r" = xyes; then :
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+CFLAGS=$save_CFLAGS
+CPPFLAGS=$save_CPPFLAGS
+LDFLAGS=$save_LDFLAGS
+LIBS=$save_LIBS
+
+
+
+else
+ LIBCRYPT_CFLAGS=$pkg_cv_LIBCRYPT_CFLAGS
+ LIBCRYPT_LIBS=$pkg_cv_LIBCRYPT_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+ $as_echo "#define HAVE_CRYPT_R 1" >>confdefs.h
+
+
+fi
+
+save_CFLAGS=$CFLAGS
+save_CPPFLAGS=$CPPFLAGS
+save_LDFLAGS=$LDFLAGS
+save_LIBS=$LIBS
+
+
+ CPPFLAGS="$LIBCRYPT_CFLAGS $CFLAGS"
+ LDFLAGS="$LIBCRYPT_LIBS $LDFLAGS"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for crypt or crypt_r" >&5
+$as_echo_n "checking for crypt or crypt_r... " >&6; }
+if ${ac_cv_crypt_crypt+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
-#include <crypt.h>
+ #ifdef HAVE_CRYPT_H
+ #include <crypt.h>
+ #endif
+ #include <unistd.h>
int
main ()
{
-struct crypt_data d;
-char *r = crypt_r("", "", &d);
+ #ifdef HAVE_CRYPT_R
+ void *x = crypt_r;
+ #else
+ void *x = crypt;
+ #endif
;
return 0;
}
-_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
-
-$as_echo "#define HAVE_CRYPT_R 1" >>confdefs.h
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ ac_cv_crypt_crypt=yes
+else
+ ac_cv_crypt_crypt=no
fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_crypt_crypt" >&5
+$as_echo "$ac_cv_crypt_crypt" >&6; }
+
+CFLAGS=$save_CFLAGS
+CPPFLAGS=$save_CPPFLAGS
+LDFLAGS=$save_LDFLAGS
+LIBS=$save_LIBS
+
-LIBS=$LIBS_SAVE
for ac_func in clock_gettime
do :
@@ -21381,6 +21533,42 @@ $as_echo "$py_cv_module__blake2" >&6; }
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _crypt" >&5
+$as_echo_n "checking for stdlib extension module _crypt... " >&6; }
+ case $py_stdlib_not_available in #(
+ *_crypt*) :
+ py_cv_module__crypt=n/a ;; #(
+ *) :
+ if true; then :
+ if test "$ac_cv_crypt_crypt" = yes; then :
+ py_cv_module__crypt=yes
+else
+ py_cv_module__crypt=missing
+fi
+else
+ py_cv_module__crypt=disabled
+fi
+ ;;
+esac
+ as_fn_append MODULE_BLOCK "MODULE__CRYPT=$py_cv_module__crypt$as_nl"
+ if test "x$py_cv_module__crypt" = xyes; then :
+
+ as_fn_append MODULE_BLOCK "MODULE__CRYPT_CFLAGS=$LIBCRYPT_CFLAGS$as_nl"
+ as_fn_append MODULE_BLOCK "MODULE__CRYPT_LDFLAGS=$LIBCRYPT_LIBS$as_nl"
+
+fi
+ if test "$py_cv_module__crypt" = yes; then
+ MODULE__CRYPT_TRUE=
+ MODULE__CRYPT_FALSE='#'
+else
+ MODULE__CRYPT_TRUE='#'
+ MODULE__CRYPT_FALSE=
+fi
+
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__crypt" >&5
+$as_echo "$py_cv_module__crypt" >&6; }
+
+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _decimal" >&5
$as_echo_n "checking for stdlib extension module _decimal... " >&6; }
case $py_stdlib_not_available in #(
@@ -22261,6 +22449,10 @@ if test -z "${MODULE__BLAKE2_TRUE}" && test -z "${MODULE__BLAKE2_FALSE}"; then
as_fn_error $? "conditional \"MODULE__BLAKE2\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
fi
+if test -z "${MODULE__CRYPT_TRUE}" && test -z "${MODULE__CRYPT_FALSE}"; then
+ as_fn_error $? "conditional \"MODULE__CRYPT\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
if test -z "${MODULE__DECIMAL_TRUE}" && test -z "${MODULE__DECIMAL_FALSE}"; then
as_fn_error $? "conditional \"MODULE__DECIMAL\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
diff --git a/configure.ac b/configure.ac
index e539d3bb87d66..0008e8a746c7b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -4190,25 +4190,46 @@ AC_CHECK_FUNCS(setpgrp,
[])
)
-# We search for both crypt and crypt_r as one or the other may be defined
-# This gets us our -lcrypt in LIBS when required on the target platform.
-# Save/restore LIBS to avoid linking libpython with libcrypt.
-LIBS_SAVE=$LIBS
-AC_SEARCH_LIBS(crypt_r, crypt)
-LIBS="$LIBS_SAVE"
-AC_SEARCH_LIBS(crypt, crypt)
+dnl We search for both crypt and crypt_r as one or the other may be defined
+dnl libxcrypt provides <crypt.h> and libcrypt with crypt_r() since
+dnl at least 3.1.1 from 2015.
+dnl FreeBSD defines crypt_r() in <unistd.h>
+AH_TEMPLATE([HAVE_CRYPT_R], [Define if you have the crypt_r() function.])
+
+PKG_CHECK_MODULES([LIBCRYPT], [libxcrypt >= 3.1.1], [
+ AC_DEFINE([HAVE_CRYPT_R], [1])
+], [
+ WITH_SAVE_ENV([
+ AC_SEARCH_LIBS([crypt_r], [crypt], [
+ AC_DEFINE([HAVE_CRYPT_R], [1])
+ if test "$ac_cv_search_crypt_r" = "none required"; then
+ LIBCRYPT_LIBS=
+ else
+ LIBCRYPT_LIBS="$ac_cv_search_crypt_r"
+ fi
+ ])
+ ])
+])
-AC_CHECK_FUNC(crypt_r,
- AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
-#include <crypt.h>
-]], [[
-struct crypt_data d;
-char *r = crypt_r("", "", &d);
-]])],
- [AC_DEFINE(HAVE_CRYPT_R, 1, [Define if you have the crypt_r() function.])],
- [])
-)
-LIBS=$LIBS_SAVE
+WITH_SAVE_ENV([
+ CPPFLAGS="$LIBCRYPT_CFLAGS $CFLAGS"
+ LDFLAGS="$LIBCRYPT_LIBS $LDFLAGS"
+ AC_CACHE_CHECK([for crypt or crypt_r], [ac_cv_crypt_crypt], [
+ AC_LINK_IFELSE([AC_LANG_PROGRAM([
+ #ifdef HAVE_CRYPT_H
+ #include <crypt.h>
+ #endif
+ #include <unistd.h>
+ ], [
+ #ifdef HAVE_CRYPT_R
+ void *x = crypt_r;
+ #else
+ void *x = crypt;
+ #endif
+ ])
+ ], [ac_cv_crypt_crypt=yes], [ac_cv_crypt_crypt=no])
+ ])
+])
AC_CHECK_FUNCS(clock_gettime, [], [
AC_CHECK_LIB(rt, clock_gettime, [
@@ -6202,6 +6223,9 @@ PY_STDLIB_MOD([_sha512], [test "$with_builtin_sha512" = yes])
PY_STDLIB_MOD([_sha3], [test "$with_builtin_sha3" = yes])
PY_STDLIB_MOD([_blake2], [test "$with_builtin_blake2" = yes])
+PY_STDLIB_MOD([_crypt],
+ [], [test "$ac_cv_crypt_crypt" = yes],
+ [$LIBCRYPT_CFLAGS], [$LIBCRYPT_LIBS])
PY_STDLIB_MOD([_decimal], [], [], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LDFLAGS])
PY_STDLIB_MOD([nis],
[], [test "$have_nis" = yes -a "$ac_cv_header_rpc_rpc_h" = yes],
diff --git a/setup.py b/setup.py
index 304168a657c85..721103e7fceb7 100644
--- a/setup.py
+++ b/setup.py
@@ -1241,20 +1241,7 @@ def detect_readline_curses(self):
self.missing.append('_curses_panel')
def detect_crypt(self):
- # crypt module.
- if VXWORKS:
- # bpo-31904: crypt() function is not provided by VxWorks.
- # DES_crypt() OpenSSL provides is too weak to implement
- # the encryption.
- self.missing.append('_crypt')
- return
-
- if self.compiler.find_library_file(self.lib_dirs, 'crypt'):
- libs = ['crypt']
- else:
- libs = []
-
- self.add(Extension('_crypt', ['_cryptmodule.c'], libraries=libs))
+ self.addext(Extension('_crypt', ['_cryptmodule.c']))
def detect_dbm_gdbm(self):
# Modules that provide persistent dictionary-like semantics. You will
1
0
https://github.com/python/cpython/commit/f840398a5fd8741653c26eb8641c48656c…
commit: f840398a5fd8741653c26eb8641c48656c9800d4
branch: main
author: Christian Heimes <christian(a)python.org>
committer: tiran <christian(a)python.org>
date: 2021-11-23T21:36:40+01:00
summary:
bpo-45873: Restore Python 3.6 compatibility (GH-29730)
Co-authored-by: Guido van Rossum <gvanrossum(a)gmail.com>
files:
M Tools/scripts/deepfreeze.py
M configure
M configure.ac
diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py
index b840c4b51d73a..e51a408ae7f6a 100644
--- a/Tools/scripts/deepfreeze.py
+++ b/Tools/scripts/deepfreeze.py
@@ -5,8 +5,10 @@
import contextlib
import os
import re
+import sys
import time
import types
+import unicodedata
from typing import Dict, FrozenSet, Tuple, TextIO
import umarshal
@@ -14,9 +16,13 @@
verbose = False
+def isprintable(b: bytes) -> bool:
+ return all(0x20 <= c < 0x7f for c in b)
+
+
def make_string_literal(b: bytes) -> str:
res = ['"']
- if b.isascii() and b.decode("ascii").isprintable():
+ if isprintable(b):
res.append(b.decode("ascii").replace("\\", "\\\\").replace("\"", "\\\""))
else:
for i in b:
diff --git a/configure b/configure
index 0684921f41a42..711b6d2805f16 100755
--- a/configure
+++ b/configure
@@ -3142,7 +3142,7 @@ case $host_os in *\ *) host_os=`echo "$host_os" | sed 's/ /-/g'`;; esac
# pybuilddir.txt will be created by --generate-posix-vars in the Makefile
rm -f pybuilddir.txt
-for ac_prog in python$PACKAGE_VERSION python3 python
+for ac_prog in python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python
do
# Extract the first word of "$ac_prog", so it can be a program name with args.
set dummy $ac_prog; ac_word=$2
@@ -3187,6 +3187,16 @@ test -n "$PYTHON_FOR_REGEN" || PYTHON_FOR_REGEN="python3"
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking Python for regen version" >&5
+$as_echo_n "checking Python for regen version... " >&6; }
+if command -v $PYTHON_FOR_REGEN >/dev/null 2>&1; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $($PYTHON_FOR_REGEN -V 2>/dev/null)" >&5
+$as_echo "$($PYTHON_FOR_REGEN -V 2>/dev/null)" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: missing" >&5
+$as_echo "missing" >&6; }
+fi
+
if test "$cross_compiling" = yes; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for python interpreter for cross build" >&5
$as_echo_n "checking for python interpreter for cross build... " >&6; }
diff --git a/configure.ac b/configure.ac
index ba22418b40694..e539d3bb87d66 100644
--- a/configure.ac
+++ b/configure.ac
@@ -96,9 +96,18 @@ AC_SUBST(host)
# pybuilddir.txt will be created by --generate-posix-vars in the Makefile
rm -f pybuilddir.txt
-AC_CHECK_PROGS(PYTHON_FOR_REGEN, python$PACKAGE_VERSION python3 python, python3)
+AC_CHECK_PROGS([PYTHON_FOR_REGEN],
+ [python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python],
+ [python3])
AC_SUBST(PYTHON_FOR_REGEN)
+AC_MSG_CHECKING([Python for regen version])
+if command -v $PYTHON_FOR_REGEN >/dev/null 2>&1; then
+ AC_MSG_RESULT([$($PYTHON_FOR_REGEN -V 2>/dev/null)])
+else
+ AC_MSG_RESULT([missing])
+fi
+
if test "$cross_compiling" = yes; then
AC_MSG_CHECKING([for python interpreter for cross build])
if test -z "$PYTHON_FOR_BUILD"; then
1
0
https://github.com/python/cpython/commit/4ae26b9c1d0c33e3db92c6f305293f9240…
commit: 4ae26b9c1d0c33e3db92c6f305293f9240dea358
branch: main
author: Victor Stinner <vstinner(a)python.org>
committer: vstinner <vstinner(a)python.org>
date: 2021-11-23T18:58:57+01:00
summary:
bpo-39026: Fix Python.h when building with Xcode (GH-29488)
Fix Python.h to build C extensions with Xcode: remove a relative
include from Include/cpython/pystate.h.
files:
A Misc/NEWS.d/next/C API/2021-11-09-15-42-11.bpo-39026.sUnYWn.rst
M Include/Python.h
M Include/cpython/pystate.h
diff --git a/Include/Python.h b/Include/Python.h
index 6e3303ac9a3b0..7260ae5cd0b4f 100644
--- a/Include/Python.h
+++ b/Include/Python.h
@@ -72,6 +72,7 @@
#include "sliceobject.h"
#include "cpython/cellobject.h"
#include "iterobject.h"
+#include "cpython/initconfig.h"
#include "pystate.h"
#include "cpython/genobject.h"
#include "descrobject.h"
@@ -83,7 +84,6 @@
#include "cpython/pytime.h"
#include "codecs.h"
#include "pyerrors.h"
-#include "cpython/initconfig.h"
#include "pythread.h"
#include "cpython/context.h"
#include "modsupport.h"
diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h
index 9ac0a298baab4..5adbf3b495316 100644
--- a/Include/cpython/pystate.h
+++ b/Include/cpython/pystate.h
@@ -2,8 +2,6 @@
# error "this header file must not be included directly"
#endif
-#include "cpython/initconfig.h"
-
PyAPI_FUNC(int) _PyInterpreterState_RequiresIDRef(PyInterpreterState *);
PyAPI_FUNC(void) _PyInterpreterState_RequireIDRef(PyInterpreterState *, int);
diff --git a/Misc/NEWS.d/next/C API/2021-11-09-15-42-11.bpo-39026.sUnYWn.rst b/Misc/NEWS.d/next/C API/2021-11-09-15-42-11.bpo-39026.sUnYWn.rst
new file mode 100644
index 0000000000000..77a0119792152
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2021-11-09-15-42-11.bpo-39026.sUnYWn.rst
@@ -0,0 +1,2 @@
+Fix Python.h to build C extensions with Xcode: remove a relative include
+from ``Include/cpython/pystate.h``.
1
0
https://github.com/python/cpython/commit/5be98e57b3c3b36d1a1176b49c73b8822c…
commit: 5be98e57b3c3b36d1a1176b49c73b8822c6380e7
branch: main
author: Guido van Rossum <guido(a)python.org>
committer: gvanrossum <gvanrossum(a)gmail.com>
date: 2021-11-23T08:56:06-08:00
summary:
bpo-45873: Get rid of bootstrap_python (#29717)
Instead we use $(PYTHON_FOR_REGEN) .../deepfreeze.py with the
frozen .h file as input, as we did for Windows in bpo-45850.
We also get rid of the code that generates the .h files
when make regen-frozen is run (i.e., .../make_frozen.py),
and the MANIFEST file.
Restore Python 3.8 and 3.9 as Windows host Python again
Co-authored-by: Kumar Aditya <59607654+kumaraditya303(a)users.noreply.github.com>
files:
A Misc/NEWS.d/next/Build/2021-11-23-04-28-40.bpo-45873.9dldZ4.rst
M Makefile.pre.in
M PCbuild/find_python.bat
M Tools/scripts/deepfreeze.py
M Tools/scripts/freeze_modules.py
M Tools/scripts/umarshal.py
M configure
M configure.ac
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 046b0e86e6758..476a64cb7a500 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -275,7 +275,6 @@ LIBOBJS= @LIBOBJS@
PYTHON= python$(EXE)
BUILDPYTHON= python$(BUILDEXE)
-BOOTSTRAP= _bootstrap_python
PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
@@ -710,9 +709,9 @@ platform: $(BUILDPYTHON) pybuilddir.txt
# problems by creating a dummy pybuilddir.txt just to allow interpreter
# initialization to succeed. It will be overwritten by generate-posix-vars
# or removed in case of failure.
-pybuilddir.txt: $(BOOTSTRAP)
+pybuilddir.txt: $(BUILDPYTHON)
@echo "none" > ./pybuilddir.txt
- ./$(BOOTSTRAP) -S -m sysconfig --generate-posix-vars ;\
+ $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\
if test $$? -ne 0 ; then \
echo "generate-posix-vars failed" ; \
rm -f ./pybuilddir.txt ; \
@@ -934,155 +933,75 @@ regen-test-frozenmain: $(BUILDPYTHON)
Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
-############################################################################
-# "Bootstrap Python" used to run deepfreeze.py
-
-BOOTSTRAP_HEADERS = \
- Python/frozen_modules/importlib._bootstrap.h \
- Python/frozen_modules/importlib._bootstrap_external.h \
- Python/frozen_modules/zipimport.h
-
-Python/bootstrap_frozen.o: Python/bootstrap_frozen.c Include/cpython/import.h $(BOOTSTRAP_HEADERS)
-
-$(BOOTSTRAP): $(LIBRARY_OBJS_OMIT_FROZEN) \
- Python/bootstrap_frozen.o Programs/python.o
- $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \
- Python/bootstrap_frozen.o \
- Programs/python.o \
- $(LIBS) $(MODLIBS) $(SYSLIBS)
-
############################################################################
# Deepfreeze targets
.PHONY: regen-deepfreeze
regen-deepfreeze: $(DEEPFREEZE_OBJS)
-DEEPFREEZE_DEPS = \
- $(BOOTSTRAP) \
- pybuilddir.txt \
- $(srcdir)/Tools/scripts/deepfreeze.py
+DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py
# BEGIN: deepfreeze modules
-Python/deepfreeze/importlib._bootstrap.c: $(srcdir)/Lib/importlib/_bootstrap.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/importlib._bootstrap.c from Lib/importlib/_bootstrap.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/importlib/_bootstrap.py -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
-
-Python/deepfreeze/importlib._bootstrap_external.c: $(srcdir)/Lib/importlib/_bootstrap_external.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/importlib._bootstrap_external.c from Lib/importlib/_bootstrap_external.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/importlib/_bootstrap_external.py -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
-
-Python/deepfreeze/zipimport.c: $(srcdir)/Lib/zipimport.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/zipimport.c from Lib/zipimport.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/zipimport.py -m zipimport -o Python/deepfreeze/zipimport.c
-
-Python/deepfreeze/abc.c: $(srcdir)/Lib/abc.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/abc.c from Lib/abc.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/abc.py -m abc -o Python/deepfreeze/abc.c
-
-Python/deepfreeze/codecs.c: $(srcdir)/Lib/codecs.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/codecs.c from Lib/codecs.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/codecs.py -m codecs -o Python/deepfreeze/codecs.c
-
-Python/deepfreeze/io.c: $(srcdir)/Lib/io.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/io.c from Lib/io.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/io.py -m io -o Python/deepfreeze/io.c
-
-Python/deepfreeze/_collections_abc.c: $(srcdir)/Lib/_collections_abc.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/_collections_abc.c from Lib/_collections_abc.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/_collections_abc.py -m _collections_abc -o Python/deepfreeze/_collections_abc.c
-
-Python/deepfreeze/_sitebuiltins.c: $(srcdir)/Lib/_sitebuiltins.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/_sitebuiltins.c from Lib/_sitebuiltins.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/_sitebuiltins.py -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
-
-Python/deepfreeze/genericpath.c: $(srcdir)/Lib/genericpath.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/genericpath.c from Lib/genericpath.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/genericpath.py -m genericpath -o Python/deepfreeze/genericpath.c
-
-Python/deepfreeze/ntpath.c: $(srcdir)/Lib/ntpath.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/ntpath.c from Lib/ntpath.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/ntpath.py -m ntpath -o Python/deepfreeze/ntpath.c
-
-Python/deepfreeze/posixpath.c: $(srcdir)/Lib/posixpath.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/posixpath.c from Lib/posixpath.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/posixpath.py -m posixpath -o Python/deepfreeze/posixpath.c
-
-Python/deepfreeze/os.c: $(srcdir)/Lib/os.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/os.c from Lib/os.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/os.py -m os -o Python/deepfreeze/os.c
-
-Python/deepfreeze/site.c: $(srcdir)/Lib/site.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/site.c from Lib/site.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/site.py -m site -o Python/deepfreeze/site.c
-
-Python/deepfreeze/stat.c: $(srcdir)/Lib/stat.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/stat.c from Lib/stat.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/stat.py -m stat -o Python/deepfreeze/stat.c
-
-Python/deepfreeze/__hello__.c: $(srcdir)/Lib/__hello__.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/__hello__.c from Lib/__hello__.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/__hello__.py -m __hello__ -o Python/deepfreeze/__hello__.c
-
-Python/deepfreeze/__phello__.c: $(srcdir)/Lib/__phello__/__init__.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/__phello__.c from Lib/__phello__/__init__.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/__phello__/__init__.py -m __phello__ -o Python/deepfreeze/__phello__.c
-
-Python/deepfreeze/__phello__.ham.c: $(srcdir)/Lib/__phello__/ham/__init__.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/__phello__.ham.c from Lib/__phello__/ham/__init__.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/__phello__/ham/__init__.py -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
-
-Python/deepfreeze/__phello__.ham.eggs.c: $(srcdir)/Lib/__phello__/ham/eggs.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/__phello__.ham.eggs.c from Lib/__phello__/ham/eggs.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/__phello__/ham/eggs.py -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
-
-Python/deepfreeze/__phello__.spam.c: $(srcdir)/Lib/__phello__/spam.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/__phello__.spam.c from Lib/__phello__/spam.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Lib/__phello__/spam.py -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
-
-Python/deepfreeze/frozen_only.c: $(srcdir)/Tools/freeze/flag.py $(DEEPFREEZE_DEPS)
- @echo "Deepfreezing Python/deepfreeze/frozen_only.c from Tools/freeze/flag.py"
- @./$(BOOTSTRAP) \
- $(srcdir)/Tools/scripts/deepfreeze.py \
- $(srcdir)/Tools/freeze/flag.py -m frozen_only -o Python/deepfreeze/frozen_only.c
+Python/deepfreeze/importlib._bootstrap.c: Python/frozen_modules/importlib._bootstrap.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
+
+Python/deepfreeze/importlib._bootstrap_external.c: Python/frozen_modules/importlib._bootstrap_external.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
+
+Python/deepfreeze/zipimport.c: Python/frozen_modules/zipimport.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c
+
+Python/deepfreeze/abc.c: Python/frozen_modules/abc.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c
+
+Python/deepfreeze/codecs.c: Python/frozen_modules/codecs.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c
+
+Python/deepfreeze/io.c: Python/frozen_modules/io.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c
+
+Python/deepfreeze/_collections_abc.c: Python/frozen_modules/_collections_abc.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c
+
+Python/deepfreeze/_sitebuiltins.c: Python/frozen_modules/_sitebuiltins.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
+
+Python/deepfreeze/genericpath.c: Python/frozen_modules/genericpath.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c
+
+Python/deepfreeze/ntpath.c: Python/frozen_modules/ntpath.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c
+
+Python/deepfreeze/posixpath.c: Python/frozen_modules/posixpath.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c
+
+Python/deepfreeze/os.c: Python/frozen_modules/os.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c
+
+Python/deepfreeze/site.c: Python/frozen_modules/site.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c
+
+Python/deepfreeze/stat.c: Python/frozen_modules/stat.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c
+
+Python/deepfreeze/__hello__.c: Python/frozen_modules/__hello__.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c
+
+Python/deepfreeze/__phello__.c: Python/frozen_modules/__phello__.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c
+
+Python/deepfreeze/__phello__.ham.c: Python/frozen_modules/__phello__.ham.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
+
+Python/deepfreeze/__phello__.ham.eggs.c: Python/frozen_modules/__phello__.ham.eggs.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
+
+Python/deepfreeze/__phello__.spam.c: Python/frozen_modules/__phello__.spam.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
+
+Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZE_DEPS)
+ $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c
# END: deepfreeze modules
@@ -1143,64 +1062,64 @@ Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN)
# BEGIN: freezing modules
Python/frozen_modules/importlib._bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py
- Programs/_freeze_module importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/frozen_modules/importlib._bootstrap.h
+ Programs/_freeze_module importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h
Python/frozen_modules/importlib._bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py
- Programs/_freeze_module importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py $(srcdir)/Python/frozen_modules/importlib._bootstrap_external.h
+ Programs/_freeze_module importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h
Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py
- Programs/_freeze_module zipimport $(srcdir)/Lib/zipimport.py $(srcdir)/Python/frozen_modules/zipimport.h
+ Programs/_freeze_module zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h
Python/frozen_modules/abc.h: Programs/_freeze_module Lib/abc.py
- Programs/_freeze_module abc $(srcdir)/Lib/abc.py $(srcdir)/Python/frozen_modules/abc.h
+ Programs/_freeze_module abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h
Python/frozen_modules/codecs.h: Programs/_freeze_module Lib/codecs.py
- Programs/_freeze_module codecs $(srcdir)/Lib/codecs.py $(srcdir)/Python/frozen_modules/codecs.h
+ Programs/_freeze_module codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h
Python/frozen_modules/io.h: Programs/_freeze_module Lib/io.py
- Programs/_freeze_module io $(srcdir)/Lib/io.py $(srcdir)/Python/frozen_modules/io.h
+ Programs/_freeze_module io $(srcdir)/Lib/io.py Python/frozen_modules/io.h
Python/frozen_modules/_collections_abc.h: Programs/_freeze_module Lib/_collections_abc.py
- Programs/_freeze_module _collections_abc $(srcdir)/Lib/_collections_abc.py $(srcdir)/Python/frozen_modules/_collections_abc.h
+ Programs/_freeze_module _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h
Python/frozen_modules/_sitebuiltins.h: Programs/_freeze_module Lib/_sitebuiltins.py
- Programs/_freeze_module _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py $(srcdir)/Python/frozen_modules/_sitebuiltins.h
+ Programs/_freeze_module _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h
Python/frozen_modules/genericpath.h: Programs/_freeze_module Lib/genericpath.py
- Programs/_freeze_module genericpath $(srcdir)/Lib/genericpath.py $(srcdir)/Python/frozen_modules/genericpath.h
+ Programs/_freeze_module genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h
Python/frozen_modules/ntpath.h: Programs/_freeze_module Lib/ntpath.py
- Programs/_freeze_module ntpath $(srcdir)/Lib/ntpath.py $(srcdir)/Python/frozen_modules/ntpath.h
+ Programs/_freeze_module ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h
Python/frozen_modules/posixpath.h: Programs/_freeze_module Lib/posixpath.py
- Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py $(srcdir)/Python/frozen_modules/posixpath.h
+ Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h
Python/frozen_modules/os.h: Programs/_freeze_module Lib/os.py
- Programs/_freeze_module os $(srcdir)/Lib/os.py $(srcdir)/Python/frozen_modules/os.h
+ Programs/_freeze_module os $(srcdir)/Lib/os.py Python/frozen_modules/os.h
Python/frozen_modules/site.h: Programs/_freeze_module Lib/site.py
- Programs/_freeze_module site $(srcdir)/Lib/site.py $(srcdir)/Python/frozen_modules/site.h
+ Programs/_freeze_module site $(srcdir)/Lib/site.py Python/frozen_modules/site.h
Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py
- Programs/_freeze_module stat $(srcdir)/Lib/stat.py $(srcdir)/Python/frozen_modules/stat.h
+ Programs/_freeze_module stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h
Python/frozen_modules/__hello__.h: Programs/_freeze_module Lib/__hello__.py
- Programs/_freeze_module __hello__ $(srcdir)/Lib/__hello__.py $(srcdir)/Python/frozen_modules/__hello__.h
+ Programs/_freeze_module __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h
Python/frozen_modules/__phello__.h: Programs/_freeze_module Lib/__phello__/__init__.py
- Programs/_freeze_module __phello__ $(srcdir)/Lib/__phello__/__init__.py $(srcdir)/Python/frozen_modules/__phello__.h
+ Programs/_freeze_module __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h
Python/frozen_modules/__phello__.ham.h: Programs/_freeze_module Lib/__phello__/ham/__init__.py
- Programs/_freeze_module __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py $(srcdir)/Python/frozen_modules/__phello__.ham.h
+ Programs/_freeze_module __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h
Python/frozen_modules/__phello__.ham.eggs.h: Programs/_freeze_module Lib/__phello__/ham/eggs.py
- Programs/_freeze_module __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py $(srcdir)/Python/frozen_modules/__phello__.ham.eggs.h
+ Programs/_freeze_module __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h
Python/frozen_modules/__phello__.spam.h: Programs/_freeze_module Lib/__phello__/spam.py
- Programs/_freeze_module __phello__.spam $(srcdir)/Lib/__phello__/spam.py $(srcdir)/Python/frozen_modules/__phello__.spam.h
+ Programs/_freeze_module __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h
Python/frozen_modules/frozen_only.h: Programs/_freeze_module Tools/freeze/flag.py
- Programs/_freeze_module frozen_only $(srcdir)/Tools/freeze/flag.py $(srcdir)/Python/frozen_modules/frozen_only.h
+ Programs/_freeze_module frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h
# END: freezing modules
@@ -2368,7 +2287,7 @@ clean-retain-profile: pycremoval
find build -name '*.py[co]' -exec rm -f {} ';' || true
-rm -f pybuilddir.txt
-rm -f Lib/lib2to3/*Grammar*.pickle
- -rm -f Programs/_testembed Programs/_freeze_module $(BOOTSTRAP)
+ -rm -f Programs/_testembed Programs/_freeze_module
-rm -f Python/deepfreeze/*.[co]
-find build -type f -a ! -name '*.gc??' -exec rm -f {} ';'
-rm -f Include/pydtrace_probes.h
diff --git a/Misc/NEWS.d/next/Build/2021-11-23-04-28-40.bpo-45873.9dldZ4.rst b/Misc/NEWS.d/next/Build/2021-11-23-04-28-40.bpo-45873.9dldZ4.rst
new file mode 100644
index 0000000000000..901a3d6650b61
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2021-11-23-04-28-40.bpo-45873.9dldZ4.rst
@@ -0,0 +1 @@
+Get rid of the ``_bootstrap_python`` build step. The deepfreeze.py script is now run using ``$(PYTHON_FOR_REGEN)`` which can be Python 3.7 or newer (on Windows, 3.8 or newer).
\ No newline at end of file
diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat
index a9f14c5277ffa..a9bbe5a6279a6 100644
--- a/PCbuild/find_python.bat
+++ b/PCbuild/find_python.bat
@@ -31,13 +31,13 @@
@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
@rem If we have Python in externals, use that one
-@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86"
+@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86"
@rem If HOST_PYTHON is recent enough, use that
-@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
+@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
@rem If py.exe finds a recent enough version, use that one
-@for %%p in (3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found
+@for %%p in (3.10 3.9 3.8) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found
@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
@set _Py_NUGET=%NUGET%
diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py
index b6d52b7454965..b840c4b51d73a 100644
--- a/Tools/scripts/deepfreeze.py
+++ b/Tools/scripts/deepfreeze.py
@@ -7,7 +7,7 @@
import re
import time
import types
-import typing
+from typing import Dict, FrozenSet, Tuple, TextIO
import umarshal
@@ -42,13 +42,14 @@ def get_localsplus(code: types.CodeType):
def get_localsplus_counts(code: types.CodeType,
- names: tuple[str, ...],
- kinds: bytes) -> tuple[int, int, int, int]:
+ names: Tuple[str, ...],
+ kinds: bytes) -> Tuple[int, int, int, int]:
nlocals = 0
nplaincellvars = 0
ncellvars = 0
nfreevars = 0
- for name, kind in zip(names, kinds, strict=True):
+ assert len(names) == len(kinds)
+ for name, kind in zip(names, kinds):
if kind & CO_FAST_LOCAL:
nlocals += 1
if kind & CO_FAST_CELL:
@@ -71,7 +72,7 @@ def get_localsplus_counts(code: types.CodeType,
PyUnicode_4BYTE_KIND = 4
-def analyze_character_width(s: str) -> tuple[int, bool]:
+def analyze_character_width(s: str) -> Tuple[int, bool]:
maxchar = ' '
for c in s:
maxchar = max(maxchar, c)
@@ -86,12 +87,17 @@ def analyze_character_width(s: str) -> tuple[int, bool]:
return kind, ascii
+def removesuffix(base: str, suffix: str) -> str:
+ if base.endswith(suffix):
+ return base[:len(base) - len(suffix)]
+ return base
+
class Printer:
- def __init__(self, file: typing.TextIO):
+ def __init__(self, file: TextIO):
self.level = 0
self.file = file
- self.cache: dict[tuple[type, object], str] = {}
+ self.cache: Dict[Tuple[type, object], str] = {}
self.hits, self.misses = 0, 0
self.patchups: list[str] = []
self.write('#include "Python.h"')
@@ -231,7 +237,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str:
# otherwise MSVC doesn't like it.
self.write(f".co_consts = {co_consts},")
self.write(f".co_names = {co_names},")
- self.write(f".co_firstinstr = (_Py_CODEUNIT *) {co_code.removesuffix('.ob_base.ob_base')}.ob_sval,")
+ self.write(f".co_firstinstr = (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval,")
self.write(f".co_exceptiontable = {co_exceptiontable},")
self.field(code, "co_flags")
self.write(".co_warmup = QUICKENING_INITIAL_WARMUP_VALUE,")
@@ -259,7 +265,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str:
self.write(f".co_freevars = {co_freevars},")
return f"& {name}.ob_base"
- def generate_tuple(self, name: str, t: tuple[object, ...]) -> str:
+ def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str:
items = [self.generate(f"{name}_{i}", it) for i, it in enumerate(t)]
self.write("static")
with self.indent():
@@ -323,7 +329,7 @@ def generate_complex(self, name: str, z: complex) -> str:
self.write(f".cval = {{ {z.real}, {z.imag} }},")
return f"&{name}.ob_base"
- def generate_frozenset(self, name: str, fs: frozenset[object]) -> str:
+ def generate_frozenset(self, name: str, fs: FrozenSet[object]) -> str:
ret = self.generate_tuple(name, tuple(sorted(fs)))
self.write("// TODO: The above tuple should be a frozenset")
return ret
@@ -336,34 +342,33 @@ def generate(self, name: str, obj: object) -> str:
# print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}")
return self.cache[key]
self.misses += 1
- match obj:
- case types.CodeType() | umarshal.Code() as code:
- val = self.generate_code(name, code)
- case tuple(t):
- val = self.generate_tuple(name, t)
- case str(s):
- val = self.generate_unicode(name, s)
- case bytes(b):
- val = self.generate_bytes(name, b)
- case True:
- return "Py_True"
- case False:
- return "Py_False"
- case int(i):
- val = self.generate_int(name, i)
- case float(x):
- val = self.generate_float(name, x)
- case complex() as z:
- val = self.generate_complex(name, z)
- case frozenset(fs):
- val = self.generate_frozenset(name, fs)
- case builtins.Ellipsis:
- return "Py_Ellipsis"
- case None:
- return "Py_None"
- case _:
- raise TypeError(
- f"Cannot generate code for {type(obj).__name__} object")
+ if isinstance(obj, types.CodeType) or isinstance(obj, umarshal.Code):
+ val = self.generate_code(name, obj)
+ elif isinstance(obj, tuple):
+ val = self.generate_tuple(name, obj)
+ elif isinstance(obj, str):
+ val = self.generate_unicode(name, obj)
+ elif isinstance(obj, bytes):
+ val = self.generate_bytes(name, obj)
+ elif obj is True:
+ return "Py_True"
+ elif obj is False:
+ return "Py_False"
+ elif isinstance(obj, int):
+ val = self.generate_int(name, obj)
+ elif isinstance(obj, float):
+ val = self.generate_float(name, obj)
+ elif isinstance(obj, complex):
+ val = self.generate_complex(name, obj)
+ elif isinstance(obj, frozenset):
+ val = self.generate_frozenset(name, obj)
+ elif obj is builtins.Ellipsis:
+ return "Py_Ellipsis"
+ elif obj is None:
+ return "Py_None"
+ else:
+ raise TypeError(
+ f"Cannot generate code for {type(obj).__name__} object")
# print(f"Cache store {key!r:.40}: {val!r:.40}")
self.cache[key] = val
return val
@@ -393,12 +398,12 @@ def decode_frozen_data(source: str) -> types.CodeType:
del lines[0]
while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None:
del lines[-1]
- values: tuple[int, ...] = ast.literal_eval("".join(lines))
+ values: Tuple[int, ...] = ast.literal_eval("".join(lines).strip())
data = bytes(values)
return umarshal.loads(data)
-def generate(source: str, filename: str, modname: str, file: typing.TextIO) -> None:
+def generate(source: str, filename: str, modname: str, file: TextIO) -> None:
if is_frozen_header(source):
code = decode_frozen_data(source)
else:
@@ -439,7 +444,7 @@ def main() -> None:
verbose = args.verbose
with open(args.file, encoding="utf-8") as f:
source = f.read()
- modname = args.module or os.path.basename(args.file).removesuffix(".py")
+ modname = args.module or removesuffix(os.path.basename(args.file), ".py")
output = args.output or modname + ".c"
with open(output, "w", encoding="utf-8") as file:
with report_time("generate"):
diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py
index 61ccae61e4f96..4281b67e27cb3 100644
--- a/Tools/scripts/freeze_modules.py
+++ b/Tools/scripts/freeze_modules.py
@@ -25,31 +25,6 @@
# need to be updated.
MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
-if sys.platform != "win32":
- TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module')
- if not os.path.isfile(TOOL):
- # When building out of the source tree, get the tool from directory
- # of the Python executable
- TOOL = os.path.dirname(sys.executable)
- TOOL = os.path.join(TOOL, 'Programs', '_freeze_module')
- TOOL = os.path.abspath(TOOL)
- if not os.path.isfile(TOOL):
- sys.exit("ERROR: missing _freeze_module")
-else:
- def find_tool():
- archs = ['amd64', 'win32']
- if platform.machine() == "ARM64":
- archs.append('arm64')
- for arch in archs:
- for exe in ['_freeze_module.exe', '_freeze_module_d.exe']:
- tool = os.path.join(ROOT_DIR, 'PCbuild', arch, exe)
- if os.path.isfile(tool):
- return tool
- sys.exit("ERROR: missing _freeze_module.exe; you need to run PCbuild/build.bat")
- TOOL = find_tool()
- del find_tool
-
-MANIFEST = os.path.join(MODULES_DIR, 'MANIFEST')
FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj')
@@ -480,45 +455,6 @@ def replace_block(lines, start_marker, end_marker, replacements, file):
return lines[:start_pos + 1] + replacements + lines[end_pos:]
-def regen_manifest(modules):
- header = 'module ispkg source frozen checksum'.split()
- widths = [5] * len(header)
- rows = []
- for mod in modules:
- info = mod.summarize()
- row = []
- for i, col in enumerate(header):
- value = info[col]
- if col == 'checksum':
- value = value[:12]
- elif col == 'ispkg':
- value = 'YES' if value else 'no'
- widths[i] = max(widths[i], len(value))
- row.append(value or '-')
- rows.append(row)
-
- modlines = [
- '# The list of frozen modules with key information.',
- '# Note that the "check_generated_files" CI job will identify',
- '# when source files were changed but regen-frozen wasn\'t run.',
- '# This file is auto-generated by Tools/scripts/freeze_modules.py.',
- ' '.join(c.center(w) for c, w in zip(header, widths)).rstrip(),
- ' '.join('-' * w for w in widths),
- ]
- for row in rows:
- for i, w in enumerate(widths):
- if header[i] == 'ispkg':
- row[i] = row[i].center(w)
- else:
- row[i] = row[i].ljust(w)
- modlines.append(' '.join(row).rstrip())
-
- print(f'# Updating {os.path.relpath(MANIFEST)}')
- with open(MANIFEST, 'w', encoding="utf-8") as outfile:
- lines = (l + '\n' for l in modlines)
- outfile.writelines(lines)
-
-
def regen_frozen(modules):
headerlines = []
parentdir = os.path.dirname(FROZEN_FILE)
@@ -648,11 +584,11 @@ def regen_makefile(modules):
deepfreezefiles.append(f"\t\t{ofile} \\")
# Also add a deepfreeze rule.
- deepfreezerules.append(f'{cfile}: $(srcdir)/{_pyfile} $(DEEPFREEZE_DEPS)')
- deepfreezerules.append(f'\t@echo "Deepfreezing {cfile} from {_pyfile}"')
- deepfreezerules.append(f"\t@./$(BOOTSTRAP) \\")
- deepfreezerules.append(f"\t\t$(srcdir)/Tools/scripts/deepfreeze.py \\")
- deepfreezerules.append(f"\t\t$(srcdir)/{_pyfile} -m {src.frozenid} -o {cfile}")
+ deepfreezerules.append(f'{cfile}: {header} $(DEEPFREEZE_DEPS)')
+ deepfreezerules.append(
+ f"\t$(PYTHON_FOR_REGEN) "
+ f"$(srcdir)/Tools/scripts/deepfreeze.py "
+ f"{header} -m {src.frozenid} -o {cfile}")
deepfreezerules.append('')
for src in _iter_sources(modules):
@@ -663,7 +599,7 @@ def regen_makefile(modules):
pyfiles.append(f'\t\t{pyfile} \\')
freeze = (f'Programs/_freeze_module {src.frozenid} '
- f'$(srcdir)/{pyfile} $(srcdir)/{header}')
+ f'$(srcdir)/{pyfile} {header}')
rules.extend([
f'{header}: Programs/_freeze_module {pyfile}',
f'\t{freeze}',
@@ -774,32 +710,6 @@ def regen_pcbuild(modules):
outfile.writelines(lines)
-#######################################
-# freezing modules
-
-def freeze_module(modname, pyfile=None, destdir=MODULES_DIR):
- """Generate the frozen module .h file for the given module."""
- tmpsuffix = f'.{int(time.time())}'
- for modname, pyfile, ispkg in resolve_modules(modname, pyfile):
- frozenfile = resolve_frozen_file(modname, destdir)
- _freeze_module(modname, pyfile, frozenfile, tmpsuffix)
-
-
-def _freeze_module(frozenid, pyfile, frozenfile, tmpsuffix):
- tmpfile = f'{frozenfile}.{int(time.time())}'
-
- argv = [TOOL, frozenid, pyfile, tmpfile]
- print('#', ' '.join(os.path.relpath(a) for a in argv), flush=True)
- try:
- subprocess.run(argv, check=True)
- except (FileNotFoundError, subprocess.CalledProcessError):
- if not os.path.exists(TOOL):
- sys.exit(f'ERROR: missing {TOOL}; you need to run "make regen-frozen"')
- raise # re-raise
-
- update_file_with_tmpfile(frozenfile, tmpfile, create=True)
-
-
#######################################
# the script
@@ -810,15 +720,7 @@ def main():
# Regen build-related files.
regen_makefile(modules)
regen_pcbuild(modules)
-
- # Freeze the target modules.
- tmpsuffix = f'.{int(time.time())}'
- for src in _iter_sources(modules):
- _freeze_module(src.frozenid, src.pyfile, src.frozenfile, tmpsuffix)
-
- # Regen files dependent of frozen file details.
regen_frozen(modules)
- regen_manifest(modules)
if __name__ == '__main__':
diff --git a/Tools/scripts/umarshal.py b/Tools/scripts/umarshal.py
index e0d18c89e7531..2eaaa7ce2d95b 100644
--- a/Tools/scripts/umarshal.py
+++ b/Tools/scripts/umarshal.py
@@ -2,7 +2,7 @@
import ast
-from typing import Any
+from typing import Any, Tuple
class Type:
@@ -55,10 +55,10 @@ def __init__(self, **kwds: Any):
def __repr__(self) -> str:
return f"Code(**{self.__dict__})"
- co_localsplusnames: tuple[str]
- co_localspluskinds: tuple[int]
+ co_localsplusnames: Tuple[str]
+ co_localspluskinds: Tuple[int]
- def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]:
+ def get_localsplus_names(self, select_kind: int) -> Tuple[str, ...]:
varnames: list[str] = []
for name, kind in zip(self.co_localsplusnames,
self.co_localspluskinds):
@@ -67,15 +67,15 @@ def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]:
return tuple(varnames)
@property
- def co_varnames(self) -> tuple[str, ...]:
+ def co_varnames(self) -> Tuple[str, ...]:
return self.get_localsplus_names(CO_FAST_LOCAL)
@property
- def co_cellvars(self) -> tuple[str, ...]:
+ def co_cellvars(self) -> Tuple[str, ...]:
return self.get_localsplus_names(CO_FAST_CELL)
@property
- def co_freevars(self) -> tuple[str, ...]:
+ def co_freevars(self) -> Tuple[str, ...]:
return self.get_localsplus_names(CO_FAST_FREE)
@property
@@ -190,118 +190,117 @@ def R_REF(obj: Any) -> Any:
obj = self.r_ref(obj, flag)
return obj
- match type:
- case Type.NULL:
- return NULL
- case Type.NONE:
- return None
- case Type.ELLIPSIS:
- return Ellipsis
- case Type.FALSE:
- return False
- case Type.TRUE:
- return True
- case Type.INT:
- return R_REF(self.r_long())
- case Type.INT64:
- return R_REF(self.r_long64())
- case Type.LONG:
- return R_REF(self.r_PyLong())
- case Type.FLOAT:
- return R_REF(self.r_float_str())
- case Type.BINARY_FLOAT:
- return R_REF(self.r_float_bin())
- case Type.COMPLEX:
- return R_REF(complex(self.r_float_str(),
- self.r_float_str()))
- case Type.BINARY_COMPLEX:
- return R_REF(complex(self.r_float_bin(),
- self.r_float_bin()))
- case Type.STRING:
- n = self.r_long()
- return R_REF(self.r_string(n))
- case Type.ASCII_INTERNED | Type.ASCII:
- n = self.r_long()
- return R_REF(self.r_string(n).decode("ascii"))
- case Type.SHORT_ASCII_INTERNED | Type.SHORT_ASCII:
- n = self.r_byte()
- return R_REF(self.r_string(n).decode("ascii"))
- case Type.INTERNED | Type.UNICODE:
- n = self.r_long()
- return R_REF(self.r_string(n).decode("utf8", "surrogatepass"))
- case Type.SMALL_TUPLE:
- n = self.r_byte()
- idx = self.r_ref_reserve(flag)
- retval: Any = tuple(self.r_object() for _ in range(n))
- self.r_ref_insert(retval, idx, flag)
- return retval
- case Type.TUPLE:
- n = self.r_long()
- idx = self.r_ref_reserve(flag)
- retval = tuple(self.r_object() for _ in range(n))
- self.r_ref_insert(retval, idx, flag)
- return retval
- case Type.LIST:
- n = self.r_long()
- retval = R_REF([])
- for _ in range(n):
- retval.append(self.r_object())
- return retval
- case Type.DICT:
- retval = R_REF({})
- while True:
- key = self.r_object()
- if key == NULL:
- break
- val = self.r_object()
- retval[key] = val
- return retval
- case Type.SET:
- n = self.r_long()
- retval = R_REF(set())
- for _ in range(n):
- v = self.r_object()
- retval.add(v)
- return retval
- case Type.FROZENSET:
- n = self.r_long()
- s: set[Any] = set()
- idx = self.r_ref_reserve(flag)
- for _ in range(n):
- v = self.r_object()
- s.add(v)
- retval = frozenset(s)
- self.r_ref_insert(retval, idx, flag)
- return retval
- case Type.CODE:
- retval = R_REF(Code())
- retval.co_argcount = self.r_long()
- retval.co_posonlyargcount = self.r_long()
- retval.co_kwonlyargcount = self.r_long()
- retval.co_stacksize = self.r_long()
- retval.co_flags = self.r_long()
- retval.co_code = self.r_object()
- retval.co_consts = self.r_object()
- retval.co_names = self.r_object()
- retval.co_localsplusnames = self.r_object()
- retval.co_localspluskinds = self.r_object()
- retval.co_filename = self.r_object()
- retval.co_name = self.r_object()
- retval.co_qualname = self.r_object()
- retval.co_firstlineno = self.r_long()
- retval.co_linetable = self.r_object()
- retval.co_endlinetable = self.r_object()
- retval.co_columntable = self.r_object()
- retval.co_exceptiontable = self.r_object()
- return retval
- case Type.REF:
- n = self.r_long()
- retval = self.refs[n]
- assert retval is not None
- return retval
- case _:
- breakpoint()
- raise AssertionError(f"Unknown type {type} {chr(type)!r}")
+ if type == Type.NULL:
+ return NULL
+ elif type == Type.NONE:
+ return None
+ elif type == Type.ELLIPSIS:
+ return Ellipsis
+ elif type == Type.FALSE:
+ return False
+ elif type == Type.TRUE:
+ return True
+ elif type == Type.INT:
+ return R_REF(self.r_long())
+ elif type == Type.INT64:
+ return R_REF(self.r_long64())
+ elif type == Type.LONG:
+ return R_REF(self.r_PyLong())
+ elif type == Type.FLOAT:
+ return R_REF(self.r_float_str())
+ elif type == Type.BINARY_FLOAT:
+ return R_REF(self.r_float_bin())
+ elif type == Type.COMPLEX:
+ return R_REF(complex(self.r_float_str(),
+ self.r_float_str()))
+ elif type == Type.BINARY_COMPLEX:
+ return R_REF(complex(self.r_float_bin(),
+ self.r_float_bin()))
+ elif type == Type.STRING:
+ n = self.r_long()
+ return R_REF(self.r_string(n))
+ elif type == Type.ASCII_INTERNED or type == Type.ASCII:
+ n = self.r_long()
+ return R_REF(self.r_string(n).decode("ascii"))
+ elif type == Type.SHORT_ASCII_INTERNED or type == Type.SHORT_ASCII:
+ n = self.r_byte()
+ return R_REF(self.r_string(n).decode("ascii"))
+ elif type == Type.INTERNED or type == Type.UNICODE:
+ n = self.r_long()
+ return R_REF(self.r_string(n).decode("utf8", "surrogatepass"))
+ elif type == Type.SMALL_TUPLE:
+ n = self.r_byte()
+ idx = self.r_ref_reserve(flag)
+ retval: Any = tuple(self.r_object() for _ in range(n))
+ self.r_ref_insert(retval, idx, flag)
+ return retval
+ elif type == Type.TUPLE:
+ n = self.r_long()
+ idx = self.r_ref_reserve(flag)
+ retval = tuple(self.r_object() for _ in range(n))
+ self.r_ref_insert(retval, idx, flag)
+ return retval
+ elif type == Type.LIST:
+ n = self.r_long()
+ retval = R_REF([])
+ for _ in range(n):
+ retval.append(self.r_object())
+ return retval
+ elif type == Type.DICT:
+ retval = R_REF({})
+ while True:
+ key = self.r_object()
+ if key == NULL:
+ break
+ val = self.r_object()
+ retval[key] = val
+ return retval
+ elif type == Type.SET:
+ n = self.r_long()
+ retval = R_REF(set())
+ for _ in range(n):
+ v = self.r_object()
+ retval.add(v)
+ return retval
+ elif type == Type.FROZENSET:
+ n = self.r_long()
+ s: set[Any] = set()
+ idx = self.r_ref_reserve(flag)
+ for _ in range(n):
+ v = self.r_object()
+ s.add(v)
+ retval = frozenset(s)
+ self.r_ref_insert(retval, idx, flag)
+ return retval
+ elif type == Type.CODE:
+ retval = R_REF(Code())
+ retval.co_argcount = self.r_long()
+ retval.co_posonlyargcount = self.r_long()
+ retval.co_kwonlyargcount = self.r_long()
+ retval.co_stacksize = self.r_long()
+ retval.co_flags = self.r_long()
+ retval.co_code = self.r_object()
+ retval.co_consts = self.r_object()
+ retval.co_names = self.r_object()
+ retval.co_localsplusnames = self.r_object()
+ retval.co_localspluskinds = self.r_object()
+ retval.co_filename = self.r_object()
+ retval.co_name = self.r_object()
+ retval.co_qualname = self.r_object()
+ retval.co_firstlineno = self.r_long()
+ retval.co_linetable = self.r_object()
+ retval.co_endlinetable = self.r_object()
+ retval.co_columntable = self.r_object()
+ retval.co_exceptiontable = self.r_object()
+ return retval
+ elif type == Type.REF:
+ n = self.r_long()
+ retval = self.refs[n]
+ assert retval is not None
+ return retval
+ else:
+ breakpoint()
+ raise AssertionError(f"Unknown type {type} {chr(type)!r}")
def loads(data: bytes) -> Any:
diff --git a/configure b/configure
index 148ef58b6a5b8..0684921f41a42 100755
--- a/configure
+++ b/configure
@@ -19441,6 +19441,7 @@ SRCDIRS="\
Parser \
Programs \
Python \
+ Python/frozen_modules \
Python/deepfreeze"
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for build directories" >&5
$as_echo_n "checking for build directories... " >&6; }
diff --git a/configure.ac b/configure.ac
index a950cc91d657b..ba22418b40694 100644
--- a/configure.ac
+++ b/configure.ac
@@ -5613,6 +5613,7 @@ SRCDIRS="\
Parser \
Programs \
Python \
+ Python/frozen_modules \
Python/deepfreeze"
AC_MSG_CHECKING(for build directories)
for dir in $SRCDIRS; do
1
0
![](https://secure.gravatar.com/avatar/cc7737cd64a84f1b5c61a160798e97ee.jpg?s=120&d=mm&r=g)
bpo-45703: Invalidate _NamespacePath cache on importlib.invalidate_ca… (GH-29384)
by miss-islington 23 Nov '21
by miss-islington 23 Nov '21
23 Nov '21
https://github.com/python/cpython/commit/ae1965ccb4b1fad63fab40fe8805d1b824…
commit: ae1965ccb4b1fad63fab40fe8805d1b8247668d3
branch: main
author: Miro Hrončok <miro(a)hroncok.cz>
committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com>
date: 2021-11-23T07:38:02-08:00
summary:
bpo-45703: Invalidate _NamespacePath cache on importlib.invalidate_ca… (GH-29384)
Consider the following directory structure:
.
└── PATH1
└── namespace
└── sub1
└── __init__.py
And both PATH1 and PATH2 in sys path:
$ PYTHONPATH=PATH1:PATH2 python3.11
>>> import namespace
>>> import namespace.sub1
>>> namespace.__path__
_NamespacePath(['.../PATH1/namespace'])
>>> ...
While this interpreter still runs, PATH2/namespace/sub2 is created:
.
├── PATH1
│ └── namespace
│ └── sub1
│ └── __init__.py
└── PATH2
└── namespace
└── sub2
└── __init__.py
The newly created module cannot be imported:
>>> ...
>>> namespace.__path__
_NamespacePath(['.../PATH1/namespace'])
>>> import namespace.sub2
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ModuleNotFoundError: No module named 'namespace.sub2'
Calling importlib.invalidate_caches() now newly allows to import it:
>>> import importlib
>>> importlib.invalidate_caches()
>>> namespace.__path__
_NamespacePath(['.../PATH1/namespace'])
>>> import namespace.sub2
>>> namespace.__path__
_NamespacePath(['.../PATH1/namespace', '.../PATH2/namespace'])
This was not previously possible.
files:
A Misc/NEWS.d/next/Library/2021-11-03-13-41-49.bpo-45703.35AagL.rst
M Doc/library/importlib.rst
M Lib/importlib/_bootstrap_external.py
M Lib/test/test_importlib/test_namespace_pkgs.py
diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
index 6b71e64bcbfa7..59c8c64acdc3b 100644
--- a/Doc/library/importlib.rst
+++ b/Doc/library/importlib.rst
@@ -145,6 +145,10 @@ Functions
.. versionadded:: 3.3
+ .. versionchanged:: 3.10
+ Namespace packages created/installed in a different :data:`sys.path`
+ location after the same namespace was already imported are noticed.
+
.. function:: reload(module)
Reload a previously imported *module*. The argument must be a module object,
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index 85c5193407d80..6970e9f0a94d4 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -1231,10 +1231,15 @@ class _NamespacePath:
using path_finder. For top-level modules, the parent module's path
is sys.path."""
+ # When invalidate_caches() is called, this epoch is incremented
+ # https://bugs.python.org/issue45703
+ _epoch = 0
+
def __init__(self, name, path, path_finder):
self._name = name
self._path = path
self._last_parent_path = tuple(self._get_parent_path())
+ self._last_epoch = self._epoch
self._path_finder = path_finder
def _find_parent_path_names(self):
@@ -1254,7 +1259,7 @@ def _get_parent_path(self):
def _recalculate(self):
# If the parent's path has changed, recalculate _path
parent_path = tuple(self._get_parent_path()) # Make a copy
- if parent_path != self._last_parent_path:
+ if parent_path != self._last_parent_path or self._epoch != self._last_epoch:
spec = self._path_finder(self._name, parent_path)
# Note that no changes are made if a loader is returned, but we
# do remember the new parent path
@@ -1262,6 +1267,7 @@ def _recalculate(self):
if spec.submodule_search_locations:
self._path = spec.submodule_search_locations
self._last_parent_path = parent_path # Save the copy
+ self._last_epoch = self._epoch
return self._path
def __iter__(self):
@@ -1355,6 +1361,9 @@ def invalidate_caches():
del sys.path_importer_cache[name]
elif hasattr(finder, 'invalidate_caches'):
finder.invalidate_caches()
+ # Also invalidate the caches of _NamespacePaths
+ # https://bugs.python.org/issue45703
+ _NamespacePath._epoch += 1
@staticmethod
def _path_hooks(path):
diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py
index f80283233f977..2ea41b7a4c5c3 100644
--- a/Lib/test/test_importlib/test_namespace_pkgs.py
+++ b/Lib/test/test_importlib/test_namespace_pkgs.py
@@ -4,6 +4,7 @@
import importlib.machinery
import os
import sys
+import tempfile
import unittest
import warnings
@@ -130,6 +131,40 @@ def test_imports(self):
self.assertEqual(foo.two.attr, 'portion2 foo two')
+class SeparatedNamespacePackagesCreatedWhileRunning(NamespacePackageTest):
+ paths = ['portion1']
+
+ def test_invalidate_caches(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # we manipulate sys.path before anything is imported to avoid
+ # accidental cache invalidation when changing it
+ sys.path.append(temp_dir)
+
+ import foo.one
+ self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+ # the module does not exist, so it cannot be imported
+ with self.assertRaises(ImportError):
+ import foo.just_created
+
+ # util.create_modules() manipulates sys.path
+ # so we must create the modules manually instead
+ namespace_path = os.path.join(temp_dir, 'foo')
+ os.mkdir(namespace_path)
+ module_path = os.path.join(namespace_path, 'just_created.py')
+ with open(module_path, 'w', encoding='utf-8') as file:
+ file.write('attr = "just_created foo"')
+
+ # the module is not known, so it cannot be imported yet
+ with self.assertRaises(ImportError):
+ import foo.just_created
+
+ # but after explicit cache invalidation, it is importable
+ importlib.invalidate_caches()
+ import foo.just_created
+ self.assertEqual(foo.just_created.attr, 'just_created foo')
+
+
class SeparatedOverlappingNamespacePackages(NamespacePackageTest):
paths = ['portion1', 'both_portions']
diff --git a/Misc/NEWS.d/next/Library/2021-11-03-13-41-49.bpo-45703.35AagL.rst b/Misc/NEWS.d/next/Library/2021-11-03-13-41-49.bpo-45703.35AagL.rst
new file mode 100644
index 0000000000000..9fa9be56b8327
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-11-03-13-41-49.bpo-45703.35AagL.rst
@@ -0,0 +1,5 @@
+When a namespace package is imported before another module from the same
+namespace is created/installed in a different :data:`sys.path` location
+while the program is running, calling the
+:func:`importlib.invalidate_caches` function will now also guarantee the new
+module is noticed.
1
0
![](https://secure.gravatar.com/avatar/cc7737cd64a84f1b5c61a160798e97ee.jpg?s=120&d=mm&r=g)
bpo-45783: Preserve file moves and deletions in the tests for the freeze tool. (GH-29527)
by encukou 23 Nov '21
by encukou 23 Nov '21
23 Nov '21
https://github.com/python/cpython/commit/8ed1495ad900dd815ff8fb97926da5312a…
commit: 8ed1495ad900dd815ff8fb97926da5312aaa23f9
branch: main
author: Eric Snow <ericsnowcurrently(a)gmail.com>
committer: encukou <encukou(a)gmail.com>
date: 2021-11-23T14:43:40+01:00
summary:
bpo-45783: Preserve file moves and deletions in the tests for the freeze tool. (GH-29527)
Use shutil.copytree rather than Git, which might be missing (or configured
differently) when testing Python built from a source release.
files:
A Misc/NEWS.d/next/Tests/2021-11-11-13-56-00.bpo-45783.8k1Rng.rst
M Lib/test/test_tools/test_freeze.py
M Tools/freeze/test/freeze.py
diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py
index 392a776f042e4..386c35a973bc2 100644
--- a/Lib/test/test_tools/test_freeze.py
+++ b/Lib/test/test_tools/test_freeze.py
@@ -5,6 +5,7 @@
import unittest
from test import support
+from test.support import os_helper
from . import imports_under_tool, skip_if_missing
skip_if_missing('freeze')
@@ -22,8 +23,8 @@ def test_freeze_simple_script(self):
print('running...')
sys.exit(0)
""")
- outdir, scriptfile, python = helper.prepare(script)
-
- executable = helper.freeze(python, scriptfile, outdir)
- text = helper.run(executable)
+ with os_helper.temp_dir() as outdir:
+ outdir, scriptfile, python = helper.prepare(script, outdir)
+ executable = helper.freeze(python, scriptfile, outdir)
+ text = helper.run(executable)
self.assertEqual(text, 'running...')
diff --git a/Misc/NEWS.d/next/Tests/2021-11-11-13-56-00.bpo-45783.8k1Rng.rst b/Misc/NEWS.d/next/Tests/2021-11-11-13-56-00.bpo-45783.8k1Rng.rst
new file mode 100644
index 0000000000000..cb0e07f27fb55
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2021-11-11-13-56-00.bpo-45783.8k1Rng.rst
@@ -0,0 +1 @@
+The test for the freeze tool now handles file moves and deletions.
diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py
index 387f1ff70b234..db2aa3101d879 100644
--- a/Tools/freeze/test/freeze.py
+++ b/Tools/freeze/test/freeze.py
@@ -11,7 +11,6 @@
SRCDIR = os.path.dirname(os.path.dirname(TOOL_ROOT))
MAKE = shutil.which('make')
-GIT = shutil.which('git')
FREEZE = os.path.join(TOOL_ROOT, 'freeze.py')
OUTDIR = os.path.join(TESTS_DIR, 'outdir')
@@ -75,36 +74,15 @@ def ensure_opt(args, name, value):
args[pos] = f'{opt}={value}'
-def git_copy_repo(newroot, oldroot):
- if not GIT:
- raise UnsupportedError('git')
-
+def copy_source_tree(newroot, oldroot):
+ print(f'copying the source tree into {newroot}...')
if os.path.exists(newroot):
- print(f'updating copied repo {newroot}...')
if newroot == SRCDIR:
raise Exception('this probably isn\'t what you wanted')
- _run_quiet([GIT, 'clean', '-d', '-f'], newroot)
- _run_quiet([GIT, 'reset'], newroot)
- _run_quiet([GIT, 'checkout', '.'], newroot)
- _run_quiet([GIT, 'pull', '-f', oldroot], newroot)
- else:
- print(f'copying repo into {newroot}...')
- _run_quiet([GIT, 'clone', oldroot, newroot])
-
- # Copy over any uncommited files.
- text = _run_stdout([GIT, 'status', '-s'], oldroot)
- for line in text.splitlines():
- _, _, relfile = line.strip().partition(' ')
- relfile = relfile.strip()
- isdir = relfile.endswith(os.path.sep)
- relfile = relfile.rstrip(os.path.sep)
- srcfile = os.path.join(oldroot, relfile)
- dstfile = os.path.join(newroot, relfile)
- os.makedirs(os.path.dirname(dstfile), exist_ok=True)
- if isdir:
- shutil.copytree(srcfile, dstfile, dirs_exist_ok=True)
- else:
- shutil.copy2(srcfile, dstfile)
+ shutil.rmtree(newroot)
+ shutil.copytree(oldroot, newroot)
+ if os.path.exists(os.path.join(newroot, 'Makefile')):
+ _run_quiet([MAKE, 'clean'], newroot)
def get_makefile_var(builddir, name):
@@ -146,12 +124,14 @@ def prepare(script=None, outdir=None):
# Write the script to disk.
if script:
scriptfile = os.path.join(outdir, 'app.py')
+ print(f'creating the script to be frozen at {scriptfile}')
with open(scriptfile, 'w') as outfile:
outfile.write(script)
- # Make a copy of the repo to avoid affecting the current build.
+ # Make a copy of the repo to avoid affecting the current build
+ # (e.g. changing PREFIX).
srcdir = os.path.join(outdir, 'cpython')
- git_copy_repo(srcdir, SRCDIR)
+ copy_source_tree(srcdir, SRCDIR)
# We use an out-of-tree build (instead of srcdir).
builddir = os.path.join(outdir, 'python-build')
@@ -172,7 +152,7 @@ def prepare(script=None, outdir=None):
raise UnsupportedError('make')
# Build python.
- print('building python...')
+ print(f'building python in {builddir}...')
if os.path.exists(os.path.join(srcdir, 'Makefile')):
# Out-of-tree builds require a clean srcdir.
_run_quiet([MAKE, '-C', srcdir, 'clean'])
1
0
![](https://secure.gravatar.com/avatar/cc7737cd64a84f1b5c61a160798e97ee.jpg?s=120&d=mm&r=g)
bpo-45878: convert `try/except` to `self.assertRaises` in `Lib/ctypes/test/test_functions.py` (GH-29721)
by serhiy-storchaka 23 Nov '21
by serhiy-storchaka 23 Nov '21
23 Nov '21
https://github.com/python/cpython/commit/b48ac6fe38b2fca9963b097c04cdecfc60…
commit: b48ac6fe38b2fca9963b097c04cdecfc6083104e
branch: main
author: Nikita Sobolev <mail(a)sobolevn.me>
committer: serhiy-storchaka <storchaka(a)gmail.com>
date: 2021-11-23T13:12:13+02:00
summary:
bpo-45878: convert `try/except` to `self.assertRaises` in `Lib/ctypes/test/test_functions.py` (GH-29721)
files:
A Misc/NEWS.d/next/Tests/2021-11-23-12-36-21.bpo-45878.eOs_Mp.rst
M Lib/ctypes/test/test_functions.py
diff --git a/Lib/ctypes/test/test_functions.py b/Lib/ctypes/test/test_functions.py
index c5456ba107ae3..f9e92e1cc6b06 100644
--- a/Lib/ctypes/test/test_functions.py
+++ b/Lib/ctypes/test/test_functions.py
@@ -35,34 +35,24 @@ def test_mro(self):
# wasn't checked, and it even crashed Python.
# Found by Greg Chapman.
- try:
+ with self.assertRaises(TypeError):
class X(object, Array):
_length_ = 5
_type_ = "i"
- except TypeError:
- pass
-
from _ctypes import _Pointer
- try:
+ with self.assertRaises(TypeError):
class X(object, _Pointer):
pass
- except TypeError:
- pass
from _ctypes import _SimpleCData
- try:
+ with self.assertRaises(TypeError):
class X(object, _SimpleCData):
_type_ = "i"
- except TypeError:
- pass
- try:
+ with self.assertRaises(TypeError):
class X(object, Structure):
_fields_ = []
- except TypeError:
- pass
-
@need_symbol('c_wchar')
def test_wchar_parm(self):
diff --git a/Misc/NEWS.d/next/Tests/2021-11-23-12-36-21.bpo-45878.eOs_Mp.rst b/Misc/NEWS.d/next/Tests/2021-11-23-12-36-21.bpo-45878.eOs_Mp.rst
new file mode 100644
index 0000000000000..fca90ef46981b
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2021-11-23-12-36-21.bpo-45878.eOs_Mp.rst
@@ -0,0 +1,2 @@
+Test ``Lib/ctypes/test/test_functions.py::test_mro`` now uses
+``self.assertRaises`` instead of ``try/except``.
1
0
https://github.com/python/cpython/commit/327c764fa2a032e34dc268164c3c9c2f95…
commit: 327c764fa2a032e34dc268164c3c9c2f95b978c4
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com>
committer: miss-islington <31488909+miss-islington(a)users.noreply.github.com>
date: 2021-11-23T02:32:31-08:00
summary:
bpo-45561: Run smelly.py tool from $(srcdir) (GH-29138)
(cherry picked from commit 457e6a6e96b5afad403a0bc892508a77beef4d33)
Co-authored-by: Neil Schemenauer <nas-github(a)arctrix.com>
files:
A Misc/NEWS.d/next/Build/2021-10-21-14-38-30.bpo-45561.PVqhZE.rst
M Makefile.pre.in
M Tools/scripts/smelly.py
diff --git a/Makefile.pre.in b/Makefile.pre.in
index ae6555c8e2fe1..86014227c3387 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1962,7 +1962,7 @@ distclean: clobber
# Check that all symbols exported by libpython start with "Py" or "_Py"
smelly: @DEF_MAKE_RULE@
- $(RUNSHARED) ./$(BUILDPYTHON) Tools/scripts/smelly.py
+ $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/smelly.py
# Find files with funny names
funny:
diff --git a/Misc/NEWS.d/next/Build/2021-10-21-14-38-30.bpo-45561.PVqhZE.rst b/Misc/NEWS.d/next/Build/2021-10-21-14-38-30.bpo-45561.PVqhZE.rst
new file mode 100644
index 0000000000000..cf5d8686b9840
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2021-10-21-14-38-30.bpo-45561.PVqhZE.rst
@@ -0,0 +1 @@
+Run smelly.py tool from $(srcdir).
diff --git a/Tools/scripts/smelly.py b/Tools/scripts/smelly.py
index fb01660dea33a..276a5ab2cc84c 100755
--- a/Tools/scripts/smelly.py
+++ b/Tools/scripts/smelly.py
@@ -108,8 +108,11 @@ def check_library(library, dynamic=False):
def check_extensions():
print(__file__)
- srcdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
- filename = os.path.join(srcdir, "pybuilddir.txt")
+ # This assumes pybuilddir.txt is in same directory as pyconfig.h.
+ # In the case of out-of-tree builds, we can't assume pybuilddir.txt is
+ # in the source folder.
+ config_dir = os.path.dirname(sysconfig.get_config_h_filename())
+ filename = os.path.join(config_dir, "pybuilddir.txt")
try:
with open(filename, encoding="utf-8") as fp:
pybuilddir = fp.readline()
@@ -118,7 +121,7 @@ def check_extensions():
return True
print(f"Check extension modules from {pybuilddir} directory")
- builddir = os.path.join(srcdir, pybuilddir)
+ builddir = os.path.join(config_dir, pybuilddir)
nsymbol = 0
for name in os.listdir(builddir):
if not name.endswith(".so"):
1
0
![](https://secure.gravatar.com/avatar/cc7737cd64a84f1b5c61a160798e97ee.jpg?s=120&d=mm&r=g)
23 Nov '21
https://github.com/python/cpython/commit/2c047f604e6a999390b5fa4ebfb6e8222c…
commit: 2c047f604e6a999390b5fa4ebfb6e8222c5cc096
branch: main
author: Petr Viktorin <encukou(a)gmail.com>
committer: encukou <encukou(a)gmail.com>
date: 2021-11-23T11:28:14+01:00
summary:
Clarify that discutils.(plat)include is for CPython's headers (GH-29578)
Change the docs to note that "include" and "platinclude" are
for CPython's headers, and not necessarily for headers of
third-party libraries.
See discussion in: https://discuss.python.org/t/clarification-on-a-wheels-header-data/9305/19
files:
M Doc/library/sysconfig.rst
diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst
index 7ef3b2489673e..713be1e02cea6 100644
--- a/Doc/library/sysconfig.rst
+++ b/Doc/library/sysconfig.rst
@@ -96,8 +96,10 @@ identifier. Python currently uses eight paths:
platform-specific.
- *platlib*: directory for site-specific, platform-specific files.
- *purelib*: directory for site-specific, non-platform-specific files.
-- *include*: directory for non-platform-specific header files.
-- *platinclude*: directory for platform-specific header files.
+- *include*: directory for non-platform-specific header files for
+ the Python C-API.
+- *platinclude*: directory for platform-specific header files for
+ the Python C-API.
- *scripts*: directory for script files.
- *data*: directory for data files.
1
0