[Python-checkins] [3.10] bpo-34602: Quadruple stack size on macOS when compiling with UBSAN (GH-27309) (GH-28280)
ambv
webhook-mailer at python.org
Wed Sep 15 14:35:02 EDT 2021
https://github.com/python/cpython/commit/2563dd2d0a1cf793afca328ae9e195b72bd2b391
commit: 2563dd2d0a1cf793afca328ae9e195b72bd2b391
branch: 3.10
author: Łukasz Langa <lukasz at langa.pl>
committer: ambv <lukasz at langa.pl>
date: 2021-09-15T20:34:57+02:00
summary:
[3.10] bpo-34602: Quadruple stack size on macOS when compiling with UBSAN (GH-27309) (GH-28280)
(cherry picked from commit be9de8721d63b9d8e032d508069daf88c06542c6)
Co-authored-by: Łukasz Langa <lukasz at langa.pl>
files:
A Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst
M Python/thread_pthread.h
M configure
M configure.ac
M pyconfig.h.in
diff --git a/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst b/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst
new file mode 100644
index 0000000000000..29a6ff92554e1
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst
@@ -0,0 +1,3 @@
+When building CPython on macOS with ``./configure
+--with-undefined-behavior-sanitizer --with-pydebug``, the stack size is now
+quadrupled to allow for the entire test suite to pass.
diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h
index ec7d737518b68..a45d842ffe73d 100644
--- a/Python/thread_pthread.h
+++ b/Python/thread_pthread.h
@@ -32,18 +32,17 @@
#define THREAD_STACK_SIZE 0 /* use default stack size */
#endif
-/* The default stack size for new threads on OSX and BSD is small enough that
+/* The default stack size for new threads on BSD is small enough that
* we'll get hard crashes instead of 'maximum recursion depth exceeded'
* exceptions.
*
- * The default stack sizes below are the empirically determined minimal stack
+ * The default stack size below is the empirically determined minimal stack
* sizes where a simple recursive function doesn't cause a hard crash.
+ *
+ * For macOS the value of THREAD_STACK_SIZE is determined in configure.ac
+ * as it also depends on the other configure options like chosen sanitizer
+ * runtimes.
*/
-#if defined(__APPLE__) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0
-#undef THREAD_STACK_SIZE
-/* Note: This matches the value of -Wl,-stack_size in configure.ac */
-#define THREAD_STACK_SIZE 0x1000000
-#endif
#if defined(__FreeBSD__) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0
#undef THREAD_STACK_SIZE
#define THREAD_STACK_SIZE 0x400000
diff --git a/configure b/configure
index 1baa145e3ed4d..15801175a3dd5 100755
--- a/configure
+++ b/configure
@@ -827,11 +827,11 @@ with_trace_refs
with_assertions
enable_optimizations
with_lto
-with_hash_algorithm
-with_tzpath
with_address_sanitizer
with_memory_sanitizer
with_undefined_behavior_sanitizer
+with_hash_algorithm
+with_tzpath
with_libs
with_system_expat
with_system_ffi
@@ -1547,12 +1547,6 @@ Optional Packages:
--with-assertions build with C assertions enabled (default is no)
--with-lto enable Link-Time-Optimization in any build (default
is no)
- --with-hash-algorithm=[fnv|siphash24]
- select hash algorithm for use in Python/pyhash.c
- (default is SipHash24)
- --with-tzpath=<list of absolute paths separated by pathsep>
- Select the default time zone search path for zoneinfo.TZPATH
-
--with-address-sanitizer
enable AddressSanitizer memory error detector,
'asan' (default is no)
@@ -1561,6 +1555,12 @@ Optional Packages:
--with-undefined-behavior-sanitizer
enable UndefinedBehaviorSanitizer undefined
behaviour detector, 'ubsan' (default is no)
+ --with-hash-algorithm=[fnv|siphash24]
+ select hash algorithm for use in Python/pyhash.c
+ (default is SipHash24)
+ --with-tzpath=<list of absolute paths separated by pathsep>
+ Select the default time zone search path for zoneinfo.TZPATH
+
--with-libs='lib1 ...' link against additional libs (default is no)
--with-system-expat build pyexpat module using an installed expat
library, see Doc/library/pyexpat.rst (default is no)
@@ -9566,6 +9566,65 @@ $as_echo "no" >&6; }
;;
esac
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-address-sanitizer" >&5
+$as_echo_n "checking for --with-address-sanitizer... " >&6; }
+
+# Check whether --with-address_sanitizer was given.
+if test "${with_address_sanitizer+set}" = set; then :
+ withval=$with_address_sanitizer;
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
+$as_echo "$withval" >&6; }
+BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS"
+LDFLAGS="-fsanitize=address $LDFLAGS"
+# ASan works by controlling memory allocation, our own malloc interferes.
+with_pymalloc="no"
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-memory-sanitizer" >&5
+$as_echo_n "checking for --with-memory-sanitizer... " >&6; }
+
+# Check whether --with-memory_sanitizer was given.
+if test "${with_memory_sanitizer+set}" = set; then :
+ withval=$with_memory_sanitizer;
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
+$as_echo "$withval" >&6; }
+BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS"
+LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS"
+# MSan works by controlling memory allocation, our own malloc interferes.
+with_pymalloc="no"
+
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-undefined-behavior-sanitizer" >&5
+$as_echo_n "checking for --with-undefined-behavior-sanitizer... " >&6; }
+
+# Check whether --with-undefined_behavior_sanitizer was given.
+if test "${with_undefined_behavior_sanitizer+set}" = set; then :
+ withval=$with_undefined_behavior_sanitizer;
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
+$as_echo "$withval" >&6; }
+BASECFLAGS="-fsanitize=undefined $BASECFLAGS"
+LDFLAGS="-fsanitize=undefined $LDFLAGS"
+with_ubsan="yes"
+
+else
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+with_ubsan="no"
+
+fi
+
+
# Set info about shared libraries.
@@ -9776,9 +9835,20 @@ then
# Issue #18075: the default maximum stack size (8MBytes) is too
# small for the default recursion limit. Increase the stack size
# to ensure that tests don't crash
- # Note: This matches the value of THREAD_STACK_SIZE in
- # thread_pthread.h
- LINKFORSHARED="-Wl,-stack_size,1000000 $LINKFORSHARED"
+ stack_size="1000000" # 16 MB
+ if test "$with_ubsan" == "yes"
+ then
+ # Undefined behavior sanitizer requires an even deeper stack
+ stack_size="4000000" # 64 MB
+ fi
+
+ LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED"
+
+
+cat >>confdefs.h <<_ACEOF
+#define THREAD_STACK_SIZE 0x$stack_size
+_ACEOF
+
if test "$enable_framework"
then
@@ -10374,61 +10444,6 @@ fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-address-sanitizer" >&5
-$as_echo_n "checking for --with-address-sanitizer... " >&6; }
-
-# Check whether --with-address_sanitizer was given.
-if test "${with_address_sanitizer+set}" = set; then :
- withval=$with_address_sanitizer;
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
-$as_echo "$withval" >&6; }
-BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS"
-LDFLAGS="-fsanitize=address $LDFLAGS"
-# ASan works by controlling memory allocation, our own malloc interferes.
-with_pymalloc="no"
-
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
-fi
-
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-memory-sanitizer" >&5
-$as_echo_n "checking for --with-memory-sanitizer... " >&6; }
-
-# Check whether --with-memory_sanitizer was given.
-if test "${with_memory_sanitizer+set}" = set; then :
- withval=$with_memory_sanitizer;
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
-$as_echo "$withval" >&6; }
-BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS"
-LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS"
-# MSan works by controlling memory allocation, our own malloc interferes.
-with_pymalloc="no"
-
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
-fi
-
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-undefined-behavior-sanitizer" >&5
-$as_echo_n "checking for --with-undefined-behavior-sanitizer... " >&6; }
-
-# Check whether --with-undefined_behavior_sanitizer was given.
-if test "${with_undefined_behavior_sanitizer+set}" = set; then :
- withval=$with_undefined_behavior_sanitizer;
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5
-$as_echo "$withval" >&6; }
-BASECFLAGS="-fsanitize=undefined $BASECFLAGS"
-LDFLAGS="-fsanitize=undefined $LDFLAGS"
-
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
-fi
-
-
# Most SVR4 platforms (e.g. Solaris) need -lsocket and -lnsl.
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for t_open in -lnsl" >&5
$as_echo_n "checking for t_open in -lnsl... " >&6; }
diff --git a/configure.ac b/configure.ac
index 3e6c07c2793ab..a75fdf918c7ea 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2562,6 +2562,47 @@ case $ac_sys_system/$ac_sys_release in
;;
esac
+AC_MSG_CHECKING(for --with-address-sanitizer)
+AC_ARG_WITH(address_sanitizer,
+ AS_HELP_STRING([--with-address-sanitizer],
+ [enable AddressSanitizer memory error detector, 'asan' (default is no)]),
+[
+AC_MSG_RESULT($withval)
+BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS"
+LDFLAGS="-fsanitize=address $LDFLAGS"
+# ASan works by controlling memory allocation, our own malloc interferes.
+with_pymalloc="no"
+],
+[AC_MSG_RESULT(no)])
+
+AC_MSG_CHECKING(for --with-memory-sanitizer)
+AC_ARG_WITH(memory_sanitizer,
+ AS_HELP_STRING([--with-memory-sanitizer],
+ [enable MemorySanitizer allocation error detector, 'msan' (default is no)]),
+[
+AC_MSG_RESULT($withval)
+BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS"
+LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS"
+# MSan works by controlling memory allocation, our own malloc interferes.
+with_pymalloc="no"
+],
+[AC_MSG_RESULT(no)])
+
+AC_MSG_CHECKING(for --with-undefined-behavior-sanitizer)
+AC_ARG_WITH(undefined_behavior_sanitizer,
+ AS_HELP_STRING([--with-undefined-behavior-sanitizer],
+ [enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no)]),
+[
+AC_MSG_RESULT($withval)
+BASECFLAGS="-fsanitize=undefined $BASECFLAGS"
+LDFLAGS="-fsanitize=undefined $LDFLAGS"
+with_ubsan="yes"
+],
+[
+AC_MSG_RESULT(no)
+with_ubsan="no"
+])
+
# Set info about shared libraries.
AC_SUBST(SHLIB_SUFFIX)
AC_SUBST(LDSHARED)
@@ -2765,9 +2806,18 @@ then
# Issue #18075: the default maximum stack size (8MBytes) is too
# small for the default recursion limit. Increase the stack size
# to ensure that tests don't crash
- # Note: This matches the value of THREAD_STACK_SIZE in
- # thread_pthread.h
- LINKFORSHARED="-Wl,-stack_size,1000000 $LINKFORSHARED"
+ stack_size="1000000" # 16 MB
+ if test "$with_ubsan" == "yes"
+ then
+ # Undefined behavior sanitizer requires an even deeper stack
+ stack_size="4000000" # 64 MB
+ fi
+
+ LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED"
+
+ AC_DEFINE_UNQUOTED(THREAD_STACK_SIZE,
+ 0x$stack_size,
+ [Custom thread stack size depending on chosen sanitizer runtimes.])
if test "$enable_framework"
then
@@ -3011,43 +3061,6 @@ esac
AC_MSG_RESULT("$TZPATH")])
AC_SUBST(TZPATH)
-AC_MSG_CHECKING(for --with-address-sanitizer)
-AC_ARG_WITH(address_sanitizer,
- AS_HELP_STRING([--with-address-sanitizer],
- [enable AddressSanitizer memory error detector, 'asan' (default is no)]),
-[
-AC_MSG_RESULT($withval)
-BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS"
-LDFLAGS="-fsanitize=address $LDFLAGS"
-# ASan works by controlling memory allocation, our own malloc interferes.
-with_pymalloc="no"
-],
-[AC_MSG_RESULT(no)])
-
-AC_MSG_CHECKING(for --with-memory-sanitizer)
-AC_ARG_WITH(memory_sanitizer,
- AS_HELP_STRING([--with-memory-sanitizer],
- [enable MemorySanitizer allocation error detector, 'msan' (default is no)]),
-[
-AC_MSG_RESULT($withval)
-BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS"
-LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS"
-# MSan works by controlling memory allocation, our own malloc interferes.
-with_pymalloc="no"
-],
-[AC_MSG_RESULT(no)])
-
-AC_MSG_CHECKING(for --with-undefined-behavior-sanitizer)
-AC_ARG_WITH(undefined_behavior_sanitizer,
- AS_HELP_STRING([--with-undefined-behavior-sanitizer],
- [enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no)]),
-[
-AC_MSG_RESULT($withval)
-BASECFLAGS="-fsanitize=undefined $BASECFLAGS"
-LDFLAGS="-fsanitize=undefined $LDFLAGS"
-],
-[AC_MSG_RESULT(no)])
-
# Most SVR4 platforms (e.g. Solaris) need -lsocket and -lnsl.
AC_CHECK_LIB(nsl, t_open, [LIBS="-lnsl $LIBS"]) # SVR4
AC_CHECK_LIB(socket, socket, [LIBS="-lsocket $LIBS"], [], $LIBS) # SVR4 sockets
diff --git a/pyconfig.h.in b/pyconfig.h.in
index 63438d857a070..49407ab62b417 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -1509,6 +1509,9 @@
(which you can't on SCO ODT 3.0). */
#undef SYS_SELECT_WITH_SYS_TIME
+/* Custom thread stack size depending on chosen sanitizer runtimes. */
+#undef THREAD_STACK_SIZE
+
/* Library needed by timemodule.c: librt may be needed for clock_gettime() */
#undef TIMEMODULE_LIB
More information about the Python-checkins
mailing list