https://github.com/python/cpython/commit/2041a95e68ebf6d13f867e214ada28affa8... commit: 2041a95e68ebf6d13f867e214ada28affa830669 branch: main author: Russell Keith-Magee <russell@keith-magee.com> committer: freakboy3742 <russell@keith-magee.com> date: 2024-12-09T13:28:57+08:00 summary: gh-126925: Modify how iOS test results are gathered (#127592) Adds a `use_system_log` config item to enable stdout/stderr redirection for Apple platforms. This log streaming is then used by a new iOS test runner script, allowing the display of test suite output at runtime. The iOS test runner script can be used by any Python project, not just the CPython test suite. files: A Lib/_apple_support.py A Lib/test/test_apple.py A Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst A Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst A iOS/testbed/__main__.py M Doc/c-api/init_config.rst M Doc/using/ios.rst M Doc/whatsnew/3.14.rst M Include/cpython/initconfig.h M Lib/test/test_capi/test_config.py M Lib/test/test_embed.py M Makefile.pre.in M Python/initconfig.c M Python/pylifecycle.c M Python/stdlib_module_names.h M iOS/README.rst M iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj M iOS/testbed/iOSTestbedTests/iOSTestbedTests.m diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index d6569ddcf586fa..7497bf241fb10e 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1281,6 +1281,15 @@ PyConfig Default: ``1`` in Python config and ``0`` in isolated config. + .. c:member:: int use_system_logger + + If non-zero, ``stdout`` and ``stderr`` will be redirected to the system + log. + + Only available on macOS 10.12 and later, and on iOS. + + Default: ``0`` (don't use system log). + .. c:member:: int user_site_directory If non-zero, add the user site directory to :data:`sys.path`. diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index 4d4eb2031ee980..aa43f75ec35a6c 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -292,10 +292,12 @@ To add Python to an iOS Xcode project: 10. Add Objective C code to initialize and use a Python interpreter in embedded mode. You should ensure that: - * :c:member:`UTF-8 mode <PyPreConfig.utf8_mode>` is *enabled*; - * :c:member:`Buffered stdio <PyConfig.buffered_stdio>` is *disabled*; - * :c:member:`Writing bytecode <PyConfig.write_bytecode>` is *disabled*; - * :c:member:`Signal handlers <PyConfig.install_signal_handlers>` are *enabled*; + * UTF-8 mode (:c:member:`PyPreConfig.utf8_mode`) is *enabled*; + * Buffered stdio (:c:member:`PyConfig.buffered_stdio`) is *disabled*; + * Writing bytecode (:c:member:`PyConfig.write_bytecode`) is *disabled*; + * Signal handlers (:c:member:`PyConfig.install_signal_handlers`) are *enabled*; + * System logging (:c:member:`PyConfig.use_system_logger`) is *enabled* + (optional, but strongly recommended); * ``PYTHONHOME`` for the interpreter is configured to point at the ``python`` subfolder of your app's bundle; and * The ``PYTHONPATH`` for the interpreter includes: @@ -324,6 +326,49 @@ modules in your app, some additional steps will be required: * If you're using a separate folder for third-party packages, ensure that folder is included as part of the ``PYTHONPATH`` configuration in step 10. +Testing a Python package +------------------------ + +The CPython source tree contains :source:`a testbed project <iOS/testbed>` that +is used to run the CPython test suite on the iOS simulator. This testbed can also +be used as a testbed project for running your Python library's test suite on iOS. + +After building or obtaining an iOS XCFramework (See :source:`iOS/README.rst` +for details), create a clone of the Python iOS testbed project by running: + +.. code-block:: bash + + $ python iOS/testbed clone --framework <path/to/Python.xcframework> --app <path/to/module1> --app <path/to/module2> app-testbed + +You will need to modify the ``iOS/testbed`` reference to point to that +directory in the CPython source tree; any folders specified with the ``--app`` +flag will be copied into the cloned testbed project. The resulting testbed will +be created in the ``app-testbed`` folder. In this example, the ``module1`` and +``module2`` would be importable modules at runtime. If your project has +additional dependencies, they can be installed into the +``app-testbed/iOSTestbed/app_packages`` folder (using ``pip install --target +app-testbed/iOSTestbed/app_packages`` or similar). + +You can then use the ``app-testbed`` folder to run the test suite for your app, +For example, if ``module1.tests`` was the entry point to your test suite, you +could run: + +.. code-block:: bash + + $ python app-testbed run -- module1.tests + +This is the equivalent of running ``python -m module1.tests`` on a desktop +Python build. Any arguments after the ``--`` will be passed to the testbed as +if they were arguments to ``python -m`` on a desktop machine. + +You can also open the testbed project in Xcode by running: + +.. code-block:: bash + + $ open app-testbed/iOSTestbed.xcodeproj + +This will allow you to use the full Xcode suite of tools for debugging. + App Store Compliance ==================== diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b300e348679438..b71d31f9742fe0 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -245,6 +245,13 @@ Other language changes making it a :term:`generic type`. (Contributed by Brian Schubert in :gh:`126012`.) +* iOS and macOS apps can now be configured to redirect ``stdout`` and + ``stderr`` content to the system log. (Contributed by Russell Keith-Magee in + :gh:`127592`.) + +* The iOS testbed is now able to stream test output while the test is running. + The testbed can also be used to run the test suite of projects other than + CPython itself. (Contributed by Russell Keith-Magee in :gh:`127592`.) New modules =========== diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index f69c586a4f96f3..8ef19f677066c2 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -179,6 +179,9 @@ typedef struct PyConfig { int use_frozen_modules; int safe_path; int int_max_str_digits; +#ifdef __APPLE__ + int use_system_logger; +#endif int cpu_count; #ifdef Py_GIL_DISABLED diff --git a/Lib/_apple_support.py b/Lib/_apple_support.py new file mode 100644 index 00000000000000..92febdcf587070 --- /dev/null +++ b/Lib/_apple_support.py @@ -0,0 +1,66 @@ +import io +import sys + + +def init_streams(log_write, stdout_level, stderr_level): + # Redirect stdout and stderr to the Apple system log. This method is + # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger + # is enabled. + sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors) + sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors) + + +class SystemLog(io.TextIOWrapper): + def __init__(self, log_write, level, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(LogStream(log_write, level), **kwargs) + + def __repr__(self): + return f"<SystemLog (level {self.buffer.level})>" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line, so split + # the string before sending it to the superclass. + for line in s.splitlines(keepends=True): + super().write(line) + + return len(s) + + +class LogStream(io.RawIOBase): + def __init__(self, log_write, level): + self.log_write = log_write + self.level = level + + def __repr__(self): + return f"<LogStream (level {self.level!r})>" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80")) + + return len(b) diff --git a/Lib/test/test_apple.py b/Lib/test/test_apple.py new file mode 100644 index 00000000000000..ab5296afad1d3f --- /dev/null +++ b/Lib/test/test_apple.py @@ -0,0 +1,155 @@ +import unittest +from _apple_support import SystemLog +from test.support import is_apple +from unittest.mock import Mock, call + +if not is_apple: + raise unittest.SkipTest("Apple-specific") + + +# Test redirection of stdout and stderr to the Apple system log. +class TestAppleSystemLogOutput(unittest.TestCase): + maxDiff = None + + def assert_writes(self, output): + self.assertEqual( + self.log_write.mock_calls, + [ + call(self.log_level, line) + for line in output + ] + ) + + self.log_write.reset_mock() + + def setUp(self): + self.log_write = Mock() + self.log_level = 42 + self.log = SystemLog(self.log_write, self.log_level, errors="replace") + + def test_repr(self): + self.assertEqual(repr(self.log), "<SystemLog (level 42)>") + self.assertEqual(repr(self.log.buffer), "<LogStream (level 42)>") + + def test_log_config(self): + self.assertIs(self.log.writable(), True) + self.assertIs(self.log.readable(), False) + + self.assertEqual("UTF-8", self.log.encoding) + self.assertEqual("replace", self.log.errors) + + self.assertIs(self.log.line_buffering, True) + self.assertIs(self.log.write_through, False) + + def test_empty_str(self): + self.log.write("") + self.log.flush() + + self.assert_writes([]) + + def test_simple_str(self): + self.log.write("hello world\n") + + self.assert_writes([b"hello world\n"]) + + def test_buffered_str(self): + self.log.write("h") + self.log.write("ello") + self.log.write(" ") + self.log.write("world\n") + self.log.write("goodbye.") + self.log.flush() + + self.assert_writes([b"hello world\n", b"goodbye."]) + + def test_manual_flush(self): + self.log.write("Hello") + + self.assert_writes([]) + + self.log.write(" world\nHere for a while...\nGoodbye") + self.assert_writes([b"Hello world\n", b"Here for a while...\n"]) + + self.log.write(" world\nHello again") + self.assert_writes([b"Goodbye world\n"]) + + self.log.flush() + self.assert_writes([b"Hello again"]) + + def test_non_ascii(self): + # Spanish + self.log.write("ol\u00e9\n") + self.assert_writes([b"ol\xc3\xa9\n"]) + + # Chinese + self.log.write("\u4e2d\u6587\n") + self.assert_writes([b"\xe4\xb8\xad\xe6\x96\x87\n"]) + + # Printing Non-BMP emoji + self.log.write("\U0001f600\n") + self.assert_writes([b"\xf0\x9f\x98\x80\n"]) + + # Non-encodable surrogates are replaced + self.log.write("\ud800\udc00\n") + self.assert_writes([b"??\n"]) + + def test_modified_null(self): + # Null characters are logged using "modified UTF-8". + self.log.write("\u0000\n") + self.assert_writes([b"\xc0\x80\n"]) + self.log.write("a\u0000\n") + self.assert_writes([b"a\xc0\x80\n"]) + self.log.write("\u0000b\n") + self.assert_writes([b"\xc0\x80b\n"]) + self.log.write("a\u0000b\n") + self.assert_writes([b"a\xc0\x80b\n"]) + + def test_nonstandard_str(self): + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + self.log.write(CustomStr("custom\n")) + self.assert_writes([b"custom\n"]) + + def test_non_str(self): + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + self.log.write(obj) + + def test_byteslike_in_buffer(self): + # The underlying buffer *can* accept bytes-like objects + self.log.buffer.write(bytearray(b"hello")) + self.log.flush() + + self.log.buffer.write(b"") + self.log.flush() + + self.log.buffer.write(b"goodbye") + self.log.flush() + + self.assert_writes([b"hello", b"goodbye"]) + + def test_non_byteslike_in_buffer(self): + for obj in ["hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + self.log.buffer.write(obj) diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 77730ad2f32085..a3179efe4a8235 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -110,6 +110,10 @@ def test_config_get(self): options.extend(( ("_pystats", bool, None), )) + if support.is_apple: + options.extend(( + ("use_system_logger", bool, None), + )) for name, option_type, sys_attr in options: with self.subTest(name=name, option_type=option_type, diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 5c38b28322deb4..7110fb889f3c8e 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -649,6 +649,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT.update({ 'legacy_windows_stdio': False, }) + if support.is_apple: + CONFIG_COMPAT['use_system_logger'] = False CONFIG_PYTHON = dict(CONFIG_COMPAT, _config_init=API_PYTHON, diff --git a/Makefile.pre.in b/Makefile.pre.in index dd8a3ab82eacd2..7b66802147dc3a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2146,7 +2146,6 @@ testuniversal: all # This must be run *after* a `make install` has completed the build. The # `--with-framework-name` argument *cannot* be used when configuring the build. XCFOLDER:=iOSTestbed.$(MULTIARCH).$(shell date +%s) -XCRESULT=$(XCFOLDER)/$(MULTIARCH).xcresult .PHONY: testios testios: @if test "$(MACHDEP)" != "ios"; then \ @@ -2165,29 +2164,12 @@ testios: echo "Cannot find a finalized iOS Python.framework. Have you run 'make install' to finalize the framework build?"; \ exit 1;\ fi - # Copy the testbed project into the build folder - cp -r $(srcdir)/iOS/testbed $(XCFOLDER) - # Copy the framework from the install location to the testbed project. - cp -r $(PYTHONFRAMEWORKPREFIX)/* $(XCFOLDER)/Python.xcframework/ios-arm64_x86_64-simulator - - # Run the test suite for the Xcode project, targeting the iOS simulator. - # If the suite fails, touch a file in the test folder as a marker - if ! xcodebuild test -project $(XCFOLDER)/iOSTestbed.xcodeproj -scheme "iOSTestbed" -destination "platform=iOS Simulator,name=iPhone SE (3rd Generation)" -resultBundlePath $(XCRESULT) -derivedDataPath $(XCFOLDER)/DerivedData ; then \ - touch $(XCFOLDER)/failed; \ - fi - # Regardless of success or failure, extract and print the test output - xcrun xcresulttool get --path $(XCRESULT) \ - --id $$( \ - xcrun xcresulttool get --path $(XCRESULT) --format json | \ - $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['actions']['_values'][0]['actionResult']['logRef']['id']['_value'])" \ - ) \ - --format json | \ - $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['subsections']['_values'][1]['subsections']['_values'][0]['emittedOutput']['_value'])" + # Clone the testbed project into the XCFOLDER + $(PYTHON_FOR_BUILD) $(srcdir)/iOS/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" - @if test -e $(XCFOLDER)/failed ; then \ - exit 1; \ - fi + # Run the testbed project + $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run -- test -uall --single-process --rerun -W # Like test, but using --slow-ci which enables all test resources and use # longer timeout. Run an optional pybuildbot.identify script to include diff --git a/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst b/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst new file mode 100644 index 00000000000000..677acf5baab3fa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst @@ -0,0 +1,2 @@ +macOS and iOS apps can now choose to redirect stdout and stderr to the +system log during interpreter configuration. diff --git a/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst b/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst new file mode 100644 index 00000000000000..fb307c7cb9bf1d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst @@ -0,0 +1,2 @@ +iOS test results are now streamed during test execution, and the deprecated +xcresulttool is no longer used. diff --git a/Python/initconfig.c b/Python/initconfig.c index 438f8a5c1cf1ce..7851b86db1f6d0 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -168,6 +168,9 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS), SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS), SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS), +#ifdef __APPLE__ + SPEC(use_system_logger, BOOL, PUBLIC, NO_SYS), +#endif SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_user_site SPEC(warn_default_encoding, BOOL, READ_ONLY, NO_SYS), @@ -884,6 +887,9 @@ config_check_consistency(const PyConfig *config) assert(config->cpu_count != 0); // config->use_frozen_modules is initialized later // by _PyConfig_InitImportConfig(). +#ifdef __APPLE__ + assert(config->use_system_logger >= 0); +#endif #ifdef Py_STATS assert(config->_pystats >= 0); #endif @@ -986,6 +992,9 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->_is_python_build = 0; config->code_debug_ranges = 1; config->cpu_count = -1; +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif #ifdef Py_GIL_DISABLED config->enable_gil = _PyConfig_GIL_DEFAULT; config->tlbc_enabled = 1; @@ -1015,6 +1024,9 @@ config_init_defaults(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif } @@ -1049,6 +1061,9 @@ PyConfig_InitIsolatedConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ceb30e9f02df2c..06418123d6dd9b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -45,7 +45,9 @@ #endif #if defined(__APPLE__) +# include <AvailabilityMacros.h> # include <mach-o/loader.h> +# include <os/log.h> #endif #ifdef HAVE_SIGNAL_H @@ -75,6 +77,9 @@ static PyStatus init_sys_streams(PyThreadState *tstate); #ifdef __ANDROID__ static PyStatus init_android_streams(PyThreadState *tstate); #endif +#if defined(__APPLE__) +static PyStatus init_apple_streams(PyThreadState *tstate); +#endif static void wait_for_thread_shutdown(PyThreadState *tstate); static void finalize_subinterpreters(void); static void call_ll_exitfuncs(_PyRuntimeState *runtime); @@ -1257,6 +1262,14 @@ init_interp_main(PyThreadState *tstate) return status; } #endif +#if defined(__APPLE__) + if (config->use_system_logger) { + status = init_apple_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } +#endif #ifdef Py_DEBUG run_presite(tstate); @@ -2933,6 +2946,75 @@ init_android_streams(PyThreadState *tstate) #endif // __ANDROID__ +#if defined(__APPLE__) + +static PyObject * +apple_log_write_impl(PyObject *self, PyObject *args) +{ + int logtype = 0; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "iy", &logtype, &text)) { + return NULL; + } + + // Call the underlying Apple logging API. The os_log unified logging APIs + // were introduced in macOS 10.12, iOS 10.0, tvOS 10.0, and watchOS 3.0; + // this call is a no-op on older versions. + #if TARGET_OS_IPHONE || (TARGET_OS_OSX && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12) + // Pass the user-provided text through explicit %s formatting + // to avoid % literals being interpreted as a formatting directive. + os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); + #endif + Py_RETURN_NONE; +} + + +static PyMethodDef apple_log_write_method = { + "apple_log_write", apple_log_write_impl, METH_VARARGS +}; + + +static PyStatus +init_apple_streams(PyThreadState *tstate) +{ + PyStatus status = _PyStatus_OK(); + PyObject *_apple_support = NULL; + PyObject *apple_log_write = NULL; + PyObject *result = NULL; + + _apple_support = PyImport_ImportModule("_apple_support"); + if (_apple_support == NULL) { + goto error; + } + + apple_log_write = PyCFunction_New(&apple_log_write_method, NULL); + if (apple_log_write == NULL) { + goto error; + } + + // Initialize the logging streams, sending stdout -> Default; stderr -> Error + result = PyObject_CallMethod( + _apple_support, "init_streams", "Oii", + apple_log_write, OS_LOG_TYPE_DEFAULT, OS_LOG_TYPE_ERROR); + if (result == NULL) { + goto error; + } + + goto done; + +error: + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Apple log streams"); + +done: + Py_XDECREF(result); + Py_XDECREF(apple_log_write); + Py_XDECREF(_apple_support); + return status; +} + +#endif // __APPLE__ + static void _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index c8cdb933bb108f..584b050fc4bb6e 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -6,6 +6,7 @@ static const char* _Py_stdlib_module_names[] = { "_abc", "_aix_support", "_android_support", +"_apple_support", "_ast", "_asyncio", "_bisect", diff --git a/iOS/README.rst b/iOS/README.rst index e33455eef8f44a..9cea98cf1abbfa 100644 --- a/iOS/README.rst +++ b/iOS/README.rst @@ -285,52 +285,42 @@ This will: * Install the Python iOS framework into the copy of the testbed project; and * Run the test suite on an "iPhone SE (3rd generation)" simulator. -While the test suite is running, Xcode does not display any console output. -After showing some Xcode build commands, the console output will print ``Testing -started``, and then appear to stop. It will remain in this state until the test -suite completes. On a 2022 M1 MacBook Pro, the test suite takes approximately 12 -minutes to run; a couple of extra minutes is required to boot and prepare the -iOS simulator. - On success, the test suite will exit and report successful completion of the -test suite. No output of the Python test suite will be displayed. - -On failure, the output of the Python test suite *will* be displayed. This will -show the details of the tests that failed. +test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 12 +minutes to run; a couple of extra minutes is required to compile the testbed +project, and then boot and prepare the iOS simulator. Debugging test failures ----------------------- -The easiest way to diagnose a single test failure is to open the testbed project -in Xcode and run the tests from there using the "Product > Test" menu item. - -To test in Xcode, you must ensure the testbed project has a copy of a compiled -framework. If you've configured your build with the default install location of -``iOS/Frameworks``, you can copy from that location into the test project. To -test on an ARM64 simulator, run:: - - $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/* - $ cp -r iOS/Frameworks/arm64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator +Running ``make test`` generates a standalone version of the ``iOS/testbed`` +project, and runs the full test suite. It does this using ``iOS/testbed`` +itself - the folder is an executable module that can be used to create and run +a clone of the testbed project. -To test on an x86-64 simulator, run:: +You can generate your own standalone testbed instance by running:: - $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/* - $ cp -r iOS/Frameworks/x86_64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator + $ python iOS/testbed clone --framework iOS/Frameworks/arm64-iphonesimulator my-testbed -To test on a physical device:: +This invocation assumes that ``iOS/Frameworks/arm64-iphonesimulator`` is the +path to the iOS simulator framework for your platform (ARM64 in this case); +``my-testbed`` is the name of the folder for the new testbed clone. - $ rm -rf iOS/testbed/Python.xcframework/ios-arm64/* - $ cp -r iOS/Frameworks/arm64-iphoneos/* iOS/testbed/Python.xcframework/ios-arm64 +You can then use the ``my-testbed`` folder to run the Python test suite, +passing in any command line arguments you may require. For example, if you're +trying to diagnose a failure in the ``os`` module, you might run:: -Alternatively, you can configure your build to install directly into the -testbed project. For a simulator, use:: + $ python my-testbed run -- test -W test_os - --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator +This is the equivalent of running ``python -m test -W test_os`` on a desktop +Python build. Any arguments after the ``--`` will be passed to testbed as if +they were arguments to ``python -m`` on a desktop machine. -For a physical device, use:: +You can also open the testbed project in Xcode by running:: - --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64 + $ open my-testbed/iOSTestbed.xcodeproj +This will allow you to use the full Xcode suite of tools for debugging. Testing on an iOS device ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py new file mode 100644 index 00000000000000..22570ee0f3ed04 --- /dev/null +++ b/iOS/testbed/__main__.py @@ -0,0 +1,365 @@ +import argparse +import asyncio +import json +import plistlib +import shutil +import subprocess +import sys +from contextlib import asynccontextmanager +from datetime import datetime +from pathlib import Path + + +DECODE_ARGS = ("UTF-8", "backslashreplace") + + +# Work around a bug involving sys.exit and TaskGroups +# (https://github.com/python/cpython/issues/101515). +def exit(*args): + raise MySystemExit(*args) + + +class MySystemExit(Exception): + pass + + +# All subprocesses are executed through this context manager so that no matter +# what happens, they can always be cancelled from another task, and they will +# always be cleaned up on exit. +@asynccontextmanager +async def async_process(*args, **kwargs): + process = await asyncio.create_subprocess_exec(*args, **kwargs) + try: + yield process + finally: + if process.returncode is None: + # Allow a reasonably long time for Xcode to clean itself up, + # because we don't want stale emulators left behind. + timeout = 10 + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout) + except TimeoutError: + print( + f"Command {args} did not terminate after {timeout} seconds " + f" - sending SIGKILL" + ) + process.kill() + + # Even after killing the process we must still wait for it, + # otherwise we'll get the warning "Exception ignored in __del__". + await asyncio.wait_for(process.wait(), timeout=1) + + +async def async_check_output(*args, **kwargs): + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs + ) as process: + stdout, stderr = await process.communicate() + if process.returncode == 0: + return stdout.decode(*DECODE_ARGS) + else: + raise subprocess.CalledProcessError( + process.returncode, + args, + stdout.decode(*DECODE_ARGS), + stderr.decode(*DECODE_ARGS), + ) + + +# Return a list of UDIDs associated with booted simulators +async def list_devices(): + # List the testing simulators, in JSON format + raw_json = await async_check_output( + "xcrun", "simctl", "--set", "testing", "list", "-j" + ) + json_data = json.loads(raw_json) + + # Filter out the booted iOS simulators + return [ + simulator["udid"] + for runtime, simulators in json_data["devices"].items() + for simulator in simulators + if runtime.split(".")[-1].startswith("iOS") and simulator["state"] == "Booted" + ] + + +async def find_device(initial_devices): + while True: + new_devices = set(await list_devices()).difference(initial_devices) + if len(new_devices) == 0: + await asyncio.sleep(1) + elif len(new_devices) == 1: + udid = new_devices.pop() + print(f"{datetime.now():%Y-%m-%d %H:%M:%S}: New test simulator detected") + print(f"UDID: {udid}") + return udid + else: + exit(f"Found more than one new device: {new_devices}") + + +async def log_stream_task(initial_devices): + # Wait up to 5 minutes for the build to complete and the simulator to boot. + udid = await asyncio.wait_for(find_device(initial_devices), 5 * 60) + + # Stream the iOS device's logs, filtering out messages that come from the + # XCTest test suite (catching NSLog messages from the test method), or + # Python itself (catching stdout/stderr content routed to the system log + # with config->use_system_logger). + args = [ + "xcrun", + "simctl", + "--set", + "testing", + "spawn", + udid, + "log", + "stream", + "--style", + "compact", + "--predicate", + ( + 'senderImagePath ENDSWITH "/iOSTestbedTests.xctest/iOSTestbedTests"' + ' OR senderImagePath ENDSWITH "/Python.framework/Python"' + ), + ] + + async with async_process( + *args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + suppress_dupes = False + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + # The iOS log streamer can sometimes lag; when it does, it outputs + # a warning about messages being dropped... often multiple times. + # Only print the first of these duplicated warnings. + if line.startswith("=== Messages dropped "): + if not suppress_dupes: + suppress_dupes = True + sys.stdout.write(line) + else: + suppress_dupes = False + sys.stdout.write(line) + + +async def xcode_test(location, simulator): + # Run the test suite on the named simulator + args = [ + "xcodebuild", + "test", + "-project", + str(location / "iOSTestbed.xcodeproj"), + "-scheme", + "iOSTestbed", + "-destination", + f"platform=iOS Simulator,name={simulator}", + "-resultBundlePath", + str(location / f"{datetime.now():%Y%m%d-%H%M%S}.xcresult"), + "-derivedDataPath", + str(location / "DerivedData"), + ] + async with async_process( + *args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + sys.stdout.write(line) + + status = await asyncio.wait_for(process.wait(), timeout=1) + exit(status) + + +def clone_testbed( + source: Path, + target: Path, + framework: Path, + apps: list[Path], +) -> None: + if target.exists(): + print(f"{target} already exists; aborting without creating project.") + sys.exit(10) + + if framework is None: + if not (source / "Python.xcframework/ios-arm64_x86_64-simulator/bin").is_dir(): + print( + f"The testbed being cloned ({source}) does not contain " + f"a simulator framework. Re-run with --framework" + ) + sys.exit(11) + else: + if not framework.is_dir(): + print(f"{framework} does not exist.") + sys.exit(12) + elif not ( + framework.suffix == ".xcframework" + or (framework / "Python.framework").is_dir() + ): + print( + f"{framework} is not an XCframework, " + f"or a simulator slice of a framework build." + ) + sys.exit(13) + + print("Cloning testbed project...") + shutil.copytree(source, target) + + if framework is not None: + if framework.suffix == ".xcframework": + print("Installing XCFramework...") + xc_framework_path = target / "Python.xcframework" + shutil.rmtree(xc_framework_path) + shutil.copytree(framework, xc_framework_path) + else: + print("Installing simulator Framework...") + sim_framework_path = ( + target / "Python.xcframework" / "ios-arm64_x86_64-simulator" + ) + shutil.rmtree(sim_framework_path) + shutil.copytree(framework, sim_framework_path) + else: + print("Using pre-existing iOS framework.") + + for app_src in apps: + print(f"Installing app {app_src.name!r}...") + app_target = target / f"iOSTestbed/app/{app_src.name}" + if app_target.is_dir(): + shutil.rmtree(app_target) + shutil.copytree(app_src, app_target) + + print(f"Testbed project created in {target}") + + +def update_plist(testbed_path, args): + # Add the test runner arguments to the testbed's Info.plist file. + info_plist = testbed_path / "iOSTestbed" / "iOSTestbed-Info.plist" + with info_plist.open("rb") as f: + info = plistlib.load(f) + + info["TestArgs"] = args + + with info_plist.open("wb") as f: + plistlib.dump(info, f) + + +async def run_testbed(simulator: str, args: list[str]): + location = Path(__file__).parent + print("Updating plist...") + update_plist(location, args) + + # Get the list of devices that are booted at the start of the test run. + # The simulator started by the test suite will be detected as the new + # entry that appears on the device list. + initial_devices = await list_devices() + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(log_stream_task(initial_devices)) + tg.create_task(xcode_test(location, simulator)) + except* MySystemExit as e: + raise SystemExit(*e.exceptions[0].args) from None + except* subprocess.CalledProcessError as e: + # Extract it from the ExceptionGroup so it can be handled by `main`. + raise e.exceptions[0] + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Manages the process of testing a Python project in the iOS simulator." + ), + ) + + subcommands = parser.add_subparsers(dest="subcommand") + + clone = subcommands.add_parser( + "clone", + description=( + "Clone the testbed project, copying in an iOS Python framework and" + "any specified application code." + ), + help="Clone a testbed project to a new location.", + ) + clone.add_argument( + "--framework", + help=( + "The location of the XCFramework (or simulator-only slice of an " + "XCFramework) to use when running the testbed" + ), + ) + clone.add_argument( + "--app", + dest="apps", + action="append", + default=[], + help="The location of any code to include in the testbed project", + ) + clone.add_argument( + "location", + help="The path where the testbed will be cloned.", + ) + + run = subcommands.add_parser( + "run", + usage="%(prog)s [-h] [--simulator SIMULATOR] -- <test arg> [<test arg> ...]", + description=( + "Run a testbed project. The arguments provided after `--` will be " + "passed to the running iOS process as if they were arguments to " + "`python -m`." + ), + help="Run a testbed project", + ) + run.add_argument( + "--simulator", + default="iPhone SE (3rd Generation)", + help="The name of the simulator to use (default: 'iPhone SE (3rd Generation)')", + ) + + try: + pos = sys.argv.index("--") + testbed_args = sys.argv[1:pos] + test_args = sys.argv[pos + 1 :] + except ValueError: + testbed_args = sys.argv[1:] + test_args = [] + + context = parser.parse_args(testbed_args) + + if context.subcommand == "clone": + clone_testbed( + source=Path(__file__).parent, + target=Path(context.location), + framework=Path(context.framework) if context.framework else None, + apps=[Path(app) for app in context.apps], + ) + elif context.subcommand == "run": + if test_args: + if not ( + Path(__file__).parent / "Python.xcframework/ios-arm64_x86_64-simulator/bin" + ).is_dir(): + print( + f"Testbed does not contain a compiled iOS framework. Use " + f"`python {sys.argv[0]} clone ...` to create a runnable " + f"clone of this testbed." + ) + sys.exit(20) + + asyncio.run( + run_testbed( + simulator=context.simulator, + args=test_args, + ) + ) + else: + print(f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)") + print() + parser.print_help(sys.stderr) + sys.exit(21) + else: + parser.print_help(sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj index 6819ac0eeed95f..c7d63909ee2453 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -263,6 +263,7 @@ runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n echo \"Installing Python modules for iOS Simulator\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n echo \"Installing Python modules for iOS Device\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n"; + showEnvVarsInLog = 0; }; 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = { isa = PBXShellScriptBuildPhase; @@ -282,6 +283,7 @@ runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\necho \"Install app package extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app_packages\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app_packages/ \"$FULL_EXT\"\ndone\necho \"Install app extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$ EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m index db00d43da85cbc..ac78456a61e65e 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -50,6 +50,8 @@ - (void)testPython { // Enforce UTF-8 encoding for stderr, stdout, file-system encoding and locale. // See https://docs.python.org/3/library/os.html#python-utf-8-mode. preconfig.utf8_mode = 1; + // Use the system logger for stdout/err + config.use_system_logger = 1; // Don't buffer stdio. We want output to appears in the log immediately config.buffered_stdio = 0; // Don't write bytecode; we can't modify the app bundle