Mailman 3 December 2023 - Python-checkins

gh-112532: Use separate mimalloc heaps for GC objects (gh-113263)
by corona10 Dec. 26, 2023

Dec. 26, 2023

https://github.com/python/cpython/commit/acf3bcc8861983dcd6896682283a480450… commit: acf3bcc8861983dcd6896682283a480450f9a1e3 branch: main author: Sam Gross <colesbury(a)gmail.com> committer: corona10 <donghee.na92(a)gmail.com> date: 2023-12-27T01:53:20+09:00 summary: gh-112532: Use separate mimalloc heaps for GC objects (gh-113263) * gh-112532: Use separate mimalloc heaps for GC objects In `--disable-gil` builds, we now use four separate heaps in anticipation of using mimalloc to find GC objects when the GIL is disabled. To support this, we also make a few changes to mimalloc: * `mi_heap_t` and `mi_tld_t` initialization is split from allocation. This allows us to have a `mi_tld_t` per-`PyThreadState`, which is important to keep interpreter isolation, since the same OS thread may run in multiple interpreters (using different PyThreadStates.) * Heap abandoning (mi_heap_collect_ex) can now be called from a different thread than the one that created the heap. This is necessary because we may clear and delete the containing PyThreadStates from a different thread during finalization and after fork(). * Use enum instead of defines and guard mimalloc includes. * The enum typedef will be convenient for future PRs that use the type. * Guarding the mimalloc includes allows us to unconditionally include pycore_mimalloc.h from other header files that rely on things like `struct _mimalloc_thread_state`. * Only define _mimalloc_thread_state in Py_GIL_DISABLED builds files: M Include/internal/mimalloc/mimalloc/internal.h M Include/internal/pycore_mimalloc.h M Include/internal/pycore_pystate.h M Include/internal/pycore_tstate.h M Objects/mimalloc/heap.c M Objects/mimalloc/init.c M Objects/obmalloc.c M Python/pylifecycle.c M Python/pystate.c diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index f076bc6a40f977..cb6e211de5bb63 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -85,6 +85,7 @@ mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; mi_heap_t* _mi_heap_main_get(void); // statically allocated main backing heap void _mi_thread_done(mi_heap_t* heap); void _mi_thread_data_collect(void); +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap); // os.c void _mi_os_init(void); // called from process init @@ -170,6 +171,7 @@ size_t _mi_bin_size(uint8_t bin); // for stats uint8_t _mi_bin(size_t size); // for stats // "heap.c" +void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id); void _mi_heap_destroy_pages(mi_heap_t* heap); void _mi_heap_collect_abandon(mi_heap_t* heap); void _mi_heap_set_default_direct(mi_heap_t* heap); diff --git a/Include/internal/pycore_mimalloc.h b/Include/internal/pycore_mimalloc.h index c29dc82a42762a..adebb559dae658 100644 --- a/Include/internal/pycore_mimalloc.h +++ b/Include/internal/pycore_mimalloc.h @@ -9,11 +9,37 @@ # error "pycore_mimalloc.h must be included before mimalloc.h" #endif +typedef enum { + _Py_MIMALLOC_HEAP_MEM = 0, // PyMem_Malloc() and friends + _Py_MIMALLOC_HEAP_OBJECT = 1, // non-GC objects + _Py_MIMALLOC_HEAP_GC = 2, // GC objects without pre-header + _Py_MIMALLOC_HEAP_GC_PRE = 3, // GC objects with pre-header + _Py_MIMALLOC_HEAP_COUNT +} _Py_mimalloc_heap_id; + #include "pycore_pymem.h" + +#ifdef WITH_MIMALLOC #define MI_DEBUG_UNINIT PYMEM_CLEANBYTE #define MI_DEBUG_FREED PYMEM_DEADBYTE #define MI_DEBUG_PADDING PYMEM_FORBIDDENBYTE +#ifdef Py_DEBUG +# define MI_DEBUG 1 +#else +# define MI_DEBUG 0 +#endif #include "mimalloc.h" +#include "mimalloc/types.h" +#include "mimalloc/internal.h" +#endif + +#ifdef Py_GIL_DISABLED +struct _mimalloc_thread_state { + mi_heap_t *current_object_heap; + mi_heap_t heaps[_Py_MIMALLOC_HEAP_COUNT]; + mi_tld_t tld; +}; +#endif #endif // Py_INTERNAL_MIMALLOC_H diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index c031a38cd6bfa3..37b45faf8a74a0 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -187,6 +187,7 @@ extern PyThreadState * _PyThreadState_New( int whence); extern void _PyThreadState_Bind(PyThreadState *tstate); extern void _PyThreadState_DeleteExcept(PyThreadState *tstate); +extern void _PyThreadState_ClearMimallocHeaps(PyThreadState *tstate); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(PyObject*) _PyThreadState_GetDict(PyThreadState *tstate); diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 17f3e865641773..856ddd5e7e5ff0 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_mimalloc.h" // struct _mimalloc_thread_state + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields @@ -16,7 +18,10 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; - // TODO: add private fields here +#ifdef Py_GIL_DISABLED + struct _mimalloc_thread_state mimalloc; +#endif + } _PyThreadStateImpl; diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c index 4eb622ed4bad76..c50e3b05590b6f 100644 --- a/Objects/mimalloc/heap.c +++ b/Objects/mimalloc/heap.c @@ -123,6 +123,9 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) const bool force = collect >= MI_FORCE; _mi_deferred_free(heap, force); + // gh-112532: we may be called from a thread that is not the owner of the heap + bool is_main_thread = _mi_is_main_thread() && heap->thread_id == _mi_thread_id(); + // note: never reclaim on collect but leave it to threads that need storage to reclaim const bool force_main = #ifdef NDEBUG @@ -130,7 +133,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) #else collect >= MI_FORCE #endif - && _mi_is_main_thread() && mi_heap_is_backing(heap) && !heap->no_reclaim; + && is_main_thread && mi_heap_is_backing(heap) && !heap->no_reclaim; if (force_main) { // the main thread is abandoned (end-of-program), try to reclaim all abandoned segments. @@ -164,7 +167,7 @@ static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) } // collect regions on program-exit (or shared library unload) - if (force && _mi_is_main_thread() && mi_heap_is_backing(heap)) { + if (force && is_main_thread && mi_heap_is_backing(heap)) { _mi_thread_data_collect(); // collect thread data cache _mi_arena_collect(true /* force purge */, &heap->tld->stats); } @@ -206,18 +209,28 @@ mi_heap_t* mi_heap_get_backing(void) { return bheap; } -mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { - mi_heap_t* bheap = mi_heap_get_backing(); - mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? - if (heap == NULL) return NULL; +void _mi_heap_init_ex(mi_heap_t* heap, mi_tld_t* tld, mi_arena_id_t arena_id) +{ _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(mi_heap_t)); - heap->tld = bheap->tld; + heap->tld = tld; heap->thread_id = _mi_thread_id(); heap->arena_id = arena_id; - _mi_random_split(&bheap->random, &heap->random); + if (heap == tld->heap_backing) { + _mi_random_init(&heap->random); + } + else { + _mi_random_split(&tld->heap_backing->random, &heap->random); + } heap->cookie = _mi_heap_random_next(heap) | 1; heap->keys[0] = _mi_heap_random_next(heap); heap->keys[1] = _mi_heap_random_next(heap); +} + +mi_decl_nodiscard mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id) { + mi_heap_t* bheap = mi_heap_get_backing(); + mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); // todo: OS allocate in secure mode? + if (heap == NULL) return NULL; + _mi_heap_init_ex(heap, bheap->tld, arena_id); heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe // push on the thread local heaps list heap->next = heap->tld->heaps; diff --git a/Objects/mimalloc/init.c b/Objects/mimalloc/init.c index 7dfa7657737117..376e14b49b7c7b 100644 --- a/Objects/mimalloc/init.c +++ b/Objects/mimalloc/init.c @@ -297,24 +297,20 @@ static bool _mi_heap_init(void) { mi_thread_data_t* td = mi_thread_data_zalloc(); if (td == NULL) return false; - mi_tld_t* tld = &td->tld; - mi_heap_t* heap = &td->heap; + _mi_tld_init(&td->tld, &td->heap); + _mi_heap_init_ex(&td->heap, &td->tld, _mi_arena_id_none()); + _mi_heap_set_default_direct(&td->heap); + } + return false; +} + +void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) { _mi_memcpy_aligned(tld, &tld_empty, sizeof(*tld)); - _mi_memcpy_aligned(heap, &_mi_heap_empty, sizeof(*heap)); - heap->thread_id = _mi_thread_id(); - _mi_random_init(&heap->random); - heap->cookie = _mi_heap_random_next(heap) | 1; - heap->keys[0] = _mi_heap_random_next(heap); - heap->keys[1] = _mi_heap_random_next(heap); - heap->tld = tld; - tld->heap_backing = heap; - tld->heaps = heap; tld->segments.stats = &tld->stats; tld->segments.os = &tld->os; tld->os.stats = &tld->stats; - _mi_heap_set_default_direct(heap); - } - return false; + tld->heap_backing = bheap; + tld->heaps = bheap; } // Free the thread local default heap (called from `mi_thread_done`) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 99c95d90658b08..883adcb1c19b6e 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -88,19 +88,37 @@ _PyMem_RawFree(void *Py_UNUSED(ctx), void *ptr) void * _PyMem_MiMalloc(void *ctx, size_t size) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_malloc(heap, size); +#else return mi_malloc(size); +#endif } void * _PyMem_MiCalloc(void *ctx, size_t nelem, size_t elsize) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_calloc(heap, nelem, elsize); +#else return mi_calloc(nelem, elsize); +#endif } void * _PyMem_MiRealloc(void *ctx, void *ptr, size_t size) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = &tstate->mimalloc.heaps[_Py_MIMALLOC_HEAP_MEM]; + return mi_heap_realloc(heap, ptr, size); +#else return mi_realloc(ptr, size); +#endif } void @@ -112,20 +130,38 @@ _PyMem_MiFree(void *ctx, void *ptr) void * _PyObject_MiMalloc(void *ctx, size_t nbytes) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_malloc(heap, nbytes); +#else return mi_malloc(nbytes); +#endif } void * _PyObject_MiCalloc(void *ctx, size_t nelem, size_t elsize) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_calloc(heap, nelem, elsize); +#else return mi_calloc(nelem, elsize); +#endif } void * _PyObject_MiRealloc(void *ctx, void *ptr, size_t nbytes) { +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); + mi_heap_t *heap = tstate->mimalloc.current_object_heap; + return mi_heap_realloc(heap, ptr, nbytes); +#else return mi_realloc(ptr, nbytes); +#endif } void diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0ec29846b0850b..1d8af26e4a1cb7 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1794,6 +1794,10 @@ finalize_interp_clear(PyThreadState *tstate) } finalize_interp_types(tstate->interp); + + /* finalize_interp_types may allocate Python objects so we may need to + abandon mimalloc segments again */ + _PyThreadState_ClearMimallocHeaps(tstate); } diff --git a/Python/pystate.c b/Python/pystate.c index 632a119ea6d4f8..84e2d6ea172f2b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -236,6 +236,8 @@ tstate_is_bound(PyThreadState *tstate) static void bind_gilstate_tstate(PyThreadState *); static void unbind_gilstate_tstate(PyThreadState *); +static void tstate_mimalloc_bind(PyThreadState *); + static void bind_tstate(PyThreadState *tstate) { @@ -256,6 +258,9 @@ bind_tstate(PyThreadState *tstate) tstate->native_thread_id = PyThread_get_thread_native_id(); #endif + // mimalloc state needs to be initialized from the active thread. + tstate_mimalloc_bind(tstate); + tstate->_status.bound = 1; } @@ -1533,6 +1538,8 @@ PyThreadState_Clear(PyThreadState *tstate) tstate->on_delete(tstate->on_delete_data); } + _PyThreadState_ClearMimallocHeaps(tstate); + tstate->_status.cleared = 1; // XXX Call _PyThreadStateSwap(runtime, NULL) here if "current". @@ -2509,3 +2516,51 @@ _PyThreadState_MustExit(PyThreadState *tstate) } return 1; } + +/********************/ +/* mimalloc support */ +/********************/ + +static void +tstate_mimalloc_bind(PyThreadState *tstate) +{ +#ifdef Py_GIL_DISABLED + struct _mimalloc_thread_state *mts = &((_PyThreadStateImpl*)tstate)->mimalloc; + + // Initialize the mimalloc thread state. This must be called from the + // same thread that will use the thread state. The "mem" heap doubles as + // the "backing" heap. + mi_tld_t *tld = &mts->tld; + _mi_tld_init(tld, &mts->heaps[_Py_MIMALLOC_HEAP_MEM]); + + // Initialize each heap + for (Py_ssize_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { + _mi_heap_init_ex(&mts->heaps[i], tld, _mi_arena_id_none()); + } + + // By default, object allocations use _Py_MIMALLOC_HEAP_OBJECT. + // _PyObject_GC_New() and similar functions temporarily override this to + // use one of the GC heaps. + mts->current_object_heap = &mts->heaps[_Py_MIMALLOC_HEAP_OBJECT]; +#endif +} + +void +_PyThreadState_ClearMimallocHeaps(PyThreadState *tstate) +{ +#ifdef Py_GIL_DISABLED + if (!tstate->_status.bound) { + // The mimalloc heaps are only initialized when the thread is bound. + return; + } + + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + for (Py_ssize_t i = 0; i < _Py_MIMALLOC_HEAP_COUNT; i++) { + // Abandon all segments in use by this thread. This pushes them to + // a shared pool to later be reclaimed by other threads. It's important + // to do this before the thread state is destroyed so that objects + // remain visible to the GC. + _mi_heap_collect_abandon(&tstate_impl->mimalloc.heaps[i]); + } +#endif +}

1 0

gh-111971: Make _PyUnicode_FromId thread-safe in --disable-gil (gh-113489)
by corona10 Dec. 26, 2023

Dec. 26, 2023

https://github.com/python/cpython/commit/8f5b9987066f46daa67b622d913ff2c51c… commit: 8f5b9987066f46daa67b622d913ff2c51c949ed4 branch: main author: Donghee Na <donghee.na(a)python.org> committer: corona10 <donghee.na92(a)gmail.com> date: 2023-12-26T16:48:33Z summary: gh-111971: Make _PyUnicode_FromId thread-safe in --disable-gil (gh-113489) files: M Include/cpython/object.h M Objects/unicodeobject.c diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 762e8a3b86ee1e..d6482f4bc689a1 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -39,6 +39,10 @@ typedef struct _Py_Identifier { // Index in PyInterpreterState.unicode.ids.array. It is process-wide // unique and must be initialized to -1. Py_ssize_t index; + // Hidden PyMutex struct for non free-threaded build. + struct { + uint8_t v; + } mutex; } _Py_Identifier; #ifndef Py_BUILD_CORE diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index ad87206b2a8200..4b03cc3f4da5fa 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1897,6 +1897,7 @@ PyUnicode_FromString(const char *u) PyObject * _PyUnicode_FromId(_Py_Identifier *id) { + PyMutex_Lock((PyMutex *)&id->mutex); PyInterpreterState *interp = _PyInterpreterState_GET(); struct _Py_unicode_ids *ids = &interp->unicode.ids; @@ -1923,14 +1924,14 @@ _PyUnicode_FromId(_Py_Identifier *id) obj = ids->array[index]; if (obj) { // Return a borrowed reference - return obj; + goto end; } } obj = PyUnicode_DecodeUTF8Stateful(id->string, strlen(id->string), NULL, NULL); if (!obj) { - return NULL; + goto end; } PyUnicode_InternInPlace(&obj); @@ -1941,7 +1942,8 @@ _PyUnicode_FromId(_Py_Identifier *id) PyObject **new_array = PyMem_Realloc(ids->array, new_size * item_size); if (new_array == NULL) { PyErr_NoMemory(); - return NULL; + obj = NULL; + goto end; } memset(&new_array[ids->size], 0, (new_size - ids->size) * item_size); ids->array = new_array; @@ -1951,6 +1953,8 @@ _PyUnicode_FromId(_Py_Identifier *id) // The array stores a strong reference ids->array[index] = obj; +end: + PyMutex_Unlock((PyMutex *)&id->mutex); // Return a borrowed reference return obj; }

1 0

[3.12] Docs: make htmllive: open browser when ready (GH-113288) (#113484)
by hugovk Dec. 26, 2023

Dec. 26, 2023

https://github.com/python/cpython/commit/f445aeac2625132c121a9921feeefd88f4… commit: f445aeac2625132c121a9921feeefd88f4030835 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-12-26T12:08:43Z summary: [3.12] Docs: make htmllive: open browser when ready (GH-113288) (#113484) Co-authored-by: Hugo van Kemenade <hugovk(a)users.noreply.github.com> files: M Doc/Makefile diff --git a/Doc/Makefile b/Doc/Makefile index 7af56e965e1be4..38fd60f2ae01d1 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -19,8 +19,12 @@ SPHINXERRORHANDLING = -W PAPEROPT_a4 = -D latex_elements.papersize=a4paper PAPEROPT_letter = -D latex_elements.papersize=letterpaper -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j $(JOBS) \ - $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) +ALLSPHINXOPTS = -b $(BUILDER) \ + -d build/doctrees \ + -j $(JOBS) \ + $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) $(SPHINXERRORHANDLING) \ + . build/$(BUILDER) $(SOURCES) .PHONY: help help: @@ -142,7 +146,7 @@ htmlview: html .PHONY: htmllive htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild -htmllive: SPHINXOPTS = --re-ignore="/venv/" +htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 htmllive: html .PHONY: clean

1 0

[3.11] Docs: make htmllive: open browser when ready (GH-113288) (#113485)
by hugovk Dec. 26, 2023

Dec. 26, 2023

https://github.com/python/cpython/commit/a3a461648e6534385686ae827a7d0ef028… commit: a3a461648e6534385686ae827a7d0ef028e0a0ba branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-12-26T12:08:02Z summary: [3.11] Docs: make htmllive: open browser when ready (GH-113288) (#113485) Co-authored-by: Hugo van Kemenade <hugovk(a)users.noreply.github.com> files: M Doc/Makefile diff --git a/Doc/Makefile b/Doc/Makefile index f01b8a4ff958d9..d1ffade8ccbff5 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -19,8 +19,12 @@ SPHINXERRORHANDLING = -W PAPEROPT_a4 = -D latex_elements.papersize=a4paper PAPEROPT_letter = -D latex_elements.papersize=letterpaper -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j $(JOBS) \ - $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) +ALLSPHINXOPTS = -b $(BUILDER) \ + -d build/doctrees \ + -j $(JOBS) \ + $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) $(SPHINXERRORHANDLING) \ + . build/$(BUILDER) $(SOURCES) .PHONY: help help: @@ -156,7 +160,7 @@ htmlview: html .PHONY: htmllive htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild -htmllive: SPHINXOPTS = --re-ignore="/venv/" +htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 htmllive: html .PHONY: clean

1 0

Docs: make htmllive: open browser when ready (#113288)
by hugovk Dec. 26, 2023

Dec. 26, 2023

https://github.com/python/cpython/commit/36adc79041f4d2764e1daf7db5bb478923… commit: 36adc79041f4d2764e1daf7db5bb478923e89a1f branch: main author: Hugo van Kemenade <hugovk(a)users.noreply.github.com> committer: hugovk <hugovk(a)users.noreply.github.com> date: 2023-12-26T05:02:13-07:00 summary: Docs: make htmllive: open browser when ready (#113288) files: M Doc/Makefile diff --git a/Doc/Makefile b/Doc/Makefile index 7af56e965e1be4..38fd60f2ae01d1 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -19,8 +19,12 @@ SPHINXERRORHANDLING = -W PAPEROPT_a4 = -D latex_elements.papersize=a4paper PAPEROPT_letter = -D latex_elements.papersize=letterpaper -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j $(JOBS) \ - $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) +ALLSPHINXOPTS = -b $(BUILDER) \ + -d build/doctrees \ + -j $(JOBS) \ + $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) $(SPHINXERRORHANDLING) \ + . build/$(BUILDER) $(SOURCES) .PHONY: help help: @@ -142,7 +146,7 @@ htmlview: html .PHONY: htmllive htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild -htmllive: SPHINXOPTS = --re-ignore="/venv/" +htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 htmllive: html .PHONY: clean

1 0

[3.12] gh-113468: Remove the "_new_ suffix from class names in pydocfodder (GH-113469) (GH-113483)
by serhiy-storchaka Dec. 26, 2023

Dec. 26, 2023

https://github.com/python/cpython/commit/1cf576a358d2020ca230f1471c62b3379a… commit: 1cf576a358d2020ca230f1471c62b3379ab97193 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2023-12-26T11:27:18Z summary: [3.12] gh-113468: Remove the "_new_ suffix from class names in pydocfodder (GH-113469) (GH-113483) (cherry picked from commit 8a3d0e4a661e6c27e4c17c818ce4187a36579e5f) Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: M Lib/test/pydocfodder.py diff --git a/Lib/test/pydocfodder.py b/Lib/test/pydocfodder.py index d0750e5a43c0c0..a3ef2231243954 100644 --- a/Lib/test/pydocfodder.py +++ b/Lib/test/pydocfodder.py @@ -2,8 +2,8 @@ import types -class A_new: - "A new-style class." +class A: + "A class." def A_method(self): "Method defined in A." @@ -41,8 +41,8 @@ def _delx(self): A_int_alias = int -class B_new(A_new): - "A new-style class, derived from A_new." +class B(A): + "A class, derived from A." def AB_method(self): "Method defined in A and B." @@ -61,8 +61,8 @@ def BD_method(self): def BCD_method(self): "Method defined in B, C and D." -class C_new(A_new): - "A new-style class, derived from A_new." +class C(A): + "A class, derived from A." def AC_method(self): "Method defined in A and C." @@ -81,8 +81,8 @@ def C_method(self): def CD_method(self): "Method defined in C and D." -class D_new(B_new, C_new): - """A new-style class, derived from B_new and C_new. +class D(B, C): + """A class, derived from B and C. """ def AD_method(self):

1 0

gh-113468: Remove the "_new_ suffix from class names in pydocfodder (GH-113469)
by serhiy-storchaka Dec. 26, 2023

Dec. 26, 2023

https://github.com/python/cpython/commit/8a3d0e4a661e6c27e4c17c818ce4187a36… commit: 8a3d0e4a661e6c27e4c17c818ce4187a36579e5f branch: main author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2023-12-26T12:54:05+02:00 summary: gh-113468: Remove the "_new_ suffix from class names in pydocfodder (GH-113469) files: M Lib/test/pydocfodder.py diff --git a/Lib/test/pydocfodder.py b/Lib/test/pydocfodder.py index d0750e5a43c0c0..a3ef2231243954 100644 --- a/Lib/test/pydocfodder.py +++ b/Lib/test/pydocfodder.py @@ -2,8 +2,8 @@ import types -class A_new: - "A new-style class." +class A: + "A class." def A_method(self): "Method defined in A." @@ -41,8 +41,8 @@ def _delx(self): A_int_alias = int -class B_new(A_new): - "A new-style class, derived from A_new." +class B(A): + "A class, derived from A." def AB_method(self): "Method defined in A and B." @@ -61,8 +61,8 @@ def BD_method(self): def BCD_method(self): "Method defined in B, C and D." -class C_new(A_new): - "A new-style class, derived from A_new." +class C(A): + "A class, derived from A." def AC_method(self): "Method defined in A and C." @@ -81,8 +81,8 @@ def C_method(self): def CD_method(self): "Method defined in C and D." -class D_new(B_new, C_new): - """A new-style class, derived from B_new and C_new. +class D(B, C): + """A class, derived from B and C. """ def AD_method(self):

1 0

gh-66515: mailbox.MH now supports folders withou the ".mh_sequences" file (GH-804)
by serhiy-storchaka Dec. 26, 2023

Dec. 26, 2023

https://github.com/python/cpython/commit/e87cadc1ce194aae2c076e81298d6e8074… commit: e87cadc1ce194aae2c076e81298d6e8074f1bb45 branch: main author: Serhiy Storchaka <storchaka(a)gmail.com> committer: serhiy-storchaka <storchaka(a)gmail.com> date: 2023-12-26T11:15:14+02:00 summary: gh-66515: mailbox.MH now supports folders withou the ".mh_sequences" file (GH-804) (for example Claws Mail IMAP-cache folders). files: A Misc/NEWS.d/next/Library/2023-04-09-21-05-43.gh-issue-66515.0DS8Ya.rst M Doc/library/mailbox.rst M Lib/mailbox.py M Lib/test/test_mailbox.py diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 3ffd0981a4a4f2..fa5b273093f583 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -644,6 +644,10 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. :class:`!MH` instances have all of the methods of :class:`Mailbox` in addition to the following: + .. versionchanged:: 3.13 + + Supported folders that don't contain a :file:`.mh_sequences` file. + .. method:: list_folders() diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 574c01475dece6..0e1d49b399d077 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -1198,7 +1198,11 @@ def remove_folder(self, folder): def get_sequences(self): """Return a name-to-key-list dictionary to define each sequence.""" results = {} - with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f: + try: + f = open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') + except FileNotFoundError: + return results + with f: all_keys = set(self.keys()) for line in f: try: @@ -1221,9 +1225,8 @@ def get_sequences(self): def set_sequences(self, sequences): """Set sequences using the given name-to-key-list dictionary.""" - f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII') + f = open(os.path.join(self._path, '.mh_sequences'), 'w', encoding='ASCII') try: - os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC)) for name, keys in sequences.items(): if len(keys) == 0: continue diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index caa7eb3d829c68..8c350eb02ccc17 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -1347,6 +1347,19 @@ def test_sequences(self): self._box.remove(key1) self.assertEqual(self._box.get_sequences(), {'flagged':[key0]}) + self._box.set_sequences({'foo':[key0]}) + self.assertEqual(self._box.get_sequences(), {'foo':[key0]}) + + def test_no_dot_mh_sequences_file(self): + path = os.path.join(self._path, 'foo.bar') + os.mkdir(path) + box = self._factory(path) + self.assertEqual(os.listdir(path), []) + self.assertEqual(box.get_sequences(), {}) + self.assertEqual(os.listdir(path), []) + box.set_sequences({}) + self.assertEqual(os.listdir(path), ['.mh_sequences']) + def test_issue2625(self): msg0 = mailbox.MHMessage(self._template % 0) msg0.add_sequence('foo') diff --git a/Misc/NEWS.d/next/Library/2023-04-09-21-05-43.gh-issue-66515.0DS8Ya.rst b/Misc/NEWS.d/next/Library/2023-04-09-21-05-43.gh-issue-66515.0DS8Ya.rst new file mode 100644 index 00000000000000..b9c52f3b8db52c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-09-21-05-43.gh-issue-66515.0DS8Ya.rst @@ -0,0 +1,3 @@ +:class:`mailbox.MH` now supports folders that do not contain a +``.mh_sequences`` file (e.g. Claws Mail IMAP-cache folders). Patch by Serhiy +Storchaka.

1 0

[3.11] gh-101778: Fix build error when there's a dangling symlink in the directory containing "ffi.h" (#113466)
by ronaldoussoren Dec. 26, 2023

Dec. 26, 2023

https://github.com/python/cpython/commit/0eea5c04b8759b6b1aeed25c140c023933… commit: 0eea5c04b8759b6b1aeed25c140c023933264228 branch: 3.11 author: Ronald Oussoren <ronaldoussoren(a)mac.com> committer: ronaldoussoren <ronaldoussoren(a)mac.com> date: 2023-12-26T10:05:48+01:00 summary: [3.11] gh-101778: Fix build error when there's a dangling symlink in the directory containing "ffi.h" (#113466) gh-101778: Fix build error when there's a dangling symlink in the directory containing "ffi.h" files: A Misc/NEWS.d/next/Build/2023-12-25-10-06-59.gh-issue-101778.JfhRkx.rst M setup.py diff --git a/Misc/NEWS.d/next/Build/2023-12-25-10-06-59.gh-issue-101778.JfhRkx.rst b/Misc/NEWS.d/next/Build/2023-12-25-10-06-59.gh-issue-101778.JfhRkx.rst new file mode 100644 index 00000000000000..3763e8404c358d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-12-25-10-06-59.gh-issue-101778.JfhRkx.rst @@ -0,0 +1,2 @@ +Fix build error when there's a dangling symlink in the directory containing +``ffi.h``. diff --git a/setup.py b/setup.py index 4f122b62e0e708..ad8fb81b218ebd 100644 --- a/setup.py +++ b/setup.py @@ -224,6 +224,7 @@ def is_macosx_sdk_path(path): def grep_headers_for(function, headers): for header in headers: + if not os.path.exists(header): continue with open(header, 'r', errors='surrogateescape') as f: if function in f.read(): return True

1 0

[3.12] Misc minor improvements to the itertools recipes (gh-113477) (gh-113478)
by rhettinger Dec. 25, 2023

Dec. 25, 2023

https://github.com/python/cpython/commit/8a95500367dbf3476f57c956331712fdd6… commit: 8a95500367dbf3476f57c956331712fdd64f8146 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: rhettinger <rhettinger(a)users.noreply.github.com> date: 2023-12-25T22:32:05Z summary: [3.12] Misc minor improvements to the itertools recipes (gh-113477) (gh-113478) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 6bcda307f256f2..a0181612d5bb67 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -795,11 +795,11 @@ which incur interpreter overhead. import random def take(n, iterable): - "Return first n items of the iterable as a list" + "Return first n items of the iterable as a list." return list(islice(iterable, n)) def prepend(value, iterable): - "Prepend a single value in front of an iterable" + "Prepend a single value in front of an iterable." # prepend(1, [2, 3, 4]) --> 1 2 3 4 return chain([value], iterable) @@ -817,15 +817,15 @@ which incur interpreter overhead. return starmap(func, repeat(args, times)) def flatten(list_of_lists): - "Flatten one level of nesting" + "Flatten one level of nesting." return chain.from_iterable(list_of_lists) def ncycles(iterable, n): - "Returns the sequence elements n times" + "Returns the sequence elements n times." return chain.from_iterable(repeat(tuple(iterable), n)) def tail(n, iterable): - "Return an iterator over the last n items" + "Return an iterator over the last n items." # tail(3, 'ABCDEFG') --> E F G return iter(collections.deque(iterable, maxlen=n)) @@ -840,7 +840,7 @@ which incur interpreter overhead. next(islice(iterator, n, n), None) def nth(iterable, n, default=None): - "Returns the nth item or a default value" + "Returns the nth item or a default value." return next(islice(iterable, n, None), default) def quantify(iterable, pred=bool): @@ -848,7 +848,7 @@ which incur interpreter overhead. return sum(map(pred, iterable)) def all_equal(iterable): - "Returns True if all the elements are equal to each other" + "Returns True if all the elements are equal to each other." g = groupby(iterable) return next(g, True) and not next(g, False) @@ -865,6 +865,30 @@ which incur interpreter overhead. # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x return next(filter(pred, iterable), default) + def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBcCAD', str.casefold) --> A B c D + seen = set() + if key is None: + for element in filterfalse(seen.__contains__, iterable): + seen.add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen.add(k) + yield element + + def unique_justseen(iterable, key=None): + "List unique elements, preserving order. Remember only the element just seen." + # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B + # unique_justseen('ABBcCAD', str.casefold) --> A B c A D + if key is None: + return map(operator.itemgetter(0), groupby(iterable)) + return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." # iter_index('AABCADEAF', 'A') --> 0 1 4 7 @@ -885,31 +909,17 @@ which incur interpreter overhead. except ValueError: pass - def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead - of a sentinel to end the loop. - - Examples: - iter_except(functools.partial(heappop, h), IndexError) # priority queue iterator - iter_except(d.popitem, KeyError) # non-blocking dict iterator - iter_except(d.popleft, IndexError) # non-blocking deque iterator - iter_except(q.get_nowait, Queue.Empty) # loop over a producer Queue - iter_except(s.pop, KeyError) # non-blocking set iterator - - """ - try: - if first is not None: - yield first() # For database APIs needing an initial cast to db.first() - while True: - yield func() - except exception: - pass + def sliding_window(iterable, n): + "Collect data into overlapping fixed-length chunks or blocks." + # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG + it = iter(iterable) + window = collections.deque(islice(it, n-1), maxlen=n) + for x in it: + window.append(x) + yield tuple(window) def grouper(iterable, n, *, incomplete='fill', fillvalue=None): - "Collect data into non-overlapping fixed-length chunks or blocks" + "Collect data into non-overlapping fixed-length chunks or blocks." # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF @@ -924,16 +934,9 @@ which incur interpreter overhead. case _: raise ValueError('Expected fill, strict, or ignore') - def sliding_window(iterable, n): - # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG - it = iter(iterable) - window = collections.deque(islice(it, n-1), maxlen=n) - for x in it: - window.append(x) - yield tuple(window) - def roundrobin(*iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + "Visit input iterables in a cycle until each is exhausted." + # roundrobin('ABC', 'D', 'EF') --> A D E B F C # Recipe credited to George Sakkis num_active = len(iterables) nexts = cycle(iter(it).__next__ for it in iterables) @@ -956,11 +959,43 @@ which incur interpreter overhead. return filterfalse(pred, t1), filter(pred, t2) def subslices(seq): - "Return all contiguous non-empty subslices of a sequence" + "Return all contiguous non-empty subslices of a sequence." # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(operator.getitem, repeat(seq), slices) + def iter_except(func, exception, first=None): + """ Call a function repeatedly until an exception is raised. + + Converts a call-until-exception interface to an iterator interface. + Like builtins.iter(func, sentinel) but uses an exception instead + of a sentinel to end the loop. + + Priority queue iterator: + iter_except(functools.partial(heappop, h), IndexError) + + Non-blocking dictionary iterator: + iter_except(d.popitem, KeyError) + + Non-blocking deque iterator: + iter_except(d.popleft, IndexError) + + Non-blocking iterator over a producer Queue: + iter_except(q.get_nowait, Queue.Empty) + + Non-blocking set iterator: + iter_except(s.pop, KeyError) + + """ + try: + if first is not None: + # For database APIs needing an initial call to db.first() + yield first() + while True: + yield func() + except exception: + pass + def before_and_after(predicate, it): """ Variant of takewhile() that allows complete access to the remainder of the iterator. @@ -972,12 +1007,12 @@ which incur interpreter overhead. >>> ''.join(remainder) # takewhile() would lose the 'd' 'dEfGhI' - Note that the first iterator must be fully - consumed before the second iterator can - generate valid results. + Note that the true iterator must be fully consumed + before the remainder iterator can generate valid results. """ it = iter(it) transition = [] + def true_iterator(): for elem in it: if predicate(elem): @@ -985,41 +1020,8 @@ which incur interpreter overhead. else: transition.append(elem) return - def remainder_iterator(): - yield from transition - yield from it - return true_iterator(), remainder_iterator() - def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBcCAD', str.lower) --> A B c D - seen = set() - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen.add(element) - yield element - # For order preserving deduplication, - # a faster but non-lazy solution is: - # yield from dict.fromkeys(iterable) - else: - for element in iterable: - k = key(element) - if k not in seen: - seen.add(k) - yield element - # For use cases that allow the last matching element to be returned, - # a faster but non-lazy solution is: - # t1, t2 = tee(iterable) - # yield from dict(zip(map(key, t1), t2)).values() - - def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.lower) --> A B c A D - if key is None: - return map(operator.itemgetter(0), groupby(iterable)) - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + return true_iterator(), chain(transition, it) The following recipes have a more mathematical flavor: @@ -1550,16 +1552,16 @@ The following recipes have a more mathematical flavor: >>> list(unique_everseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) + >>> list(unique_everseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBcCAD', str.lower)) + >>> list(unique_everseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'D'] >>> list(unique_justseen('AAAABBBCCDAABBB')) ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) + >>> list(unique_justseen('ABBCcAD', str.casefold)) ['A', 'B', 'C', 'A', 'D'] - >>> list(unique_justseen('ABBcCAD', str.lower)) + >>> list(unique_justseen('ABBcCAD', str.casefold)) ['A', 'B', 'c', 'A', 'D'] >>> d = dict(a=1, b=2, c=3)

1 0