From webhook-mailer at python.org  Wed Dec  1 07:09:42 2021
From: webhook-mailer at python.org (markshannon)
Date: Wed, 01 Dec 2021 12:09:42 -0000
Subject: [Python-checkins] bpo-45753: Interpreter internal tweaks (GH-29575)
Message-ID: <mailman.4.1638360582.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/49444fb807ecb396462c8e5f547eeb5c6bc5d4de
commit: 49444fb807ecb396462c8e5f547eeb5c6bc5d4de
branch: main
author: Mark Shannon <mark at hotpy.org>
committer: markshannon <mark at hotpy.org>
date: 2021-12-01T12:09:36Z
summary:

bpo-45753: Interpreter internal tweaks (GH-29575)

* Split exit paths into exceptional and non-exceptional.

* Move exit tracing code to individual bytecodes.

* Wrap all trace entry and exit events in macros to make them clearer and easier to enhance.

* Move return sequence into RETURN_VALUE, YIELD_VALUE and YIELD_FROM. Distinguish between normal trace events and dtrace events.

files:
M Include/internal/pycore_code.h
M Include/internal/pycore_frame.h
M Python/ceval.c

diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 194af46a3a274..d4d1392d05bde 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -137,24 +137,25 @@ _GetSpecializedCacheEntryForInstruction(const _Py_CODEUNIT *first_instr, int nex
 #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
 #define QUICKENING_WARMUP_COLDEST 1
 
-static inline void
-PyCodeObject_IncrementWarmup(PyCodeObject * co)
-{
-    co->co_warmup++;
-}
+int _Py_Quicken(PyCodeObject *code);
 
-/* Used by the interpreter to determine when a code object should be quickened */
+/* Returns 1 if quickening occurs.
+ * -1 if an error occurs
+ * 0 otherwise */
 static inline int
-PyCodeObject_IsWarmedUp(PyCodeObject * co)
+_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code)
 {
-    return (co->co_warmup == 0);
+    if (code->co_warmup != 0) {
+        code->co_warmup++;
+        if (code->co_warmup == 0) {
+            return _Py_Quicken(code) ? -1 : 1;
+        }
+    }
+    return 0;
 }
 
-int _Py_Quicken(PyCodeObject *code);
-
 extern Py_ssize_t _Py_QuickenedCount;
 
-
 /* "Locals plus" for a code object is the set of locals + cell vars +
  * free vars.  This relates to variable names as well as offsets into
  * the "fast locals" storage array of execution frames.  The compiler
diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h
index 0015de8f8fcb4..b0e51a6ddc461 100644
--- a/Include/internal/pycore_frame.h
+++ b/Include/internal/pycore_frame.h
@@ -19,6 +19,11 @@ enum _framestate {
 
 typedef signed char PyFrameState;
 
+/*
+    frame->f_lasti refers to the index of the last instruction,
+    unless it's -1 in which case next_instr should be first_instr.
+*/
+
 typedef struct _interpreter_frame {
     PyFunctionObject *f_func; /* Strong reference */
     PyObject *f_globals; /* Borrowed reference */
diff --git a/Python/ceval.c b/Python/ceval.c
index c5477b30f7e56..97c684479abdc 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -102,8 +102,8 @@ static InterpreterFrame *
 _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
                         PyObject *locals, PyObject* const* args,
                         size_t argcount, PyObject *kwnames);
-static int
-_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame);
+static void
+_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame *frame);
 
 #define NAME_ERROR_MSG \
     "name '%.200s' is not defined"
@@ -1390,7 +1390,7 @@ eval_frame_handle_pending(PyThreadState *tstate)
 
 /* Get opcode and oparg from original instructions, not quickened form. */
 #define TRACING_NEXTOPARG() do { \
-        _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(co->co_code))[INSTR_OFFSET()]; \
+        _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code))[INSTR_OFFSET()]; \
         opcode = _Py_OPCODE(word); \
         oparg = _Py_OPARG(word); \
     } while (0)
@@ -1462,20 +1462,20 @@ eval_frame_handle_pending(PyThreadState *tstate)
 #ifdef LLTRACE
 #define PUSH(v)         { (void)(BASIC_PUSH(v), \
                           lltrace && prtrace(tstate, TOP(), "push")); \
-                          assert(STACK_LEVEL() <= co->co_stacksize); }
+                          assert(STACK_LEVEL() <= frame->f_code->co_stacksize); }
 #define POP()           ((void)(lltrace && prtrace(tstate, TOP(), "pop")), \
                          BASIC_POP())
 #define STACK_GROW(n)   do { \
                           assert(n >= 0); \
                           (void)(BASIC_STACKADJ(n), \
                           lltrace && prtrace(tstate, TOP(), "stackadj")); \
-                          assert(STACK_LEVEL() <= co->co_stacksize); \
+                          assert(STACK_LEVEL() <= frame->f_code->co_stacksize); \
                         } while (0)
 #define STACK_SHRINK(n) do { \
                             assert(n >= 0); \
                             (void)(lltrace && prtrace(tstate, TOP(), "stackadj")); \
                             (void)(BASIC_STACKADJ(-(n))); \
-                            assert(STACK_LEVEL() <= co->co_stacksize); \
+                            assert(STACK_LEVEL() <= frame->f_code->co_stacksize); \
                         } while (0)
 #define EXT_POP(STACK_POINTER) ((void)(lltrace && \
                                 prtrace(tstate, (STACK_POINTER)[-1], "ext_pop")), \
@@ -1537,6 +1537,40 @@ eval_frame_handle_pending(PyThreadState *tstate)
     STAT_INC(LOAD_##attr_or_method, hit); \
     Py_INCREF(res);
 
+#define TRACE_FUNCTION_EXIT() \
+    if (cframe.use_tracing) { \
+        if (trace_function_exit(tstate, frame, retval)) { \
+            Py_DECREF(retval); \
+            goto exit_unwind; \
+        } \
+    }
+
+#define DTRACE_FUNCTION_EXIT() \
+    if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \
+        dtrace_function_return(frame); \
+    }
+
+#define TRACE_FUNCTION_UNWIND()  \
+    if (cframe.use_tracing) { \
+        /* Since we are already unwinding, \
+         * we dont't care if this raises */ \
+        trace_function_exit(tstate, frame, NULL); \
+    }
+
+#define TRACE_FUNCTION_ENTRY() \
+    if (cframe.use_tracing) { \
+        if (trace_function_entry(tstate, frame)) { \
+            goto exit_unwind; \
+        } \
+    }
+
+#define DTRACE_FUNCTION_ENTRY()  \
+    if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \
+        dtrace_function_entry(frame); \
+    }
+
+
+
 static int
 trace_function_entry(PyThreadState *tstate, InterpreterFrame *frame)
 {
@@ -1576,6 +1610,24 @@ trace_function_entry(PyThreadState *tstate, InterpreterFrame *frame)
     return 0;
 }
 
+static int
+trace_function_exit(PyThreadState *tstate, InterpreterFrame *frame, PyObject *retval)
+{
+    if (tstate->c_tracefunc) {
+        if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj,
+                                    tstate, frame, PyTrace_RETURN, retval)) {
+            return -1;
+        }
+    }
+    if (tstate->c_profilefunc) {
+        if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj,
+                                    tstate, frame, PyTrace_RETURN, retval)) {
+            return -1;
+        }
+    }
+    return 0;
+}
+
 static PyObject *
 make_coro(PyThreadState *tstate, PyFunctionObject *func,
           PyObject *locals,
@@ -1583,7 +1635,8 @@ make_coro(PyThreadState *tstate, PyFunctionObject *func,
           PyObject *kwnames);
 
 static int
-skip_backwards_over_extended_args(PyCodeObject *code, int offset) {
+skip_backwards_over_extended_args(PyCodeObject *code, int offset)
+{
     _Py_CODEUNIT *instrs = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code);
     while (offset > 0 && _Py_OPCODE(instrs[offset-1]) == EXTENDED_ARG) {
         offset--;
@@ -1591,6 +1644,14 @@ skip_backwards_over_extended_args(PyCodeObject *code, int offset) {
     return offset;
 }
 
+static InterpreterFrame *
+pop_frame(PyThreadState *tstate, InterpreterFrame *frame)
+{
+    InterpreterFrame *prev_frame = frame->previous;
+    _PyEvalFrameClearAndPop(tstate, frame);
+    return prev_frame;
+}
+
 PyObject* _Py_HOT_FUNCTION
 _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int throwflag)
 {
@@ -1606,7 +1667,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
 #endif
     int opcode;        /* Current opcode */
     int oparg;         /* Current opcode argument, if any */
-    PyObject *retval = NULL;            /* Return value */
     _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
 
     CFrame cframe;
@@ -1625,82 +1685,77 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
     frame->previous = prev_cframe->current_frame;
     cframe.current_frame = frame;
 
+    /* support for generator.throw() */
+    if (throwflag) {
+        if (_Py_EnterRecursiveCall(tstate, "")) {
+            tstate->recursion_remaining--;
+            goto exit_unwind;
+        }
+        TRACE_FUNCTION_ENTRY();
+        DTRACE_FUNCTION_ENTRY();
+        goto resume_with_error;
+    }
+
+    /* Local "register" variables.
+     * These are cached values from the frame and code object.  */
+
+    PyObject *names;
+    PyObject *consts;
+    _Py_CODEUNIT *first_instr;
+    _Py_CODEUNIT *next_instr;
+    PyObject **stack_pointer;
+
+/* Sets the above local variables from the frame */
+#define SET_LOCALS_FROM_FRAME() \
+    { \
+        PyCodeObject *co = frame->f_code; \
+        names = co->co_names; \
+        consts = co->co_consts; \
+        first_instr = co->co_firstinstr; \
+    } \
+    assert(frame->f_lasti >= -1); \
+    next_instr = first_instr + frame->f_lasti + 1; \
+    stack_pointer = _PyFrame_GetStackPointer(frame); \
+    /* Set stackdepth to -1. \
+        Update when returning or calling trace function. \
+        Having stackdepth <= 0 ensures that invalid \
+        values are not visible to the cycle GC. \
+        We choose -1 rather than 0 to assist debugging. \
+        */ \
+    frame->stacktop = -1;
+
+
 start_frame:
     if (_Py_EnterRecursiveCall(tstate, "")) {
         tstate->recursion_remaining--;
-        goto exit_eval_frame;
+        goto exit_unwind;
     }
 
     assert(tstate->cframe == &cframe);
     assert(frame == cframe.current_frame);
 
-    if (cframe.use_tracing) {
-        if (trace_function_entry(tstate, frame)) {
-            goto exit_eval_frame;
-        }
-    }
-
-    if (PyDTrace_FUNCTION_ENTRY_ENABLED())
-        dtrace_function_entry(frame);
+    TRACE_FUNCTION_ENTRY();
+    DTRACE_FUNCTION_ENTRY();
 
-    PyCodeObject *co = frame->f_code;
-    /* Increment the warmup counter and quicken if warm enough
-     * _Py_Quicken is idempotent so we don't worry about overflow */
-    if (!PyCodeObject_IsWarmedUp(co)) {
-        PyCodeObject_IncrementWarmup(co);
-        if (PyCodeObject_IsWarmedUp(co)) {
-            if (_Py_Quicken(co)) {
-                goto exit_eval_frame;
-            }
-        }
+    if (_Py_IncrementCountAndMaybeQuicken(frame->f_code) < 0) {
+        goto exit_unwind;
     }
+    frame->f_state = FRAME_EXECUTING;
 
 resume_frame:
-    co = frame->f_code;
-    PyObject *names = co->co_names;
-    PyObject *consts = co->co_consts;
-    _Py_CODEUNIT *first_instr = co->co_firstinstr;
-    /*
-       frame->f_lasti refers to the index of the last instruction,
-       unless it's -1 in which case next_instr should be first_instr.
-
-       YIELD_FROM sets frame->f_lasti to itself, in order to repeatedly yield
-       multiple values.
-
-       When the PREDICT() macros are enabled, some opcode pairs follow in
-       direct succession. A successful prediction effectively links the two
-       codes together as if they were a single new opcode, but the value
-       of frame->f_lasti is correctly updated so potential inlined calls
-       or lookups of frame->f_lasti are aways correct when the macros are used.
-    */
-    assert(frame->f_lasti >= -1);
-    _Py_CODEUNIT *next_instr = first_instr + frame->f_lasti + 1;
-    PyObject **stack_pointer = _PyFrame_GetStackPointer(frame);
-    /* Set stackdepth to -1.
-     * Update when returning or calling trace function.
-       Having stackdepth <= 0 ensures that invalid
-       values are not visible to the cycle GC.
-       We choose -1 rather than 0 to assist debugging.
-     */
-    frame->stacktop = -1;
-    frame->f_state = FRAME_EXECUTING;
+    SET_LOCALS_FROM_FRAME();
 
 #ifdef LLTRACE
     _Py_IDENTIFIER(__ltrace__);
     {
         int r = _PyDict_ContainsId(GLOBALS(), &PyId___ltrace__);
         if (r < 0) {
-            goto exit_eval_frame;
+            goto exit_unwind;
         }
         lltrace = r;
     }
 #endif
 
-    if (throwflag) { /* support for generator.throw() */
-        throwflag = 0;
-        goto error;
-    }
-
 #ifdef Py_DEBUG
     /* _PyEval_EvalFrameDefault() must not be called with an exception set,
        because it can clear it (directly or indirectly) and so the
@@ -1711,7 +1766,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
 check_eval_breaker:
     {
         assert(STACK_LEVEL() >= 0); /* else underflow */
-        assert(STACK_LEVEL() <= co->co_stacksize);  /* else overflow */
+        assert(STACK_LEVEL() <= frame->f_code->co_stacksize);  /* else overflow */
         assert(!_PyErr_Occurred(tstate));
 
         /* Do periodic things.  Doing this every time through
@@ -2418,11 +2473,24 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
         }
 
         TARGET(RETURN_VALUE) {
-            retval = POP();
+            PyObject *retval = POP();
             assert(EMPTY());
             frame->f_state = FRAME_RETURNED;
             _PyFrame_SetStackPointer(frame, stack_pointer);
-            goto exiting;
+            TRACE_FUNCTION_EXIT();
+            DTRACE_FUNCTION_EXIT();
+            _Py_LeaveRecursiveCall(tstate);
+            if (frame->depth) {
+                frame = cframe.current_frame = pop_frame(tstate, frame);
+                _PyFrame_StackPush(frame, retval);
+                goto resume_frame;
+            }
+            /* Restore previous cframe and return. */
+            tstate->cframe = cframe.previous;
+            tstate->cframe->use_tracing = cframe.use_tracing;
+            assert(tstate->cframe->current_frame == frame->previous);
+            assert(!_PyErr_Occurred(tstate));
+            return retval;
         }
 
         TARGET(GET_AITER) {
@@ -2562,9 +2630,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
         }
 
         TARGET(YIELD_FROM) {
+            assert(frame->depth == 0);
             PyObject *v = POP();
             PyObject *receiver = TOP();
             PySendResult gen_status;
+            PyObject *retval;
             if (tstate->c_tracefunc == NULL) {
                 gen_status = PyIter_Send(receiver, v, &retval);
             } else {
@@ -2610,13 +2680,22 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
             frame->f_lasti -= 1;
             frame->f_state = FRAME_SUSPENDED;
             _PyFrame_SetStackPointer(frame, stack_pointer);
-            goto exiting;
+            TRACE_FUNCTION_EXIT();
+            DTRACE_FUNCTION_EXIT();
+            _Py_LeaveRecursiveCall(tstate);
+            /* Restore previous cframe and return. */
+            tstate->cframe = cframe.previous;
+            tstate->cframe->use_tracing = cframe.use_tracing;
+            assert(tstate->cframe->current_frame == frame->previous);
+            assert(!_PyErr_Occurred(tstate));
+            return retval;
         }
 
         TARGET(YIELD_VALUE) {
-            retval = POP();
+            assert(frame->depth == 0);
+            PyObject *retval = POP();
 
-            if (co->co_flags & CO_ASYNC_GENERATOR) {
+            if (frame->f_code->co_flags & CO_ASYNC_GENERATOR) {
                 PyObject *w = _PyAsyncGenValueWrapperNew(retval);
                 Py_DECREF(retval);
                 if (w == NULL) {
@@ -2627,7 +2706,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
             }
             frame->f_state = FRAME_SUSPENDED;
             _PyFrame_SetStackPointer(frame, stack_pointer);
-            goto exiting;
+            TRACE_FUNCTION_EXIT();
+            DTRACE_FUNCTION_EXIT();
+            _Py_LeaveRecursiveCall(tstate);
+            /* Restore previous cframe and return. */
+            tstate->cframe = cframe.previous;
+            tstate->cframe->use_tracing = cframe.use_tracing;
+            assert(tstate->cframe->current_frame == frame->previous);
+            assert(!_PyErr_Occurred(tstate));
+            return retval;
         }
 
         TARGET(GEN_START) {
@@ -3100,7 +3187,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
             format_exc_check_arg(
                 tstate, PyExc_UnboundLocalError,
                 UNBOUNDLOCAL_ERROR_MSG,
-                PyTuple_GetItem(co->co_localsplusnames, oparg)
+                PyTuple_GetItem(frame->f_code->co_localsplusnames, oparg)
                 );
             goto error;
         }
@@ -3125,15 +3212,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
                 Py_DECREF(oldobj);
                 DISPATCH();
             }
-            format_exc_unbound(tstate, co, oparg);
+            format_exc_unbound(tstate, frame->f_code, oparg);
             goto error;
         }
 
         TARGET(LOAD_CLASSDEREF) {
             PyObject *name, *value, *locals = LOCALS();
             assert(locals);
-            assert(oparg >= 0 && oparg < co->co_nlocalsplus);
-            name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg);
+            assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus);
+            name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg);
             if (PyDict_CheckExact(locals)) {
                 value = PyDict_GetItemWithError(locals, name);
                 if (value != NULL) {
@@ -3156,7 +3243,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
                 PyObject *cell = GETLOCAL(oparg);
                 value = PyCell_GET(cell);
                 if (value == NULL) {
-                    format_exc_unbound(tstate, co, oparg);
+                    format_exc_unbound(tstate, frame->f_code, oparg);
                     goto error;
                 }
                 Py_INCREF(value);
@@ -3169,7 +3256,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
             PyObject *cell = GETLOCAL(oparg);
             PyObject *value = PyCell_GET(cell);
             if (value == NULL) {
-                format_exc_unbound(tstate, co, oparg);
+                format_exc_unbound(tstate, frame->f_code, oparg);
                 goto error;
             }
             Py_INCREF(value);
@@ -3918,18 +4005,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
         TARGET(JUMP_ABSOLUTE) {
             PREDICTED(JUMP_ABSOLUTE);
             assert(oparg < INSTR_OFFSET());
-            /* Increment the warmup counter and quicken if warm enough
-            * _Py_Quicken is idempotent so we don't worry about overflow */
-            if (!PyCodeObject_IsWarmedUp(co)) {
-                PyCodeObject_IncrementWarmup(co);
-                if (PyCodeObject_IsWarmedUp(co)) {
-                    if (_Py_Quicken(co)) {
-                        goto error;
-                    }
-                    int nexti = INSTR_OFFSET();
-                    first_instr = co->co_firstinstr;
-                    next_instr = first_instr + nexti;
+            int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code);
+            if (err) {
+                if (err < 0) {
+                    goto error;
                 }
+                /* Update first_instr and next_instr to point to newly quickened code */
+                int nexti = INSTR_OFFSET();
+                first_instr = frame->f_code->co_firstinstr;
+                next_instr = first_instr + nexti;
             }
             JUMPTO(oparg);
             CHECK_EVAL_BREAKER();
@@ -4036,7 +4120,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
             PyObject *iter;
             if (PyCoro_CheckExact(iterable)) {
                 /* `iterable` is a coroutine */
-                if (!(co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
+                if (!(frame->f_code->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
                     /* and it is used in a 'yield from' expression of a
                        regular generator. */
                     Py_DECREF(iterable);
@@ -4906,7 +4990,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
 #else
         case DO_TRACING: {
 #endif
-            int instr_prev = skip_backwards_over_extended_args(co, frame->f_lasti);
+            int instr_prev = skip_backwards_over_extended_args(frame->f_code, frame->f_lasti);
             frame->f_lasti = INSTR_OFFSET();
             TRACING_NEXTOPARG();
             if (PyDTrace_LINE_ENABLED()) {
@@ -5016,7 +5100,7 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
         {
             format_exc_check_arg(tstate, PyExc_UnboundLocalError,
                 UNBOUNDLOCAL_ERROR_MSG,
-                PyTuple_GetItem(co->co_localsplusnames, oparg)
+                PyTuple_GetItem(frame->f_code->co_localsplusnames, oparg)
             );
             goto error;
         }
@@ -5039,8 +5123,7 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
         }
 
         if (tstate->c_tracefunc != NULL) {
-            /* Make sure state is set to FRAME_EXECUTING for tracing */
-            assert(frame->f_state == FRAME_EXECUTING);
+            /* Make sure state is set to FRAME_UNWINDING for tracing */
             frame->f_state = FRAME_UNWINDING;
             call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj,
                            tstate, frame);
@@ -5051,9 +5134,8 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
         /* We can't use frame->f_lasti here, as RERAISE may have set it */
         int offset = INSTR_OFFSET()-1;
         int level, handler, lasti;
-        if (get_exception_handler(co, offset, &level, &handler, &lasti) == 0) {
+        if (get_exception_handler(frame->f_code, offset, &level, &handler, &lasti) == 0) {
             // No handlers, so exit.
-            assert(retval == NULL);
             assert(_PyErr_Occurred(tstate));
 
             /* Pop remaining stack entries. */
@@ -5065,7 +5147,9 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
             assert(STACK_LEVEL() == 0);
             _PyFrame_SetStackPointer(frame, stack_pointer);
             frame->f_state = FRAME_RAISED;
-            goto exiting;
+            TRACE_FUNCTION_UNWIND();
+            DTRACE_FUNCTION_EXIT();
+            goto exit_unwind;
         }
 
         assert(STACK_LEVEL() >= level);
@@ -5106,48 +5190,22 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
         DISPATCH();
     }
 
-exiting:
-    if (cframe.use_tracing) {
-        if (tstate->c_tracefunc) {
-            if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj,
-                                     tstate, frame, PyTrace_RETURN, retval)) {
-                Py_CLEAR(retval);
-            }
-        }
-        if (tstate->c_profilefunc) {
-            if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj,
-                                     tstate, frame, PyTrace_RETURN, retval)) {
-                Py_CLEAR(retval);
-            }
-        }
-    }
-
-    /* pop frame */
-exit_eval_frame:
-    if (PyDTrace_FUNCTION_RETURN_ENABLED())
-        dtrace_function_return(frame);
+exit_unwind:
+    assert(_PyErr_Occurred(tstate));
     _Py_LeaveRecursiveCall(tstate);
-
-    if (frame->depth) {
-        cframe.current_frame = frame->previous;
-        _PyFrame_StackPush(cframe.current_frame, retval);
-        if (_PyEvalFrameClearAndPop(tstate, frame)) {
-            retval = NULL;
-        }
-        frame = cframe.current_frame;
-        if (retval == NULL) {
-            assert(_PyErr_Occurred(tstate));
-            throwflag = 1;
-        }
-        retval = NULL;
-        goto resume_frame;
+    if (frame->depth == 0) {
+        /* Restore previous cframe and exit */
+        tstate->cframe = cframe.previous;
+        tstate->cframe->use_tracing = cframe.use_tracing;
+        assert(tstate->cframe->current_frame == frame->previous);
+        return NULL;
     }
+    frame = cframe.current_frame = pop_frame(tstate, frame);
+
+resume_with_error:
+    SET_LOCALS_FROM_FRAME();
+    goto error;
 
-    /* Restore previous cframe. */
-    tstate->cframe = cframe.previous;
-    tstate->cframe->use_tracing = cframe.use_tracing;
-    assert(tstate->cframe->current_frame == frame->previous);
-    return _Py_CheckFunctionResult(tstate, NULL, retval, __func__);
 }
 
 static void
@@ -5770,15 +5828,14 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
     return NULL;
 }
 
-static int
+static void
 _PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame)
 {
-    --tstate->recursion_remaining;
+    tstate->recursion_remaining--;
     assert(frame->frame_obj == NULL || frame->frame_obj->f_owns_frame == 0);
     _PyFrame_Clear(frame);
-    ++tstate->recursion_remaining;
+    tstate->recursion_remaining++;
     _PyThreadState_PopFrame(tstate, frame);
-    return 0;
 }
 
 PyObject *
@@ -5812,9 +5869,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
     }
     PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0);
     assert(_PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame));
-    if (_PyEvalFrameClearAndPop(tstate, frame)) {
-        retval = NULL;
-    }
+    _PyEvalFrameClearAndPop(tstate, frame);
     return retval;
 }
 


From webhook-mailer at python.org  Wed Dec  1 07:24:55 2021
From: webhook-mailer at python.org (asvetlov)
Date: Wed, 01 Dec 2021 12:24:55 -0000
Subject: [Python-checkins] bpo-45896: Fix docs default asyncio event loop on
 Windows (GH-29857)
Message-ID: <mailman.5.1638361496.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/f27bef30438d2f07f19de91e021f34b77ccc4b20
commit: f27bef30438d2f07f19de91e021f34b77ccc4b20
branch: main
author: Rob <r4226 at pm.me>
committer: asvetlov <andrew.svetlov at gmail.com>
date: 2021-12-01T14:24:46+02:00
summary:

bpo-45896: Fix docs default asyncio event loop on Windows (GH-29857)

files:
M Doc/library/asyncio-eventloop.rst

diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index 5a3369415a745..497128ee17f37 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -1243,9 +1243,10 @@ async/await code consider using the high-level
 
 .. note::
 
-   The default asyncio event loop on **Windows** does not support
-   subprocesses. See :ref:`Subprocess Support on Windows
-   <asyncio-windows-subprocess>` for details.
+   On Windows, the default event loop :class:`ProactorEventLoop` supports
+   subprocesses, whereas :class:`SelectorEventLoop` does not. See
+   :ref:`Subprocess Support on Windows <asyncio-windows-subprocess>` for
+   details.
 
 .. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \
                       stdin=subprocess.PIPE, stdout=subprocess.PIPE, \


From webhook-mailer at python.org  Wed Dec  1 10:40:59 2021
From: webhook-mailer at python.org (asvetlov)
Date: Wed, 01 Dec 2021 15:40:59 -0000
Subject: [Python-checkins] bpo-45896: Fix docs default asyncio event loop on
 Windows (GH-29857) (GH-29878)
Message-ID: <mailman.6.1638373261.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/4203a5d1918ca874e305806b787e3c8c6fc35e3e
commit: 4203a5d1918ca874e305806b787e3c8c6fc35e3e
branch: 3.9
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: asvetlov <andrew.svetlov at gmail.com>
date: 2021-12-01T17:40:50+02:00
summary:

bpo-45896: Fix docs default asyncio event loop on Windows (GH-29857) (GH-29878)

(cherry picked from commit f27bef30438d2f07f19de91e021f34b77ccc4b20)

Co-authored-by: Rob <r4226 at pm.me>

Co-authored-by: Rob <r4226 at pm.me>

files:
M Doc/library/asyncio-eventloop.rst

diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index d70aeae766ae3..73799d005c157 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -1239,9 +1239,10 @@ async/await code consider using the high-level
 
 .. note::
 
-   The default asyncio event loop on **Windows** does not support
-   subprocesses. See :ref:`Subprocess Support on Windows
-   <asyncio-windows-subprocess>` for details.
+   On Windows, the default event loop :class:`ProactorEventLoop` supports
+   subprocesses, whereas :class:`SelectorEventLoop` does not. See
+   :ref:`Subprocess Support on Windows <asyncio-windows-subprocess>` for
+   details.
 
 .. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \
                       stdin=subprocess.PIPE, stdout=subprocess.PIPE, \


From webhook-mailer at python.org  Wed Dec  1 10:44:18 2021
From: webhook-mailer at python.org (asvetlov)
Date: Wed, 01 Dec 2021 15:44:18 -0000
Subject: [Python-checkins] bpo-45896: Fix docs default asyncio event loop on
 Windows (GH-29857) (GH-29877)
Message-ID: <mailman.7.1638373459.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/e99c5e039b380199843db4e06974883d9f3ddad0
commit: e99c5e039b380199843db4e06974883d9f3ddad0
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: asvetlov <andrew.svetlov at gmail.com>
date: 2021-12-01T17:44:09+02:00
summary:

bpo-45896: Fix docs default asyncio event loop on Windows (GH-29857) (GH-29877)

(cherry picked from commit f27bef30438d2f07f19de91e021f34b77ccc4b20)

Co-authored-by: Rob <r4226 at pm.me>

files:
M Doc/library/asyncio-eventloop.rst

diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index 44798dc3e23cb..252fb426e5b39 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -1247,9 +1247,10 @@ async/await code consider using the high-level
 
 .. note::
 
-   The default asyncio event loop on **Windows** does not support
-   subprocesses. See :ref:`Subprocess Support on Windows
-   <asyncio-windows-subprocess>` for details.
+   On Windows, the default event loop :class:`ProactorEventLoop` supports
+   subprocesses, whereas :class:`SelectorEventLoop` does not. See
+   :ref:`Subprocess Support on Windows <asyncio-windows-subprocess>` for
+   details.
 
 .. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \
                       stdin=subprocess.PIPE, stdout=subprocess.PIPE, \


From webhook-mailer at python.org  Wed Dec  1 13:20:37 2021
From: webhook-mailer at python.org (ericsnowcurrently)
Date: Wed, 01 Dec 2021 18:20:37 -0000
Subject: [Python-checkins] bpo-45952: Get the C analyzer tool working again.
 (gh-29882)
Message-ID: <mailman.8.1638382838.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/ee94aa0850191712e6adfc1f4a9df08ec3240195
commit: ee94aa0850191712e6adfc1f4a9df08ec3240195
branch: main
author: Eric Snow <ericsnowcurrently at gmail.com>
committer: ericsnowcurrently <ericsnowcurrently at gmail.com>
date: 2021-12-01T11:20:20-07:00
summary:

bpo-45952: Get the C analyzer tool working again. (gh-29882)

There wasn't much that needed to be done. Mostly it was just a few new files that got added.

https://bugs.python.org/issue45952

files:
M Tools/c-analyzer/TODO
M Tools/c-analyzer/c_parser/__init__.py
M Tools/c-analyzer/c_parser/parser/__init__.py
M Tools/c-analyzer/c_parser/preprocessor/__main__.py
M Tools/c-analyzer/cpython/__main__.py
M Tools/c-analyzer/cpython/_parser.py

diff --git a/Tools/c-analyzer/TODO b/Tools/c-analyzer/TODO
index 1fd8052268be0..4b9b2857e1d1e 100644
--- a/Tools/c-analyzer/TODO
+++ b/Tools/c-analyzer/TODO
@@ -1,3 +1,11 @@
+# For up-to-date results, run:
+#   ./python Tools/c-analyzer/c-analyzer.py check --format summary
+# or
+#   ./python Tools/c-analyzer/c-analyzer.py analyze
+
+
+#######################################
+# non-PyObject (61)
 
 # allocator (16)
 Objects/obmalloc.c:_PyMem                                        static PyMemAllocatorEx _PyMem
@@ -32,12 +40,7 @@ Objects/dictobject.c:empty_keys_struct                           static PyDictKe
 Python/fileutils.c:_Py_open_cloexec_works                        int _Py_open_cloexec_works
 
 
-# freelists
-Objects/dictobject.c:keys_free_list                              static PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]
-Objects/dictobject.c:numfreekeys                                 static int numfreekeys
-
-
-# other non-object (43)
+# other non-object (40)
 Modules/_tracemalloc.c:allocators                                static struct { PyMemAllocatorEx mem; PyMemAllocatorEx raw; PyMemAllocatorEx obj; } allocators
 Modules/_tracemalloc.c:tables_lock                               static PyThread_type_lock tables_lock
 Modules/_tracemalloc.c:tracemalloc_filenames                     static _Py_hashtable_t *tracemalloc_filenames
@@ -81,30 +84,7 @@ Python/pylifecycle.c:fatal_error():reentrant                     static int reen
 
 
 #######################################
-# PyObject (960)
-
-# freelists (10 + 10)
-Modules/_collectionsmodule.c:freeblocks                          static block *freeblocks[MAXFREEBLOCKS]
-Modules/_collectionsmodule.c:numfreeblocks                       static Py_ssize_t numfreeblocks
-Objects/dictobject.c:free_list                                   static PyDictObject *free_list[PyDict_MAXFREELIST]
-Objects/dictobject.c:numfree                                     static int numfree
-Objects/exceptions.c:memerrors_freelist                          static PyBaseExceptionObject *memerrors_freelist
-Objects/exceptions.c:memerrors_numfree                           static int memerrors_numfree
-Objects/floatobject.c:free_list                                  static PyFloatObject *free_list
-Objects/floatobject.c:numfree                                    static int numfree
-Objects/frameobject.c:free_list                                  static PyFrameObject *free_list
-Objects/frameobject.c:numfree                                    static int numfree
-Objects/genobject.c:ag_asend_freelist                            static PyAsyncGenASend *ag_asend_freelist[_PyAsyncGen_MAXFREELIST]
-Objects/genobject.c:ag_asend_freelist_free                       static int ag_asend_freelist_free
-Objects/genobject.c:ag_value_freelist                            static _PyAsyncGenWrappedValue *ag_value_freelist[_PyAsyncGen_MAXFREELIST]
-Objects/genobject.c:ag_value_freelist_free                       static int ag_value_freelist_free
-Objects/listobject.c:free_list                                   static PyListObject *free_list[PyList_MAXFREELIST]
-Objects/listobject.c:numfree                                     static int numfree
-Objects/tupleobject.c:free_list                                  static PyTupleObject *free_list[PyTuple_MAXSAVESIZE]
-Objects/tupleobject.c:numfree                                    static int numfree[PyTuple_MAXSAVESIZE]
-Python/context.c:ctx_freelist                                    static PyContext *ctx_freelist
-Python/context.c:ctx_freelist_len                                static int ctx_freelist_len
-
+# PyObject (919)
 
 # singletons (7)
 Objects/boolobject.c:_Py_FalseStruct                             static struct _longobject _Py_FalseStruct
@@ -116,16 +96,8 @@ Objects/object.c:_Py_NotImplementedStruct                        PyObject _Py_No
 Objects/sliceobject.c:_Py_EllipsisObject                         PyObject _Py_EllipsisObject
 
 
-# module vars (9)
-Modules/_functoolsmodule.c:kwd_mark                              static PyObject *kwd_mark
-Modules/_localemodule.c:Error                                    static PyObject *Error
-Modules/_threadmodule.c:ThreadError                              static PyObject *ThreadError
+# module vars (1)
 Modules/_tracemalloc.c:unknown_filename                          static PyObject *unknown_filename
-Modules/signalmodule.c:DefaultHandler                            static PyObject *DefaultHandler
-Modules/signalmodule.c:IgnoreHandler                             static PyObject *IgnoreHandler
-Modules/signalmodule.c:IntHandler                                static PyObject *IntHandler
-Modules/signalmodule.c:ItimerError                               static PyObject *ItimerError
-Objects/exceptions.c:errnomap                                    static PyObject *errnomap
 
 
 # other (non-cache) (5)
@@ -136,26 +108,15 @@ Modules/signalmodule.c:Handlers                                  static volatile
 Objects/setobject.c:_dummy_struct                                static PyObject _dummy_struct
 
 
-# caches (5)
-Modules/posixmodule.c:posix_putenv_garbage                       static PyObject *posix_putenv_garbage
-Objects/sliceobject.c:slice_cache                                static PySliceObject *slice_cache
-Objects/typeobject.c:method_cache                                static struct method_cache_entry method_cache[1 << MCACHE_SIZE_EXP]
-Objects/unicodeobject.c:interned                                 static PyObject *interned
+# caches (1)
 Python/import.c:extensions                                       static PyObject *extensions
 
 
-# cached constants - non-str (15)
+# cached constants - non-str (6)
 Modules/_io/_iomodule.c:_PyIO_empty_bytes                        PyObject *_PyIO_empty_bytes
 Modules/_io/bufferedio.c:_PyIO_trap_eintr():eintr_int            static PyObject *eintr_int
-Modules/posixmodule.c:billion                                    static PyObject *billion
-Modules/posixmodule.c:wait_helper():struct_rusage                static PyObject *struct_rusage
-Objects/bytesobject.c:characters                                 static PyBytesObject *characters[UCHAR_MAX + 1]
-Objects/bytesobject.c:nullstring                                 static PyBytesObject *nullstring
-Objects/codeobject.c:PyCode_NewEmpty():nulltuple                 static PyObject *nulltuple
-Objects/dictobject.c:empty_values                                static PyObject *empty_values[1]
+Objects/dictobject.c:empty_values_struct                         static PyDictValues
 Objects/listobject.c:indexerr                                    static PyObject *indexerr
-Objects/longobject.c:small_ints                                  static PyLongObject small_ints[NSMALLNEGINTS + NSMALLPOSINTS]
-Objects/setobject.c:emptyfrozenset                               static PyObject *emptyfrozenset
 Python/context.c:_token_missing                                  static PyObject *_token_missing
 Python/hamt.c:_empty_hamt                                        static PyHamtObject *_empty_hamt
 
@@ -662,15 +623,6 @@ Modules/itertoolsmodule.c:takewhile_type                         static PyTypeOb
 Modules/itertoolsmodule.c:tee_type                               static PyTypeObject tee_type
 Modules/itertoolsmodule.c:teedataobject_type                     static PyTypeObject teedataobject_type
 Modules/itertoolsmodule.c:ziplongest_type                        static PyTypeObject ziplongest_type
-Modules/posixmodule.c:DirEntryType                               static PyTypeObject DirEntryType
-Modules/posixmodule.c:ScandirIteratorType                        static PyTypeObject ScandirIteratorType
-Modules/posixmodule.c:SchedParamType                             static PyTypeObject* SchedParamType
-Modules/posixmodule.c:StatResultType                             static PyTypeObject* StatResultType
-Modules/posixmodule.c:StatVFSResultType                          static PyTypeObject* StatVFSResultType
-Modules/posixmodule.c:TerminalSizeType                           static PyTypeObject* TerminalSizeType
-Modules/posixmodule.c:TimesResultType                            static PyTypeObject* TimesResultType
-Modules/posixmodule.c:UnameResultType                            static PyTypeObject* UnameResultType
-Modules/posixmodule.c:WaitidResultType                           static PyTypeObject* WaitidResultType
 Modules/signalmodule.c:SiginfoType                               static PyTypeObject SiginfoType
 Modules/timemodule.c:StructTimeType                              static PyTypeObject StructTimeType
 Modules/xxsubtype.c:spamdict_type                                static PyTypeObject spamdict_type
diff --git a/Tools/c-analyzer/c_parser/__init__.py b/Tools/c-analyzer/c_parser/__init__.py
index 39455ddbf1a0c..fc10aff94505d 100644
--- a/Tools/c-analyzer/c_parser/__init__.py
+++ b/Tools/c-analyzer/c_parser/__init__.py
@@ -1,3 +1,4 @@
+from c_common.fsutil import match_glob as _match_glob
 from .parser import parse as _parse
 from .preprocessor import get_preprocessor as _get_preprocessor
 
@@ -5,23 +6,32 @@
 def parse_file(filename, *,
                match_kind=None,
                get_file_preprocessor=None,
+               file_maxsizes=None,
                ):
     if get_file_preprocessor is None:
         get_file_preprocessor = _get_preprocessor()
-    yield from _parse_file(filename, match_kind, get_file_preprocessor)
+    yield from _parse_file(
+            filename, match_kind, get_file_preprocessor, file_maxsizes)
 
 
 def parse_files(filenames, *,
                 match_kind=None,
                 get_file_preprocessor=None,
+                file_maxsizes=None,
                 ):
     if get_file_preprocessor is None:
         get_file_preprocessor = _get_preprocessor()
     for filename in filenames:
-        yield from _parse_file(filename, match_kind, get_file_preprocessor)
+        yield from _parse_file(
+                filename, match_kind, get_file_preprocessor, file_maxsizes)
 
 
-def _parse_file(filename, match_kind, get_file_preprocessor):
+def _parse_file(filename, match_kind, get_file_preprocessor, maxsizes):
+    srckwargs = {}
+    maxsize = _resolve_max_size(filename, maxsizes)
+    if maxsize:
+        srckwargs['maxtext'], srckwargs['maxlines'] = maxsize
+
     # Preprocess the file.
     preprocess = get_file_preprocessor(filename)
     preprocessed = preprocess()
@@ -30,7 +40,7 @@ def _parse_file(filename, match_kind, get_file_preprocessor):
 
     # Parse the lines.
     srclines = ((l.file, l.data) for l in preprocessed if l.kind == 'source')
-    for item in _parse(srclines):
+    for item in _parse(srclines, **srckwargs):
         if match_kind is not None and not match_kind(item.kind):
             continue
         if not item.filename:
@@ -38,6 +48,22 @@ def _parse_file(filename, match_kind, get_file_preprocessor):
         yield item
 
 
+def _resolve_max_size(filename, maxsizes):
+    for pattern, maxsize in (maxsizes.items() if maxsizes else ()):
+        if _match_glob(filename, pattern):
+            break
+    else:
+        return None
+    if not maxsize:
+        return None, None
+    maxtext, maxlines = maxsize
+    if maxtext is not None:
+        maxtext = int(maxtext)
+    if maxlines is not None:
+        maxlines = int(maxlines)
+    return maxtext, maxlines
+
+
 def parse_signature(text):
     raise NotImplementedError
 
diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py
index df70aae66b776..b5eae2ed92d0d 100644
--- a/Tools/c-analyzer/c_parser/parser/__init__.py
+++ b/Tools/c-analyzer/c_parser/parser/__init__.py
@@ -120,12 +120,12 @@
 from ._info import SourceInfo
 
 
-def parse(srclines):
+def parse(srclines, **srckwargs):
     if isinstance(srclines, str):  # a filename
         raise NotImplementedError
 
     anon_name = anonymous_names()
-    for result in _parse(srclines, anon_name):
+    for result in _parse(srclines, anon_name, **srckwargs):
         yield ParsedItem.from_raw(result)
 
 
@@ -152,17 +152,19 @@ def anon_name(prefix='anon-'):
 _logger = logging.getLogger(__name__)
 
 
-def _parse(srclines, anon_name):
+def _parse(srclines, anon_name, **srckwargs):
     from ._global import parse_globals
 
-    source = _iter_source(srclines)
-    #source = _iter_source(srclines, showtext=True)
+    source = _iter_source(srclines, **srckwargs)
     for result in parse_globals(source, anon_name):
         # XXX Handle blocks here instead of in parse_globals().
         yield result
 
 
-def _iter_source(lines, *, maxtext=20_000, maxlines=700, showtext=False):
+# We use defaults that cover most files.  Files with bigger declarations
+# are covered elsewhere (MAX_SIZES in cpython/_parser.py).
+
+def _iter_source(lines, *, maxtext=10_000, maxlines=200, showtext=False):
     maxtext = maxtext if maxtext and maxtext > 0 else None
     maxlines = maxlines if maxlines and maxlines > 0 else None
     filestack = []
diff --git a/Tools/c-analyzer/c_parser/preprocessor/__main__.py b/Tools/c-analyzer/c_parser/preprocessor/__main__.py
index bfc61949a76e4..55aa8752dce72 100644
--- a/Tools/c-analyzer/c_parser/preprocessor/__main__.py
+++ b/Tools/c-analyzer/c_parser/preprocessor/__main__.py
@@ -43,7 +43,7 @@ def add_common_cli(parser, *, get_preprocessor=_get_preprocessor):
     def process_args(args, *, argv):
         ns = vars(args)
 
-        process_fail_arg(args, argv)
+        process_fail_arg(args, argv=argv)
         ignore_exc = ns.pop('ignore_exc')
         # We later pass ignore_exc to _get_preprocessor().
 
diff --git a/Tools/c-analyzer/cpython/__main__.py b/Tools/c-analyzer/cpython/__main__.py
index 06ec871ba75e3..be331d50427d5 100644
--- a/Tools/c-analyzer/cpython/__main__.py
+++ b/Tools/c-analyzer/cpython/__main__.py
@@ -111,6 +111,7 @@ def cmd_parse(filenames=None, **kwargs):
     c_parser.cmd_parse(
         filenames,
         relroot=REPO_ROOT,
+        file_maxsizes=_parser.MAX_SIZES,
         **kwargs
     )
 
@@ -127,6 +128,7 @@ def cmd_check(filenames=None, **kwargs):
         relroot=REPO_ROOT,
         _analyze=_analyzer.analyze,
         _CHECKS=CHECKS,
+        file_maxsizes=_parser.MAX_SIZES,
         **kwargs
     )
 
@@ -141,6 +143,7 @@ def cmd_analyze(filenames=None, **kwargs):
         relroot=REPO_ROOT,
         _analyze=_analyzer.analyze,
         formats=formats,
+        file_maxsizes=_parser.MAX_SIZES,
         **kwargs
     )
 
diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py
index 8526b2af15a23..90b470c8196c0 100644
--- a/Tools/c-analyzer/cpython/_parser.py
+++ b/Tools/c-analyzer/cpython/_parser.py
@@ -14,6 +14,10 @@
 GLOB_ALL = '**/*'
 
 
+def _abs(relfile):
+    return os.path.join(REPO_ROOT, relfile)
+
+
 def clean_lines(text):
     """Clear out comments, blank lines, and leading/trailing whitespace."""
     lines = (line.strip() for line in text.splitlines())
@@ -22,7 +26,7 @@ def clean_lines(text):
              if line and not line.startswith('#'))
     glob_all = f'{GLOB_ALL} '
     lines = (re.sub(r'^[*] ', glob_all, line) for line in lines)
-    lines = (os.path.join(REPO_ROOT, line) for line in lines)
+    lines = (_abs(line) for line in lines)
     return list(lines)
 
 
@@ -55,26 +59,31 @@ def clean_lines(text):
 
 # Windows
 Modules/_winapi.c               # windows.h
+Modules/expat/winconfig.h
 Modules/overlapped.c            # winsock.h
 Python/dynload_win.c            # windows.h
-Modules/expat/winconfig.h
 Python/thread_nt.h
 
 # other OS-dependent
+Python/dynload_aix.c            # sys/ldr.h
 Python/dynload_dl.c             # dl.h
 Python/dynload_hpux.c           # dl.h
-Python/dynload_aix.c            # sys/ldr.h
 Python/thread_pthread.h
 
 # only huge constants (safe but parsing is slow)
+Modules/_blake2/impl/blake2-kat.h
 Modules/_ssl_data.h
+Modules/_ssl_data_300.h
+Modules/_ssl_data_111.h
+Modules/cjkcodecs/mappings_*.h
 Modules/unicodedata_db.h
 Modules/unicodename_db.h
-Modules/cjkcodecs/mappings_*.h
 Objects/unicodetype_db.h
-Python/importlib.h
-Python/importlib_external.h
-Python/importlib_zipimport.h
+
+# generated
+Python/frozen_modules/*.h
+Python/opcode_targets.h
+Python/stdlib_module_names.h
 
 # @end=conf@
 ''')
@@ -126,35 +135,40 @@ def clean_lines(text):
 Parser/**/*.c	Py_BUILD_CORE	1
 Objects/**/*.c	Py_BUILD_CORE	1
 
-Modules/faulthandler.c	Py_BUILD_CORE	1
+Modules/_asynciomodule.c	Py_BUILD_CORE	1
+Modules/_collectionsmodule.c	Py_BUILD_CORE	1
+Modules/_ctypes/_ctypes.c	Py_BUILD_CORE	1
+Modules/_ctypes/cfield.c	Py_BUILD_CORE	1
+Modules/_cursesmodule.c	Py_BUILD_CORE	1
+Modules/_datetimemodule.c	Py_BUILD_CORE	1
 Modules/_functoolsmodule.c	Py_BUILD_CORE	1
-Modules/gcmodule.c	Py_BUILD_CORE	1
-Modules/getpath.c	Py_BUILD_CORE	1
+Modules/_heapqmodule.c	Py_BUILD_CORE	1
 Modules/_io/*.c	Py_BUILD_CORE	1
-Modules/itertoolsmodule.c	Py_BUILD_CORE	1
 Modules/_localemodule.c	Py_BUILD_CORE	1
-Modules/main.c	Py_BUILD_CORE	1
-Modules/posixmodule.c	Py_BUILD_CORE	1
-Modules/signalmodule.c	Py_BUILD_CORE	1
+Modules/_operator.c	Py_BUILD_CORE	1
+Modules/_posixsubprocess.c	Py_BUILD_CORE	1
+Modules/_sre.c	Py_BUILD_CORE	1
 Modules/_threadmodule.c	Py_BUILD_CORE	1
 Modules/_tracemalloc.c	Py_BUILD_CORE	1
-Modules/_asynciomodule.c	Py_BUILD_CORE	1
-Modules/mathmodule.c	Py_BUILD_CORE	1
-Modules/cmathmodule.c	Py_BUILD_CORE	1
 Modules/_weakref.c	Py_BUILD_CORE	1
+Modules/_zoneinfo.c	Py_BUILD_CORE	1
+Modules/atexitmodule.c	Py_BUILD_CORE	1
+Modules/cmathmodule.c	Py_BUILD_CORE	1
+Modules/faulthandler.c	Py_BUILD_CORE	1
+Modules/gcmodule.c	Py_BUILD_CORE	1
+Modules/getpath.c	Py_BUILD_CORE	1
+Modules/itertoolsmodule.c	Py_BUILD_CORE	1
+Modules/main.c	Py_BUILD_CORE	1
+Modules/mathmodule.c	Py_BUILD_CORE	1
+Modules/posixmodule.c	Py_BUILD_CORE	1
 Modules/sha256module.c	Py_BUILD_CORE	1
 Modules/sha512module.c	Py_BUILD_CORE	1
-Modules/_datetimemodule.c	Py_BUILD_CORE	1
-Modules/_ctypes/cfield.c	Py_BUILD_CORE	1
-Modules/_heapqmodule.c	Py_BUILD_CORE	1
-Modules/_posixsubprocess.c	Py_BUILD_CORE	1
-Modules/_sre.c	Py_BUILD_CORE	1
-Modules/_collectionsmodule.c	Py_BUILD_CORE	1
-Modules/_zoneinfo.c	Py_BUILD_CORE	1
+Modules/signalmodule.c	Py_BUILD_CORE	1
+Modules/symtablemodule.c	Py_BUILD_CORE	1
+Modules/timemodule.c	Py_BUILD_CORE	1
 Modules/unicodedata.c	Py_BUILD_CORE	1
-Modules/_cursesmodule.c	Py_BUILD_CORE	1
-Modules/_ctypes/_ctypes.c	Py_BUILD_CORE	1
 Objects/stringlib/codecs.h	Py_BUILD_CORE	1
+Objects/stringlib/unicode_format.h	Py_BUILD_CORE	1
 Python/ceval_gil.h	Py_BUILD_CORE	1
 Python/condvar.h	Py_BUILD_CORE	1
 
@@ -244,6 +258,7 @@ def clean_lines(text):
 Modules/sre_lib.h	LOCAL(type)	static inline type
 Modules/sre_lib.h	SRE(F)	sre_ucs2_##F
 Objects/stringlib/codecs.h	STRINGLIB_IS_UNICODE	1
+Include/internal/pycore_bitutils.h	_Py__has_builtin(B)	0
 
 # @end=tsv@
 ''')[1:]
@@ -264,6 +279,18 @@ def clean_lines(text):
     './Include/cpython/',
 ]
 
+MAX_SIZES = {
+    _abs('Include/**/*.h'): (5_000, 500),
+    _abs('Modules/_ctypes/ctypes.h'): (5_000, 500),
+    _abs('Modules/_datetimemodule.c'): (20_000, 300),
+    _abs('Modules/posixmodule.c'): (20_000, 500),
+    _abs('Modules/termios.c'): (10_000, 800),
+    _abs('Modules/_testcapimodule.c'): (20_000, 400),
+    _abs('Modules/expat/expat.h'): (10_000, 400),
+    _abs('Objects/stringlib/unicode_format.h'): (10_000, 400),
+    _abs('Objects/typeobject.c'): (20_000, 200),
+}
+
 
 def get_preprocessor(*,
                      file_macros=None,
@@ -298,6 +325,7 @@ def parse_file(filename, *,
         filename,
         match_kind=match_kind,
         get_file_preprocessor=get_file_preprocessor,
+        file_maxsizes=MAX_SIZES,
     )
 
 
@@ -317,5 +345,6 @@ def parse_files(filenames=None, *,
         filenames,
         match_kind=match_kind,
         get_file_preprocessor=get_file_preprocessor,
+        file_maxsizes=MAX_SIZES,
         **file_kwargs
     )


From webhook-mailer at python.org  Wed Dec  1 14:23:51 2021
From: webhook-mailer at python.org (DinoV)
Date: Wed, 01 Dec 2021 19:23:51 -0000
Subject: [Python-checkins] bpo-30533: Add docs for
 `inspect.getmembers_static` (#29874)
Message-ID: <mailman.9.1638386632.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/c2bb29ce9ae4adb6a8123285ad3585907cd4cc73
commit: c2bb29ce9ae4adb6a8123285ad3585907cd4cc73
branch: main
author: Weipeng Hong <hongweichen8888 at sina.com>
committer: DinoV <dinoviehland at gmail.com>
date: 2021-12-01T11:23:46-08:00
summary:

bpo-30533: Add docs for `inspect.getmembers_static` (#29874)

* Add docs for `inspect.getmembers_static`

* update

files:
M Doc/library/inspect.rst
M Doc/whatsnew/3.11.rst

diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst
index 1074f977184bb..711f510d7dcb0 100644
--- a/Doc/library/inspect.rst
+++ b/Doc/library/inspect.rst
@@ -275,6 +275,24 @@ attributes:
       listed in the metaclass' custom :meth:`__dir__`.
 
 
+.. function:: getmembers_static(object[, predicate])
+
+    Return all the members of an object in a list of ``(name, value)``
+    pairs sorted by name without triggering dynamic lookup via the descriptor
+    protocol, __getattr__ or __getattribute__. Optionally, only return members
+    that satisfy a given predicate.
+
+    .. note::
+
+        :func:`getmembers_static` may not be able to retrieve all members
+        that getmembers can fetch (like dynamically created attributes)
+        and may find members that getmembers can't (like descriptors
+        that raise AttributeError). It can also return descriptor objects
+        instead of instance members in some cases.
+
+    .. versionadded:: 3.11
+
+
 .. function:: getmodulename(path)
 
    Return the name of the module named by the file *path*, without including the
diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index 6853c04143512..345b2df43f403 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -207,6 +207,13 @@ fractions
   (Contributed by Mark Dickinson in :issue:`44547`.)
 
 
+inspect
+-------
+* Add :func:`inspect.getmembers_static`: return all members without
+  triggering dynamic lookup via the descriptor protocol. (Contributed by
+  Weipeng Hong in :issue:`30533`.)
+
+
 math
 ----
 * Add :func:`math.exp2`: return 2 raised to the power of x.


From webhook-mailer at python.org  Wed Dec  1 14:43:34 2021
From: webhook-mailer at python.org (tiran)
Date: Wed, 01 Dec 2021 19:43:34 -0000
Subject: [Python-checkins] bpo-40280: Emscripten defaults to
 --with-ensurepip=no (GH-29873)
Message-ID: <mailman.10.1638387815.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/9deb83468c56c7484645e6e3a6d0183cd6a0afd7
commit: 9deb83468c56c7484645e6e3a6d0183cd6a0afd7
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-01T20:43:07+01:00
summary:

bpo-40280: Emscripten defaults to --with-ensurepip=no (GH-29873)

files:
M configure
M configure.ac

diff --git a/configure b/configure
index 382e030974727..4dfd4a565d14e 100755
--- a/configure
+++ b/configure
@@ -20268,7 +20268,15 @@ $as_echo_n "checking for ensurepip... " >&6; }
 if test "${with_ensurepip+set}" = set; then :
   withval=$with_ensurepip;
 else
-  with_ensurepip=upgrade
+
+      case $ac_sys_system in #(
+  Emscripten) :
+    $with_ensurepip=no ;; #(
+  *) :
+    with_ensurepip=upgrade
+       ;;
+esac
+
 fi
 
 case $with_ensurepip in #(
diff --git a/configure.ac b/configure.ac
index 1e5fef337ca5d..38f943fa9d77c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -5871,7 +5871,12 @@ AC_ARG_WITH(ensurepip,
     [AS_HELP_STRING([--with-ensurepip@<:@=install|upgrade|no@:>@],
         ["install" or "upgrade" using bundled pip (default is upgrade)])],
     [],
-    [with_ensurepip=upgrade])
+    [
+      AS_CASE([$ac_sys_system],
+        [Emscripten], [$with_ensurepip=no],
+        [with_ensurepip=upgrade]
+      )
+    ])
 AS_CASE($with_ensurepip,
     [yes|upgrade],[ENSUREPIP=upgrade],
     [install],[ENSUREPIP=install],


From webhook-mailer at python.org  Wed Dec  1 17:16:36 2021
From: webhook-mailer at python.org (tiran)
Date: Wed, 01 Dec 2021 22:16:36 -0000
Subject: [Python-checkins] bpo-40280: Emscripten with_ensurepip=no,
 second attempt (GH-29884)
Message-ID: <mailman.11.1638396997.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/309110f37cdfc78d160ed08ae8faa6f6160ba87e
commit: 309110f37cdfc78d160ed08ae8faa6f6160ba87e
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-01T23:16:27+01:00
summary:

bpo-40280: Emscripten with_ensurepip=no, second attempt (GH-29884)

files:
M configure
M configure.ac

diff --git a/configure b/configure
index 4dfd4a565d14e..0aceffb35065a 100755
--- a/configure
+++ b/configure
@@ -20271,7 +20271,7 @@ else
 
       case $ac_sys_system in #(
   Emscripten) :
-    $with_ensurepip=no ;; #(
+    with_ensurepip=no ;; #(
   *) :
     with_ensurepip=upgrade
        ;;
diff --git a/configure.ac b/configure.ac
index 38f943fa9d77c..99fa94d102909 100644
--- a/configure.ac
+++ b/configure.ac
@@ -5873,7 +5873,7 @@ AC_ARG_WITH(ensurepip,
     [],
     [
       AS_CASE([$ac_sys_system],
-        [Emscripten], [$with_ensurepip=no],
+        [Emscripten], [with_ensurepip=no],
         [with_ensurepip=upgrade]
       )
     ])


From webhook-mailer at python.org  Thu Dec  2 03:52:42 2021
From: webhook-mailer at python.org (asvetlov)
Date: Thu, 02 Dec 2021 08:52:42 -0000
Subject: [Python-checkins] docs: Improve example for urlparse() (GH-29816)
Message-ID: <mailman.12.1638435163.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/226d22ff2d209495621550eb78e81ed4c0fe0152
commit: 226d22ff2d209495621550eb78e81ed4c0fe0152
branch: main
author: Christian Clauss <cclauss at me.com>
committer: asvetlov <andrew.svetlov at gmail.com>
date: 2021-12-02T10:52:32+02:00
summary:

docs: Improve example for urlparse() (GH-29816)

files:
M Doc/library/urllib.parse.rst

diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
index a060cc9ba7fdd..1478b34bc9551 100644
--- a/Doc/library/urllib.parse.rst
+++ b/Doc/library/urllib.parse.rst
@@ -48,17 +48,29 @@ or on combining URL components into a URL string.
    result, except for a leading slash in the *path* component, which is retained if
    present.  For example:
 
+   .. doctest::
+      :options: +NORMALIZE_WHITESPACE
+
       >>> from urllib.parse import urlparse
-      >>> o = urlparse('http://www.cwi.nl:80/%7Eguido/Python.html')
-      >>> o   # doctest: +NORMALIZE_WHITESPACE
-      ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html',
-                  params='', query='', fragment='')
+      >>> urlparse("scheme://netloc/path;parameters?query#fragment")
+      ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='',
+                  query='query', fragment='fragment')
+      >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?"
+      ...              "highlight=params#url-parsing")
+      >>> o
+      ParseResult(scheme='http', netloc='docs.python.org:80',
+                  path='/3/library/urllib.parse.html', params='',
+                  query='highlight=params', fragment='url-parsing')
       >>> o.scheme
       'http'
+      >>> o.netloc
+      'docs.python.org:80'
+      >>> o.hostname
+      'docs.python.org'
       >>> o.port
       80
-      >>> o.geturl()
-      'http://www.cwi.nl:80/%7Eguido/Python.html'
+      >>> o._replace(fragment="").geturl()
+      'http://docs.python.org:80/3/library/urllib.parse.html?highlight=params'
 
    Following the syntax specifications in :rfc:`1808`, urlparse recognizes
    a netloc only if it is properly introduced by '//'.  Otherwise the
@@ -92,31 +104,30 @@ or on combining URL components into a URL string.
    The return value is a :term:`named tuple`, which means that its items can
    be accessed by index or as named attributes, which are:
 
-   +------------------+-------+--------------------------+----------------------+
-   | Attribute        | Index | Value                    | Value if not present |
-   +==================+=======+==========================+======================+
-   | :attr:`scheme`   | 0     | URL scheme specifier     | *scheme* parameter   |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`netloc`   | 1     | Network location part    | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`path`     | 2     | Hierarchical path        | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`params`   | 3     | Parameters for last path | empty string         |
-   |                  |       | element                  |                      |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`query`    | 4     | Query component          | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`fragment` | 5     | Fragment identifier      | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`username` |       | User name                | :const:`None`        |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`password` |       | Password                 | :const:`None`        |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`hostname` |       | Host name (lower case)   | :const:`None`        |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`port`     |       | Port number as integer,  | :const:`None`        |
-   |                  |       | if present               |                      |
-   +------------------+-------+--------------------------+----------------------+
+   +------------------+-------+-------------------------+------------------------+
+   | Attribute        | Index | Value                   | Value if not present   |
+   +==================+=======+=========================+========================+
+   | :attr:`scheme`   | 0     | URL scheme specifier    | *scheme* parameter     |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`netloc`   | 1     | Network location part   | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`path`     | 2     | Hierarchical path       | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`params`   | 3     | No longer used          | always an empty string |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`query`    | 4     | Query component         | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`fragment` | 5     | Fragment identifier     | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`username` |       | User name               | :const:`None`          |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`password` |       | Password                | :const:`None`          |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`hostname` |       | Host name (lower case)  | :const:`None`          |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`port`     |       | Port number as integer, | :const:`None`          |
+   |                  |       | if present              |                        |
+   +------------------+-------+-------------------------+------------------------+
 
    Reading the :attr:`port` attribute will raise a :exc:`ValueError` if
    an invalid port is specified in the URL.  See section


From webhook-mailer at python.org  Thu Dec  2 04:17:47 2021
From: webhook-mailer at python.org (miss-islington)
Date: Thu, 02 Dec 2021 09:17:47 -0000
Subject: [Python-checkins] bpo-40280: Emscripten has no support for
 subprocesses (GH-29872)
Message-ID: <mailman.13.1638436668.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/cb2b3c8d3566ae46b3b8d0718019e1c98484589e
commit: cb2b3c8d3566ae46b3b8d0718019e1c98484589e
branch: main
author: Christian Heimes <christian at python.org>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-02T01:17:37-08:00
summary:

bpo-40280: Emscripten has no support for subprocesses (GH-29872)



Fixes ``platform`` and ``help()`` on emscripten.

Signed-off-by: Christian Heimes <christian at python.org>

Automerge-Triggered-By: GH:tiran

files:
M Lib/platform.py
M Lib/pydoc.py

diff --git a/Lib/platform.py b/Lib/platform.py
index 9e9b42238fb76..3f3f25a2c92d3 100755
--- a/Lib/platform.py
+++ b/Lib/platform.py
@@ -607,7 +607,10 @@ def _syscmd_file(target, default=''):
         # XXX Others too ?
         return default
 
-    import subprocess
+    try:
+        import subprocess
+    except ImportError:
+        return default
     target = _follow_symlinks(target)
     # "file" output is locale dependent: force the usage of the C locale
     # to get deterministic behavior.
@@ -746,7 +749,10 @@ def from_subprocess():
         """
         Fall back to `uname -p`
         """
-        import subprocess
+        try:
+            import subprocess
+        except ImportError:
+            return None
         try:
             return subprocess.check_output(
                 ['uname', '-p'],
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
index 3a2ff218f8319..7d52359c9a0ce 100755
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -1556,6 +1556,8 @@ def getpager():
         return plainpager
     if not sys.stdin.isatty() or not sys.stdout.isatty():
         return plainpager
+    if sys.platform == "emscripten":
+        return plainpager
     use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER')
     if use_pager:
         if sys.platform == 'win32': # pipes completely broken in Windows


From webhook-mailer at python.org  Thu Dec  2 05:25:03 2021
From: webhook-mailer at python.org (asvetlov)
Date: Thu, 02 Dec 2021 10:25:03 -0000
Subject: [Python-checkins] docs: Improve example for urlparse() (GH-29816)
 (GH-29889)
Message-ID: <mailman.14.1638440705.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/0003d57e22dc983d85ec734bc5031360b2cfc537
commit: 0003d57e22dc983d85ec734bc5031360b2cfc537
branch: 3.9
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: asvetlov <andrew.svetlov at gmail.com>
date: 2021-12-02T12:24:51+02:00
summary:

docs: Improve example for urlparse() (GH-29816) (GH-29889)

(cherry picked from commit 226d22ff2d209495621550eb78e81ed4c0fe0152)

Co-authored-by: Christian Clauss <cclauss at me.com>

files:
M Doc/library/urllib.parse.rst

diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
index c03765259c3d5..b7430e9d4d670 100644
--- a/Doc/library/urllib.parse.rst
+++ b/Doc/library/urllib.parse.rst
@@ -48,17 +48,29 @@ or on combining URL components into a URL string.
    result, except for a leading slash in the *path* component, which is retained if
    present.  For example:
 
+   .. doctest::
+      :options: +NORMALIZE_WHITESPACE
+
       >>> from urllib.parse import urlparse
-      >>> o = urlparse('http://www.cwi.nl:80/%7Eguido/Python.html')
-      >>> o   # doctest: +NORMALIZE_WHITESPACE
-      ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html',
-                  params='', query='', fragment='')
+      >>> urlparse("scheme://netloc/path;parameters?query#fragment")
+      ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='',
+                  query='query', fragment='fragment')
+      >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?"
+      ...              "highlight=params#url-parsing")
+      >>> o
+      ParseResult(scheme='http', netloc='docs.python.org:80',
+                  path='/3/library/urllib.parse.html', params='',
+                  query='highlight=params', fragment='url-parsing')
       >>> o.scheme
       'http'
+      >>> o.netloc
+      'docs.python.org:80'
+      >>> o.hostname
+      'docs.python.org'
       >>> o.port
       80
-      >>> o.geturl()
-      'http://www.cwi.nl:80/%7Eguido/Python.html'
+      >>> o._replace(fragment="").geturl()
+      'http://docs.python.org:80/3/library/urllib.parse.html?highlight=params'
 
    Following the syntax specifications in :rfc:`1808`, urlparse recognizes
    a netloc only if it is properly introduced by '//'.  Otherwise the
@@ -92,31 +104,30 @@ or on combining URL components into a URL string.
    The return value is a :term:`named tuple`, which means that its items can
    be accessed by index or as named attributes, which are:
 
-   +------------------+-------+--------------------------+----------------------+
-   | Attribute        | Index | Value                    | Value if not present |
-   +==================+=======+==========================+======================+
-   | :attr:`scheme`   | 0     | URL scheme specifier     | *scheme* parameter   |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`netloc`   | 1     | Network location part    | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`path`     | 2     | Hierarchical path        | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`params`   | 3     | Parameters for last path | empty string         |
-   |                  |       | element                  |                      |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`query`    | 4     | Query component          | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`fragment` | 5     | Fragment identifier      | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`username` |       | User name                | :const:`None`        |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`password` |       | Password                 | :const:`None`        |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`hostname` |       | Host name (lower case)   | :const:`None`        |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`port`     |       | Port number as integer,  | :const:`None`        |
-   |                  |       | if present               |                      |
-   +------------------+-------+--------------------------+----------------------+
+   +------------------+-------+-------------------------+------------------------+
+   | Attribute        | Index | Value                   | Value if not present   |
+   +==================+=======+=========================+========================+
+   | :attr:`scheme`   | 0     | URL scheme specifier    | *scheme* parameter     |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`netloc`   | 1     | Network location part   | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`path`     | 2     | Hierarchical path       | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`params`   | 3     | No longer used          | always an empty string |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`query`    | 4     | Query component         | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`fragment` | 5     | Fragment identifier     | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`username` |       | User name               | :const:`None`          |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`password` |       | Password                | :const:`None`          |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`hostname` |       | Host name (lower case)  | :const:`None`          |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`port`     |       | Port number as integer, | :const:`None`          |
+   |                  |       | if present              |                        |
+   +------------------+-------+-------------------------+------------------------+
 
    Reading the :attr:`port` attribute will raise a :exc:`ValueError` if
    an invalid port is specified in the URL.  See section


From webhook-mailer at python.org  Thu Dec  2 05:25:18 2021
From: webhook-mailer at python.org (asvetlov)
Date: Thu, 02 Dec 2021 10:25:18 -0000
Subject: [Python-checkins] docs: Improve example for urlparse() (GH-29816)
 (GH-29888)
Message-ID: <mailman.15.1638440719.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/eac07e5ab02474c43aa9924753d4d774f8ff6828
commit: eac07e5ab02474c43aa9924753d4d774f8ff6828
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: asvetlov <andrew.svetlov at gmail.com>
date: 2021-12-02T12:25:13+02:00
summary:

docs: Improve example for urlparse() (GH-29816) (GH-29888)

(cherry picked from commit 226d22ff2d209495621550eb78e81ed4c0fe0152)

Co-authored-by: Christian Clauss <cclauss at me.com>

files:
M Doc/library/urllib.parse.rst

diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst
index a060cc9ba7fdd..1478b34bc9551 100644
--- a/Doc/library/urllib.parse.rst
+++ b/Doc/library/urllib.parse.rst
@@ -48,17 +48,29 @@ or on combining URL components into a URL string.
    result, except for a leading slash in the *path* component, which is retained if
    present.  For example:
 
+   .. doctest::
+      :options: +NORMALIZE_WHITESPACE
+
       >>> from urllib.parse import urlparse
-      >>> o = urlparse('http://www.cwi.nl:80/%7Eguido/Python.html')
-      >>> o   # doctest: +NORMALIZE_WHITESPACE
-      ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html',
-                  params='', query='', fragment='')
+      >>> urlparse("scheme://netloc/path;parameters?query#fragment")
+      ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='',
+                  query='query', fragment='fragment')
+      >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?"
+      ...              "highlight=params#url-parsing")
+      >>> o
+      ParseResult(scheme='http', netloc='docs.python.org:80',
+                  path='/3/library/urllib.parse.html', params='',
+                  query='highlight=params', fragment='url-parsing')
       >>> o.scheme
       'http'
+      >>> o.netloc
+      'docs.python.org:80'
+      >>> o.hostname
+      'docs.python.org'
       >>> o.port
       80
-      >>> o.geturl()
-      'http://www.cwi.nl:80/%7Eguido/Python.html'
+      >>> o._replace(fragment="").geturl()
+      'http://docs.python.org:80/3/library/urllib.parse.html?highlight=params'
 
    Following the syntax specifications in :rfc:`1808`, urlparse recognizes
    a netloc only if it is properly introduced by '//'.  Otherwise the
@@ -92,31 +104,30 @@ or on combining URL components into a URL string.
    The return value is a :term:`named tuple`, which means that its items can
    be accessed by index or as named attributes, which are:
 
-   +------------------+-------+--------------------------+----------------------+
-   | Attribute        | Index | Value                    | Value if not present |
-   +==================+=======+==========================+======================+
-   | :attr:`scheme`   | 0     | URL scheme specifier     | *scheme* parameter   |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`netloc`   | 1     | Network location part    | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`path`     | 2     | Hierarchical path        | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`params`   | 3     | Parameters for last path | empty string         |
-   |                  |       | element                  |                      |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`query`    | 4     | Query component          | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`fragment` | 5     | Fragment identifier      | empty string         |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`username` |       | User name                | :const:`None`        |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`password` |       | Password                 | :const:`None`        |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`hostname` |       | Host name (lower case)   | :const:`None`        |
-   +------------------+-------+--------------------------+----------------------+
-   | :attr:`port`     |       | Port number as integer,  | :const:`None`        |
-   |                  |       | if present               |                      |
-   +------------------+-------+--------------------------+----------------------+
+   +------------------+-------+-------------------------+------------------------+
+   | Attribute        | Index | Value                   | Value if not present   |
+   +==================+=======+=========================+========================+
+   | :attr:`scheme`   | 0     | URL scheme specifier    | *scheme* parameter     |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`netloc`   | 1     | Network location part   | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`path`     | 2     | Hierarchical path       | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`params`   | 3     | No longer used          | always an empty string |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`query`    | 4     | Query component         | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`fragment` | 5     | Fragment identifier     | empty string           |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`username` |       | User name               | :const:`None`          |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`password` |       | Password                | :const:`None`          |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`hostname` |       | Host name (lower case)  | :const:`None`          |
+   +------------------+-------+-------------------------+------------------------+
+   | :attr:`port`     |       | Port number as integer, | :const:`None`          |
+   |                  |       | if present              |                        |
+   +------------------+-------+-------------------------+------------------------+
 
    Reading the :attr:`port` attribute will raise a :exc:`ValueError` if
    an invalid port is specified in the URL.  See section


From webhook-mailer at python.org  Thu Dec  2 05:43:42 2021
From: webhook-mailer at python.org (vstinner)
Date: Thu, 02 Dec 2021 10:43:42 -0000
Subject: [Python-checkins] bpo-45954: Rename PyConfig.no_debug_ranges to
 code_debug_ranges (GH-29886)
Message-ID: <mailman.16.1638441823.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/a6c3b0faa1d55e36539caf19bd3bcf1dea12df84
commit: a6c3b0faa1d55e36539caf19bd3bcf1dea12df84
branch: main
author: Victor Stinner <vstinner at python.org>
committer: vstinner <vstinner at python.org>
date: 2021-12-02T11:43:37+01:00
summary:

bpo-45954: Rename PyConfig.no_debug_ranges to code_debug_ranges (GH-29886)

Rename PyConfig.no_debug_ranges to PyConfig.code_debug_ranges and
invert the value.

Document -X no_debug_ranges and PYTHONNODEBUGRANGES env var in
PyConfig.code_debug_ranges documentation.

files:
M Doc/c-api/init_config.rst
M Include/cpython/initconfig.h
M Lib/test/_test_embed_set_config.py
M Lib/test/support/__init__.py
M Lib/test/test_embed.py
M Objects/codeobject.c
M Programs/_testembed.c
M Python/initconfig.c

diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index 989660caeda06..3642a8bf7fbc7 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -596,13 +596,16 @@ PyConfig
 
       .. versionadded:: 3.10
 
-   .. c:member:: int no_debug_ranges
+   .. c:member:: int code_debug_ranges
 
-      If equals to ``1``, disables the inclusion of the end line and column
+      If equals to ``0``, disables the inclusion of the end line and column
       mappings in code objects. Also disables traceback printing carets to
       specific error locations.
 
-      Default: ``0``.
+      Set to ``0`` by the :envvar:`PYTHONNODEBUGRANGES` environment variable
+      and by the :option:`-X no_debug_ranges <-X>` command line option.
+
+      Default: ``1``.
 
       .. versionadded:: 3.11
 
diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
index 76f3c5268ee5f..2be4c25ec0b09 100644
--- a/Include/cpython/initconfig.h
+++ b/Include/cpython/initconfig.h
@@ -143,7 +143,7 @@ typedef struct PyConfig {
     int faulthandler;
     int tracemalloc;
     int import_time;
-    int no_debug_ranges;
+    int code_debug_ranges;
     int show_ref_count;
     int dump_refs;
     wchar_t *dump_refs_file;
diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py
index 23c927e2646bc..d05907ecfe744 100644
--- a/Lib/test/_test_embed_set_config.py
+++ b/Lib/test/_test_embed_set_config.py
@@ -1,7 +1,7 @@
 # bpo-42260: Test _PyInterpreterState_GetConfigCopy()
 # and _PyInterpreterState_SetConfig().
 #
-# Test run in a subinterpreter since set_config(get_config())
+# Test run in a subprocess since set_config(get_config())
 # does reset sys attributes to their state of the Python startup
 # (before the site module is run).
 
@@ -61,7 +61,7 @@ def test_set_invalid(self):
             'faulthandler',
             'tracemalloc',
             'import_time',
-            'no_debug_ranges',
+            'code_debug_ranges',
             'show_ref_count',
             'dump_refs',
             'malloc_stats',
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 6d84a8bea420e..f8faa41ad439c 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -449,7 +449,7 @@ def has_no_debug_ranges():
     except ImportError:
         raise unittest.SkipTest("_testinternalcapi required")
     config = _testinternalcapi.get_config()
-    return bool(config['no_debug_ranges'])
+    return not bool(config['code_debug_ranges'])
 
 def requires_debug_ranges(reason='requires co_positions / debug_ranges'):
     return unittest.skipIf(has_no_debug_ranges(), reason)
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 94cdd98b2dbba..4781c7f2a9d12 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -386,7 +386,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'faulthandler': 0,
         'tracemalloc': 0,
         'import_time': 0,
-        'no_debug_ranges': 0,
+        'code_debug_ranges': 1,
         'show_ref_count': 0,
         'dump_refs': 0,
         'malloc_stats': 0,
@@ -818,7 +818,7 @@ def test_init_from_config(self):
             'hash_seed': 123,
             'tracemalloc': 2,
             'import_time': 1,
-            'no_debug_ranges': 1,
+            'code_debug_ranges': 0,
             'show_ref_count': 1,
             'malloc_stats': 1,
 
@@ -878,7 +878,7 @@ def test_init_compat_env(self):
             'hash_seed': 42,
             'tracemalloc': 2,
             'import_time': 1,
-            'no_debug_ranges': 1,
+            'code_debug_ranges': 0,
             'malloc_stats': 1,
             'inspect': 1,
             'optimization_level': 2,
@@ -908,7 +908,7 @@ def test_init_python_env(self):
             'hash_seed': 42,
             'tracemalloc': 2,
             'import_time': 1,
-            'no_debug_ranges': 1,
+            'code_debug_ranges': 0,
             'malloc_stats': 1,
             'inspect': 1,
             'optimization_level': 2,
diff --git a/Objects/codeobject.c b/Objects/codeobject.c
index 5ab8641a17caa..a413b183be8ed 100644
--- a/Objects/codeobject.c
+++ b/Objects/codeobject.c
@@ -381,7 +381,7 @@ _PyCode_New(struct _PyCodeConstructor *con)
 
     // Discard the endlinetable and columntable if we are opted out of debug
     // ranges.
-    if (_Py_GetConfig()->no_debug_ranges) {
+    if (!_Py_GetConfig()->code_debug_ranges) {
         con->endlinetable = Py_None;
         con->columntable = Py_None;
     }
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index 8077c470c07c5..fc5f44d5eb626 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -531,7 +531,7 @@ static int test_init_from_config(void)
     config.import_time = 1;
 
     putenv("PYTHONNODEBUGRANGES=0");
-    config.no_debug_ranges = 1;
+    config.code_debug_ranges = 0;
 
     config.show_ref_count = 1;
     /* FIXME: test dump_refs: bpo-34223 */
diff --git a/Python/initconfig.c b/Python/initconfig.c
index c916e2f7c0714..53b624fdd72ec 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -614,7 +614,7 @@ config_check_consistency(const PyConfig *config)
     assert(config->faulthandler >= 0);
     assert(config->tracemalloc >= 0);
     assert(config->import_time >= 0);
-    assert(config->no_debug_ranges >= 0);
+    assert(config->code_debug_ranges >= 0);
     assert(config->show_ref_count >= 0);
     assert(config->dump_refs >= 0);
     assert(config->malloc_stats >= 0);
@@ -740,6 +740,7 @@ _PyConfig_InitCompatConfig(PyConfig *config)
     config->legacy_windows_stdio = -1;
 #endif
     config->use_frozen_modules = -1;
+    config->code_debug_ranges = 1;
 }
 
 
@@ -904,7 +905,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_ATTR(faulthandler);
     COPY_ATTR(tracemalloc);
     COPY_ATTR(import_time);
-    COPY_ATTR(no_debug_ranges);
+    COPY_ATTR(code_debug_ranges);
     COPY_ATTR(show_ref_count);
     COPY_ATTR(dump_refs);
     COPY_ATTR(dump_refs_file);
@@ -1012,7 +1013,7 @@ _PyConfig_AsDict(const PyConfig *config)
     SET_ITEM_INT(faulthandler);
     SET_ITEM_INT(tracemalloc);
     SET_ITEM_INT(import_time);
-    SET_ITEM_INT(no_debug_ranges);
+    SET_ITEM_INT(code_debug_ranges);
     SET_ITEM_INT(show_ref_count);
     SET_ITEM_INT(dump_refs);
     SET_ITEM_INT(malloc_stats);
@@ -1291,7 +1292,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
     GET_UINT(faulthandler);
     GET_UINT(tracemalloc);
     GET_UINT(import_time);
-    GET_UINT(no_debug_ranges);
+    GET_UINT(code_debug_ranges);
     GET_UINT(show_ref_count);
     GET_UINT(dump_refs);
     GET_UINT(malloc_stats);
@@ -1853,7 +1854,7 @@ config_read_complex_options(PyConfig *config)
 
     if (config_get_env(config, "PYTHONNODEBUGRANGES")
        || config_get_xoption(config, L"no_debug_ranges")) {
-        config->no_debug_ranges = 1;
+        config->code_debug_ranges = 0;
     }
 
     PyStatus status;


From webhook-mailer at python.org  Thu Dec  2 06:19:35 2021
From: webhook-mailer at python.org (tiran)
Date: Thu, 02 Dec 2021 11:19:35 -0000
Subject: [Python-checkins] bpo-40280: Optimize ints and and startup on wasm
 (GH-29887)
Message-ID: <mailman.17.1638443976.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/cb8f491f46e262549f6c447b31625cab7c20a60a
commit: cb8f491f46e262549f6c447b31625cab7c20a60a
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-02T12:19:30+01:00
summary:

bpo-40280: Optimize ints and and startup on wasm (GH-29887)

files:
M Include/pyport.h
M Python/pylifecycle.c

diff --git a/Include/pyport.h b/Include/pyport.h
index 953f75c970d83..81b1bde841e08 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -87,13 +87,17 @@ Used in:  Py_SAFE_DOWNCAST
 
 /* If PYLONG_BITS_IN_DIGIT is not defined then we'll use 30-bit digits if all
    the necessary integer types are available, and we're on a 64-bit platform
-   (as determined by SIZEOF_VOID_P); otherwise we use 15-bit digits. */
+   (as determined by SIZEOF_VOID_P); otherwise we use 15-bit digits.
+
+   From pyodide: WASM has 32 bit pointers but has native 64 bit arithmetic
+   so it is more efficient to use 30 bit digits.
+ */
 
 #ifndef PYLONG_BITS_IN_DIGIT
-#if SIZEOF_VOID_P >= 8
-#define PYLONG_BITS_IN_DIGIT 30
+#if SIZEOF_VOID_P >= 8 || defined(__wasm__)
+#  define PYLONG_BITS_IN_DIGIT 30
 #else
-#define PYLONG_BITS_IN_DIGIT 15
+#  define PYLONG_BITS_IN_DIGIT 15
 #endif
 #endif
 
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 74d269b7a5646..c6b2a1eb96148 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -2145,7 +2145,11 @@ is_valid_fd(int fd)
     if (fd < 0) {
         return 0;
     }
-#if defined(F_GETFD) && (defined(__linux__) || defined(__APPLE__) || defined(MS_WINDOWS))
+#if defined(F_GETFD) && ( \
+        defined(__linux__) || \
+        defined(__APPLE__) || \
+        defined(MS_WINDOWS) || \
+        defined(__wasm__))
     int res;
     _Py_BEGIN_SUPPRESS_IPH
     res = fcntl(fd, F_GETFD);


From webhook-mailer at python.org  Thu Dec  2 11:50:07 2021
From: webhook-mailer at python.org (ethanfurman)
Date: Thu, 02 Dec 2021 16:50:07 -0000
Subject: [Python-checkins] bpo-45535: Improve output of Enum ``dir()``
 (GH-29316)
Message-ID: <mailman.18.1638463807.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/b2afdc95cc8f4e9228148730949a43cef0323f15
commit: b2afdc95cc8f4e9228148730949a43cef0323f15
branch: main
author: Alex Waygood <Alex.Waygood at Gmail.com>
committer: ethanfurman <ethan at stoneleaf.us>
date: 2021-12-02T08:49:52-08:00
summary:

bpo-45535: Improve output of Enum ``dir()`` (GH-29316)

Modify the ``EnumType.__dir__()`` and ``Enum.__dir__()`` to ensure
that user-defined methods and methods inherited from mixin classes always
show up in the output of `help()`. This change also makes it easier for
IDEs to provide auto-completion.

files:
A Misc/NEWS.d/next/Library/2021-10-29-16-28-06.bpo-45535.n8NiOE.rst
M Doc/howto/enum.rst
M Doc/library/enum.rst
M Lib/enum.py
M Lib/test/test_enum.py

diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst
index b0b17b1669f1a..91bef218b9299 100644
--- a/Doc/howto/enum.rst
+++ b/Doc/howto/enum.rst
@@ -997,11 +997,12 @@ Plain :class:`Enum` classes always evaluate as :data:`True`.
 """""""""""""""""""""""""""""
 
 If you give your enum subclass extra methods, like the `Planet`_
-class below, those methods will show up in a :func:`dir` of the member,
-but not of the class::
+class below, those methods will show up in a :func:`dir` of the member and the
+class. Attributes defined in an :func:`__init__` method will only show up in a
+:func:`dir` of the member::
 
     >>> dir(Planet)
-    ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__members__', '__module__']
+    ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__init__', '__members__', '__module__', 'surface_gravity']
     >>> dir(Planet.EARTH)
     ['__class__', '__doc__', '__module__', 'mass', 'name', 'radius', 'surface_gravity', 'value']
 
diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst
index 850b491680c68..572048a255bf0 100644
--- a/Doc/library/enum.rst
+++ b/Doc/library/enum.rst
@@ -162,7 +162,8 @@ Data Types
    .. method:: EnumType.__dir__(cls)
 
       Returns ``['__class__', '__doc__', '__members__', '__module__']`` and the
-      names of the members in *cls*::
+      names of the members in ``cls``. User-defined methods and methods from
+      mixin classes will also be included::
 
         >>> dir(Color)
         ['BLUE', 'GREEN', 'RED', '__class__', '__doc__', '__members__', '__module__']
@@ -260,7 +261,7 @@ Data Types
    .. method:: Enum.__dir__(self)
 
       Returns ``['__class__', '__doc__', '__module__', 'name', 'value']`` and
-      any public methods defined on *self.__class__*::
+      any public methods defined on ``self.__class__`` or a mixin class::
 
          >>> from datetime import date
          >>> class Weekday(Enum):
diff --git a/Lib/enum.py b/Lib/enum.py
index 461d276eed862..8efc38c3d78db 100644
--- a/Lib/enum.py
+++ b/Lib/enum.py
@@ -635,10 +635,60 @@ def __delattr__(cls, attr):
         super().__delattr__(attr)
 
     def __dir__(self):
-        return (
-                ['__class__', '__doc__', '__members__', '__module__']
-                + self._member_names_
-                )
+        # Start off with the desired result for dir(Enum)
+        cls_dir = {'__class__', '__doc__', '__members__', '__module__'}
+        add_to_dir = cls_dir.add
+        mro = self.__mro__
+        this_module = globals().values()
+        is_from_this_module = lambda cls: any(cls is thing for thing in this_module)
+        first_enum_base = next(cls for cls in mro if is_from_this_module(cls))
+        enum_dict = Enum.__dict__
+        sentinel = object()
+        # special-case __new__
+        ignored = {'__new__', *filter(_is_sunder, enum_dict)}
+        add_to_ignored = ignored.add
+
+        # We want these added to __dir__
+        # if and only if they have been user-overridden
+        enum_dunders = set(filter(_is_dunder, enum_dict))
+
+        # special-case __new__
+        if self.__new__ is not first_enum_base.__new__:
+            add_to_dir('__new__')
+
+        for cls in mro:
+            # Ignore any classes defined in this module
+            if cls is object or is_from_this_module(cls):
+                continue
+
+            cls_lookup = cls.__dict__
+
+            # If not an instance of EnumType,
+            # ensure all attributes excluded from that class's `dir()` are ignored here.
+            if not isinstance(cls, EnumType):
+                cls_lookup = set(cls_lookup).intersection(dir(cls))
+
+            for attr_name in cls_lookup:
+                # Already seen it? Carry on
+                if attr_name in cls_dir or attr_name in ignored:
+                    continue
+                # Sunders defined in Enum.__dict__ are already in `ignored`,
+                # But sunders defined in a subclass won't be (we want all sunders excluded).
+                elif _is_sunder(attr_name):
+                    add_to_ignored(attr_name)
+                # Not an "enum dunder"? Add it to dir() output.
+                elif attr_name not in enum_dunders:
+                    add_to_dir(attr_name)
+                # Is an "enum dunder", and is defined by a class from enum.py? Ignore it.
+                elif getattr(self, attr_name) is getattr(first_enum_base, attr_name, sentinel):
+                    add_to_ignored(attr_name)
+                # Is an "enum dunder", and is either user-defined or defined by a mixin class?
+                # Add it to dir() output.
+                else:
+                    add_to_dir(attr_name)
+
+        # sort the output before returning it, so that the result is deterministic.
+        return sorted(cls_dir)
 
     def __getattr__(cls, name):
         """
@@ -985,13 +1035,10 @@ def __dir__(self):
         """
         Returns all members and all public methods
         """
-        added_behavior = [
-                m
-                for cls in self.__class__.mro()
-                for m in cls.__dict__
-                if m[0] != '_' and m not in self._member_map_
-                ] + [m for m in self.__dict__ if m[0] != '_']
-        return (['__class__', '__doc__', '__module__'] + added_behavior)
+        cls = type(self)
+        to_exclude = {'__members__', '__init__', '__new__', *cls._member_names_}
+        filtered_self_dict = (name for name in self.__dict__ if not name.startswith('_'))
+        return sorted({'name', 'value', *dir(cls), *filtered_self_dict} - to_exclude)
 
     def __format__(self, format_spec):
         """
diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py
index 7d220871a35cc..eecb9fd4835c4 100644
--- a/Lib/test/test_enum.py
+++ b/Lib/test/test_enum.py
@@ -203,59 +203,340 @@ class Holiday(date, Enum):
             IDES_OF_MARCH = 2013, 3, 15
         self.Holiday = Holiday
 
-    def test_dir_on_class(self):
-        Season = self.Season
-        self.assertEqual(
-            set(dir(Season)),
-            set(['__class__', '__doc__', '__members__', '__module__',
-                'SPRING', 'SUMMER', 'AUTUMN', 'WINTER']),
-            )
+        class DateEnum(date, Enum): pass
+        self.DateEnum = DateEnum
 
-    def test_dir_on_item(self):
-        Season = self.Season
-        self.assertEqual(
-            set(dir(Season.WINTER)),
-            set(['__class__', '__doc__', '__module__', 'name', 'value']),
-            )
+        class FloatEnum(float, Enum): pass
+        self.FloatEnum = FloatEnum
 
-    def test_dir_with_added_behavior(self):
-        class Test(Enum):
+        class Wowser(Enum):
             this = 'that'
             these = 'those'
             def wowser(self):
+                """Wowser docstring"""
                 return ("Wowser! I'm %s!" % self.name)
-        self.assertEqual(
-                set(dir(Test)),
-                set(['__class__', '__doc__', '__members__', '__module__', 'this', 'these']),
-                )
-        self.assertEqual(
-                set(dir(Test.this)),
-                set(['__class__', '__doc__', '__module__', 'name', 'value', 'wowser']),
-                )
+            @classmethod
+            def classmethod_wowser(cls): pass
+            @staticmethod
+            def staticmethod_wowser(): pass
+        self.Wowser = Wowser
+
+        class IntWowser(IntEnum):
+            this = 1
+            these = 2
+            def wowser(self):
+                """Wowser docstring"""
+                return ("Wowser! I'm %s!" % self.name)
+            @classmethod
+            def classmethod_wowser(cls): pass
+            @staticmethod
+            def staticmethod_wowser(): pass
+        self.IntWowser = IntWowser
+
+        class FloatWowser(float, Enum):
+            this = 3.14
+            these = 4.2
+            def wowser(self):
+                """Wowser docstring"""
+                return ("Wowser! I'm %s!" % self.name)
+            @classmethod
+            def classmethod_wowser(cls): pass
+            @staticmethod
+            def staticmethod_wowser(): pass
+        self.FloatWowser = FloatWowser
+
+        class WowserNoMembers(Enum):
+            def wowser(self): pass
+            @classmethod
+            def classmethod_wowser(cls): pass
+            @staticmethod
+            def staticmethod_wowser(): pass
+        class SubclassOfWowserNoMembers(WowserNoMembers): pass
+        self.WowserNoMembers = WowserNoMembers
+        self.SubclassOfWowserNoMembers = SubclassOfWowserNoMembers
+
+        class IntWowserNoMembers(IntEnum):
+            def wowser(self): pass
+            @classmethod
+            def classmethod_wowser(cls): pass
+            @staticmethod
+            def staticmethod_wowser(): pass
+        self.IntWowserNoMembers = IntWowserNoMembers
+
+        class FloatWowserNoMembers(float, Enum):
+            def wowser(self): pass
+            @classmethod
+            def classmethod_wowser(cls): pass
+            @staticmethod
+            def staticmethod_wowser(): pass
+        self.FloatWowserNoMembers = FloatWowserNoMembers
+
+        class EnumWithInit(Enum):
+            def __init__(self, greeting, farewell):
+                self.greeting = greeting
+                self.farewell = farewell
+            ENGLISH = 'hello', 'goodbye'
+            GERMAN = 'Guten Morgen', 'Auf Wiedersehen'
+            def some_method(self): pass
+        self.EnumWithInit = EnumWithInit
 
-    def test_dir_on_sub_with_behavior_on_super(self):
         # see issue22506
-        class SuperEnum(Enum):
+        class SuperEnum1(Enum):
             def invisible(self):
                 return "did you see me?"
-        class SubEnum(SuperEnum):
+        class SubEnum1(SuperEnum1):
             sample = 5
-        self.assertEqual(
-                set(dir(SubEnum.sample)),
-                set(['__class__', '__doc__', '__module__', 'name', 'value', 'invisible']),
-                )
+        self.SubEnum1 = SubEnum1
 
-    def test_dir_on_sub_with_behavior_including_instance_dict_on_super(self):
-        # see issue40084
-        class SuperEnum(IntEnum):
+        class SuperEnum2(IntEnum):
             def __new__(cls, value, description=""):
                 obj = int.__new__(cls, value)
                 obj._value_ = value
                 obj.description = description
                 return obj
-        class SubEnum(SuperEnum):
+        class SubEnum2(SuperEnum2):
             sample = 5
-        self.assertTrue({'description'} <= set(dir(SubEnum.sample)))
+        self.SubEnum2 = SubEnum2
+
+    def test_dir_basics_for_all_enums(self):
+        enums_for_tests = (
+            # Generic enums in enum.py
+            Enum,
+            IntEnum,
+            StrEnum,
+            # Generic enums defined outside of enum.py
+            self.DateEnum,
+            self.FloatEnum,
+            # Concrete enums derived from enum.py generics
+            self.Grades,
+            self.Season,
+            # Concrete enums derived from generics defined outside of enum.py
+            self.Konstants,
+            self.Holiday,
+            # Standard enum with added behaviour & members
+            self.Wowser,
+            # Mixin-enum-from-enum.py with added behaviour & members
+            self.IntWowser,
+            # Mixin-enum-from-oustide-enum.py with added behaviour & members
+            self.FloatWowser,
+            # Equivalents of the three immediately above, but with no members
+            self.WowserNoMembers,
+            self.IntWowserNoMembers,
+            self.FloatWowserNoMembers,
+            # Enum with members and an __init__ method
+            self.EnumWithInit,
+            # Special cases to test
+            self.SubEnum1,
+            self.SubEnum2
+        )
+
+        for cls in enums_for_tests:
+            with self.subTest(cls=cls):
+                cls_dir = dir(cls)
+                # test that dir is deterministic
+                self.assertEqual(cls_dir, dir(cls))
+                # test that dir is sorted
+                self.assertEqual(list(cls_dir), sorted(cls_dir))
+                # test that there are no dupes in dir
+                self.assertEqual(len(cls_dir), len(set(cls_dir)))
+                # test that there are no sunders in dir
+                self.assertFalse(any(enum._is_sunder(attr) for attr in cls_dir))
+                self.assertNotIn('__new__', cls_dir)
+
+                for attr in ('__class__', '__doc__', '__members__', '__module__'):
+                    with self.subTest(attr=attr):
+                        self.assertIn(attr, cls_dir)
+
+    def test_dir_for_enum_with_members(self):
+        enums_for_test = (
+            # Enum with members
+            self.Season,
+            # IntEnum with members
+            self.Grades,
+            # Two custom-mixin enums with members
+            self.Konstants,
+            self.Holiday,
+            # several enums-with-added-behaviour and members
+            self.Wowser,
+            self.IntWowser,
+            self.FloatWowser,
+            # An enum with an __init__ method and members
+            self.EnumWithInit,
+            # Special cases to test
+            self.SubEnum1,
+            self.SubEnum2
+        )
+
+        for cls in enums_for_test:
+            cls_dir = dir(cls)
+            member_names = cls._member_names_
+            with self.subTest(cls=cls):
+                self.assertTrue(all(member_name in cls_dir for member_name in member_names))
+                for member in cls:
+                    member_dir = dir(member)
+                    # test that dir is deterministic
+                    self.assertEqual(member_dir, dir(member))
+                    # test that dir is sorted
+                    self.assertEqual(list(member_dir), sorted(member_dir))
+                    # test that there are no dupes in dir
+                    self.assertEqual(len(member_dir), len(set(member_dir)))
+
+                    for attr_name in cls_dir:
+                        with self.subTest(attr_name=attr_name):
+                            if attr_name in {'__members__', '__init__', '__new__', *member_names}:
+                                self.assertNotIn(attr_name, member_dir)
+                            else:
+                                self.assertIn(attr_name, member_dir)
+
+                    self.assertFalse(any(enum._is_sunder(attr) for attr in member_dir))
+
+    def test_dir_for_enums_with_added_behaviour(self):
+        enums_for_test = (
+            self.Wowser,
+            self.IntWowser,
+            self.FloatWowser,
+            self.WowserNoMembers,
+            self.SubclassOfWowserNoMembers,
+            self.IntWowserNoMembers,
+            self.FloatWowserNoMembers
+        )
+
+        for cls in enums_for_test:
+            with self.subTest(cls=cls):
+                self.assertIn('wowser', dir(cls))
+                self.assertIn('classmethod_wowser', dir(cls))
+                self.assertIn('staticmethod_wowser', dir(cls))
+                self.assertTrue(all(
+                    all(attr in dir(member) for attr in ('wowser', 'classmethod_wowser', 'staticmethod_wowser'))
+                    for member in cls
+                ))
+
+        self.assertEqual(dir(self.WowserNoMembers), dir(self.SubclassOfWowserNoMembers))
+        # Check classmethods are present
+        self.assertIn('from_bytes', dir(self.IntWowser))
+        self.assertIn('from_bytes', dir(self.IntWowserNoMembers))
+
+    def test_help_output_on_enum_members(self):
+        added_behaviour_enums = (
+            self.Wowser,
+            self.IntWowser,
+            self.FloatWowser
+        )
+
+        for cls in added_behaviour_enums:
+            with self.subTest(cls=cls):
+                rendered_doc = pydoc.render_doc(cls.this)
+                self.assertIn('Wowser docstring', rendered_doc)
+                if cls in {self.IntWowser, self.FloatWowser}:
+                    self.assertIn('float(self)', rendered_doc)
+
+    def test_dir_for_enum_with_init(self):
+        EnumWithInit = self.EnumWithInit
+
+        cls_dir = dir(EnumWithInit)
+        self.assertIn('__init__', cls_dir)
+        self.assertIn('some_method', cls_dir)
+        self.assertNotIn('greeting', cls_dir)
+        self.assertNotIn('farewell', cls_dir)
+
+        member_dir = dir(EnumWithInit.ENGLISH)
+        self.assertNotIn('__init__', member_dir)
+        self.assertIn('some_method', member_dir)
+        self.assertIn('greeting', member_dir)
+        self.assertIn('farewell', member_dir)
+
+    def test_mixin_dirs(self):
+        from datetime import date
+
+        enums_for_test = (
+            # generic mixins from enum.py
+            (IntEnum, int),
+            (StrEnum, str),
+            # generic mixins from outside enum.py
+            (self.FloatEnum, float),
+            (self.DateEnum, date),
+            # concrete mixin from enum.py
+            (self.Grades, int),
+            # concrete mixin from outside enum.py
+            (self.Holiday, date),
+            # concrete mixin from enum.py with added behaviour
+            (self.IntWowser, int),
+            # concrete mixin from outside enum.py with added behaviour
+            (self.FloatWowser, float)
+        )
+
+        enum_dict = Enum.__dict__
+        enum_dir = dir(Enum)
+        enum_module_names = enum.__all__
+        is_from_enum_module = lambda cls: cls.__name__ in enum_module_names
+        is_enum_dunder = lambda attr: enum._is_dunder(attr) and attr in enum_dict
+
+        def attr_is_inherited_from_object(cls, attr_name):
+            for base in cls.__mro__:
+                if attr_name in base.__dict__:
+                    return base is object
+            return False
+
+        # General tests
+        for enum_cls, mixin_cls in enums_for_test:
+            with self.subTest(enum_cls=enum_cls):
+                cls_dir = dir(enum_cls)
+                cls_dict = enum_cls.__dict__
+
+                mixin_attrs = [
+                    x for x in dir(mixin_cls)
+                    if not attr_is_inherited_from_object(cls=mixin_cls, attr_name=x)
+                ]
+
+                first_enum_base = next(
+                    base for base in enum_cls.__mro__
+                    if is_from_enum_module(base)
+                )
+
+                for attr in mixin_attrs:
+                    with self.subTest(attr=attr):
+                        if enum._is_sunder(attr):
+                            # Unlikely, but no harm in testing
+                            self.assertNotIn(attr, cls_dir)
+                        elif attr in {'__class__', '__doc__', '__members__', '__module__'}:
+                            self.assertIn(attr, cls_dir)
+                        elif is_enum_dunder(attr):
+                            if is_from_enum_module(enum_cls):
+                                self.assertNotIn(attr, cls_dir)
+                            elif getattr(enum_cls, attr) is getattr(first_enum_base, attr):
+                                self.assertNotIn(attr, cls_dir)
+                            else:
+                                self.assertIn(attr, cls_dir)
+                        else:
+                            self.assertIn(attr, cls_dir)
+
+        # Some specific examples
+        int_enum_dir = dir(IntEnum)
+        self.assertIn('imag', int_enum_dir)
+        self.assertIn('__rfloordiv__', int_enum_dir)
+        self.assertNotIn('__format__', int_enum_dir)
+        self.assertNotIn('__hash__', int_enum_dir)
+        self.assertNotIn('__init_subclass__', int_enum_dir)
+        self.assertNotIn('__subclasshook__', int_enum_dir)
+
+        class OverridesFormatOutsideEnumModule(Enum):
+            def __format__(self, *args, **kwargs):
+                return super().__format__(*args, **kwargs)
+            SOME_MEMBER = 1
+
+        self.assertIn('__format__', dir(OverridesFormatOutsideEnumModule))
+        self.assertIn('__format__', dir(OverridesFormatOutsideEnumModule.SOME_MEMBER))
+
+    def test_dir_on_sub_with_behavior_on_super(self):
+        # see issue22506
+        self.assertEqual(
+                set(dir(self.SubEnum1.sample)),
+                set(['__class__', '__doc__', '__module__', 'name', 'value', 'invisible']),
+                )
+
+    def test_dir_on_sub_with_behavior_including_instance_dict_on_super(self):
+        # see issue40084
+        self.assertTrue({'description'} <= set(dir(self.SubEnum2.sample)))
 
     def test_enum_in_enum_out(self):
         Season = self.Season
@@ -4156,7 +4437,8 @@ def test_convert(self):
         self.assertEqual(test_type.CONVERT_TEST_NAME_E, 5)
         # Ensure that test_type only picked up names matching the filter.
         self.assertEqual([name for name in dir(test_type)
-                          if name[0:2] not in ('CO', '__')],
+                          if name[0:2] not in ('CO', '__')
+                          and name not in dir(IntEnum)],
                          [], msg='Names other than CONVERT_TEST_* found.')
 
     @unittest.skipUnless(python_version == (3, 8),
@@ -4207,7 +4489,8 @@ def test_convert(self):
         self.assertEqual(test_type.CONVERT_STR_TEST_2, 'goodbye')
         # Ensure that test_type only picked up names matching the filter.
         self.assertEqual([name for name in dir(test_type)
-                          if name[0:2] not in ('CO', '__')],
+                          if name[0:2] not in ('CO', '__')
+                          and name not in dir(StrEnum)],
                          [], msg='Names other than CONVERT_STR_* found.')
 
     def test_convert_repr_and_str(self):
diff --git a/Misc/NEWS.d/next/Library/2021-10-29-16-28-06.bpo-45535.n8NiOE.rst b/Misc/NEWS.d/next/Library/2021-10-29-16-28-06.bpo-45535.n8NiOE.rst
new file mode 100644
index 0000000000000..bda1b407a0ee0
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-29-16-28-06.bpo-45535.n8NiOE.rst
@@ -0,0 +1 @@
+Improve output of ``dir()`` with Enums.


From webhook-mailer at python.org  Thu Dec  2 12:43:06 2021
From: webhook-mailer at python.org (tiran)
Date: Thu, 02 Dec 2021 17:43:06 -0000
Subject: [Python-checkins] bpo-40280: Update what's new (GH-29893)
Message-ID: <mailman.19.1638466987.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/a31173c5ceb1708df687f942d714bdecae7cb759
commit: a31173c5ceb1708df687f942d714bdecae7cb759
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-02T18:42:56+01:00
summary:

bpo-40280: Update what's new (GH-29893)

files:
M Doc/whatsnew/3.11.rst

diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index 345b2df43f403..1ec629d8229cb 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -585,6 +585,11 @@ Build Changes
   ``libdb`` 3.x and 4.x are no longer supported.
   (Contributed by Christian Heimes in :issue:`45747`.)
 
+* CPython now has experimental support for cross compiling to WebAssembly
+  platform ``wasm32-emscripten``. The effort is inspired by previous work
+  like Pyodide.
+  (Contributed by Christian Heimes and Ethan Smith in :issue:`40280`.)
+
 C API Changes
 =============
 


From webhook-mailer at python.org  Thu Dec  2 15:24:37 2021
From: webhook-mailer at python.org (zware)
Date: Thu, 02 Dec 2021 20:24:37 -0000
Subject: [Python-checkins] bpo-45916: Use HTTPS link for The Perils of
 Floating Point (GH-29896)
Message-ID: <mailman.20.1638476678.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/9f2f7e42269db74a89fc8cd74d82a875787f01d7
commit: 9f2f7e42269db74a89fc8cd74d82a875787f01d7
branch: main
author: Zachary Ware <zach at python.org>
committer: zware <zachary.ware at gmail.com>
date: 2021-12-02T14:24:25-06:00
summary:

bpo-45916: Use HTTPS link for The Perils of Floating Point (GH-29896)

files:
M Doc/tutorial/floatingpoint.rst

diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst
index b98de6e56a003..7212b40be8377 100644
--- a/Doc/tutorial/floatingpoint.rst
+++ b/Doc/tutorial/floatingpoint.rst
@@ -133,7 +133,7 @@ with inexact values become comparable to one another::
 
 Binary floating-point arithmetic holds many surprises like this.  The problem
 with "0.1" is explained in precise detail below, in the "Representation Error"
-section.  See `The Perils of Floating Point <http://www.lahey.com/float.htm>`_
+section.  See `The Perils of Floating Point <https://www.lahey.com/float.htm>`_
 for a more complete account of other common surprises.
 
 As that says near the end, "there are no easy answers."  Still, don't be unduly


From webhook-mailer at python.org  Thu Dec  2 15:45:21 2021
From: webhook-mailer at python.org (miss-islington)
Date: Thu, 02 Dec 2021 20:45:21 -0000
Subject: [Python-checkins] bpo-45916: Use HTTPS link for The Perils of
 Floating Point (GH-29896)
Message-ID: <mailman.21.1638477921.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/f6648e229edf07a1e4897244d7d34989dd9ea647
commit: f6648e229edf07a1e4897244d7d34989dd9ea647
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-02T12:45:09-08:00
summary:

bpo-45916: Use HTTPS link for The Perils of Floating Point (GH-29896)

(cherry picked from commit 9f2f7e42269db74a89fc8cd74d82a875787f01d7)

Co-authored-by: Zachary Ware <zach at python.org>

files:
M Doc/tutorial/floatingpoint.rst

diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst
index b98de6e56a003..7212b40be8377 100644
--- a/Doc/tutorial/floatingpoint.rst
+++ b/Doc/tutorial/floatingpoint.rst
@@ -133,7 +133,7 @@ with inexact values become comparable to one another::
 
 Binary floating-point arithmetic holds many surprises like this.  The problem
 with "0.1" is explained in precise detail below, in the "Representation Error"
-section.  See `The Perils of Floating Point <http://www.lahey.com/float.htm>`_
+section.  See `The Perils of Floating Point <https://www.lahey.com/float.htm>`_
 for a more complete account of other common surprises.
 
 As that says near the end, "there are no easy answers."  Still, don't be unduly


From webhook-mailer at python.org  Thu Dec  2 19:08:51 2021
From: webhook-mailer at python.org (zooba)
Date: Fri, 03 Dec 2021 00:08:51 -0000
Subject: [Python-checkins] bpo-45582: Port getpath[p].c to Python (GH-29041)
Message-ID: <mailman.22.1638490132.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/99fcf1505218464c489d419d4500f126b6d6dc28
commit: 99fcf1505218464c489d419d4500f126b6d6dc28
branch: main
author: Steve Dower <steve.dower at python.org>
committer: zooba <steve.dower at microsoft.com>
date: 2021-12-03T00:08:42Z
summary:

bpo-45582: Port getpath[p].c to Python (GH-29041)

The getpath.py file is frozen at build time and executed as code over a namespace. It is never imported, nor is it meant to be importable or reusable. However, it should be easier to read, modify, and patch than the previous code.

This commit attempts to preserve every previously tested quirk, but these may be changed in the future to better align platforms.

files:
A Lib/test/test_getpath.py
A Misc/NEWS.d/next/Core and Builtins/2021-10-23-00-39-31.bpo-45582.YONPuo.rst
A Modules/getpath.py
A Modules/getpath_noop.c
D PC/getpathp.c
M .gitignore
M Doc/c-api/init_config.rst
M Include/cpython/fileutils.h
M Include/cpython/initconfig.h
M Include/internal/pycore_fileutils.h
M Include/internal/pycore_initconfig.h
M Include/internal/pycore_pathconfig.h
M Lib/ntpath.py
M Lib/posixpath.py
M Lib/site.py
M Lib/sysconfig.py
M Lib/test/_test_embed_set_config.py
M Lib/test/test_embed.py
M Lib/test/test_ntpath.py
M Lib/test/test_site.py
M Lib/test/test_sysconfig.py
M Lib/test/test_venv.py
M Makefile.pre.in
M Modules/_testinternalcapi.c
M Modules/clinic/posixmodule.c.h
M Modules/getpath.c
M Modules/main.c
M Modules/posixmodule.c
M PC/config_minimal.c
M PC/pyconfig.h
M PCbuild/_freeze_module.vcxproj
M PCbuild/_freeze_module.vcxproj.filters
M PCbuild/python.vcxproj
M PCbuild/pythoncore.vcxproj
M PCbuild/pythoncore.vcxproj.filters
M Python/dynload_win.c
M Python/fileutils.c
M Python/initconfig.c
M Python/pathconfig.c
M Python/pylifecycle.c

diff --git a/.gitignore b/.gitignore
index 0363244bdaf63..7e80446d4d4a6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -74,6 +74,7 @@ Mac/pythonw
 Misc/python.pc
 Misc/python-embed.pc
 Misc/python-config.sh
+Modules/getpath.h
 Modules/Setup.config
 Modules/Setup.local
 Modules/Setup.stdlib
diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
index 3642a8bf7fbc7..7c8fb7b1dee5d 100644
--- a/Doc/c-api/init_config.rst
+++ b/Doc/c-api/init_config.rst
@@ -479,6 +479,9 @@ PyConfig
 
       Fields which are already initialized are left unchanged.
 
+      Fields for :ref:`path configuration <init-path-config>` are no longer
+      calculated or modified when calling this function, as of Python 3.11.
+
       The :c:func:`PyConfig_Read` function only parses
       :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv`
       is set to ``2`` after arguments are parsed. Since Python arguments are
@@ -493,6 +496,12 @@ PyConfig
          parsed, and arguments are only parsed if
          :c:member:`PyConfig.parse_argv` equals ``1``.
 
+      .. versionchanged:: 3.11
+         :c:func:`PyConfig_Read` no longer calculates all paths, and so fields
+         listed under :ref:`Python Path Configuration <init-path-config>` may
+         no longer be updated until :c:func:`Py_InitializeFromConfig` is
+         called.
+
    .. c:function:: void PyConfig_Clear(PyConfig *config)
 
       Release configuration memory.
@@ -848,12 +857,19 @@ PyConfig
 
       Default: value of the ``PLATLIBDIR`` macro which is set by the
       :option:`configure --with-platlibdir option <--with-platlibdir>`
-      (default: ``"lib"``).
+      (default: ``"lib"``, or ``"DLLs"`` on Windows).
 
       Part of the :ref:`Python Path Configuration <init-path-config>` input.
 
       .. versionadded:: 3.9
 
+      .. versionchanged:: 3.11
+         This macro is now used on Windows to locate the standard
+         library extension modules, typically under ``DLLs``. However,
+         for compatibility, note that this value is ignored for any
+         non-standard layouts, including in-tree builds and virtual
+         environments.
+
    .. c:member:: wchar_t* pythonpath_env
 
       Module search paths (:data:`sys.path`) as a string separated by ``DELIM``
@@ -870,9 +886,9 @@ PyConfig
 
       Module search paths: :data:`sys.path`.
 
-      If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, the
-      function calculating the :ref:`Python Path Configuration <init-path-config>`
-      overrides the :c:member:`~PyConfig.module_search_paths` and sets
+      If :c:member:`~PyConfig.module_search_paths_set` is equal to 0,
+      :c:func:`Py_InitializeFromConfig` will replace
+      :c:member:`~PyConfig.module_search_paths` and sets
       :c:member:`~PyConfig.module_search_paths_set` to ``1``.
 
       Default: empty list (``module_search_paths``) and ``0``
@@ -944,16 +960,16 @@ PyConfig
 
    .. c:member:: int pathconfig_warnings
 
-      On Unix, if non-zero, calculating the :ref:`Python Path Configuration
-      <init-path-config>` can log warnings into ``stderr``. If equals to 0,
-      suppress these warnings.
-
-      It has no effect on Windows.
+      If non-zero, calculation of path configuration is allowed to log
+      warnings into ``stderr``. If equals to 0, suppress these warnings.
 
       Default: ``1`` in Python mode, ``0`` in isolated mode.
 
       Part of the :ref:`Python Path Configuration <init-path-config>` input.
 
+      .. versionchanged:: 3.11
+         Now also applies on Windows.
+
    .. c:member:: wchar_t* prefix
 
       The site-specific directory prefix where the platform independent Python
@@ -1305,10 +1321,9 @@ variables, command line arguments (:c:member:`PyConfig.argv` is not parsed)
 and user site directory. The C standard streams (ex: ``stdout``) and the
 LC_CTYPE locale are left unchanged. Signal handlers are not installed.
 
-Configuration files are still used with this configuration. Set the
-:ref:`Python Path Configuration <init-path-config>` ("output fields") to ignore these
-configuration files and avoid the function computing the default path
-configuration.
+Configuration files are still used with this configuration to determine
+paths that are unspecified. Ensure :c:member:`PyConfig.home` is specified
+to avoid computing the default path configuration.
 
 
 .. _init-python-config:
diff --git a/Include/cpython/fileutils.h b/Include/cpython/fileutils.h
index ccf37e9468d61..7ab2290184a2f 100644
--- a/Include/cpython/fileutils.h
+++ b/Include/cpython/fileutils.h
@@ -135,12 +135,6 @@ PyAPI_FUNC(wchar_t*) _Py_wrealpath(
     size_t resolved_path_len);
 #endif
 
-#ifndef MS_WINDOWS
-PyAPI_FUNC(int) _Py_isabs(const wchar_t *path);
-#endif
-
-PyAPI_FUNC(int) _Py_abspath(const wchar_t *path, wchar_t **abspath_p);
-
 PyAPI_FUNC(wchar_t*) _Py_wgetcwd(
     wchar_t *buf,
     /* Number of characters of 'buf' buffer
diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h
index 2be4c25ec0b09..2ba1224d9b0b0 100644
--- a/Include/cpython/initconfig.h
+++ b/Include/cpython/initconfig.h
@@ -213,6 +213,9 @@ typedef struct PyConfig {
     // If non-zero, disallow threads, subprocesses, and fork.
     // Default: 0.
     int _isolated_interpreter;
+
+    // If non-zero, we believe we're running from a source tree.
+    int _is_python_build;
 } PyConfig;
 
 PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config);
diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h
index d1caf9c237234..38df73b745bcb 100644
--- a/Include/internal/pycore_fileutils.h
+++ b/Include/internal/pycore_fileutils.h
@@ -74,14 +74,15 @@ extern int _Py_EncodeNonUnicodeWchar_InPlace(
     Py_ssize_t size);
 #endif
 
+extern int _Py_isabs(const wchar_t *path);
+extern int _Py_abspath(const wchar_t *path, wchar_t **abspath_p);
 extern wchar_t * _Py_join_relfile(const wchar_t *dirname,
                                   const wchar_t *relfile);
 extern int _Py_add_relfile(wchar_t *dirname,
                            const wchar_t *relfile,
                            size_t bufsize);
 extern size_t _Py_find_basename(const wchar_t *filename);
-PyAPI_FUNC(int) _Py_normalize_path(const wchar_t *path,
-                                   wchar_t *buf, const size_t buf_len);
+PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size);
 
 
 // Macros to protect CRT calls against instant termination when passed an
diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h
index 9014fcd41d868..be545f495dc0f 100644
--- a/Include/internal/pycore_initconfig.h
+++ b/Include/internal/pycore_initconfig.h
@@ -166,6 +166,10 @@ extern PyStatus _PyConfig_SetPyArgv(
 PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config);
 PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict);
 
+extern void _Py_DumpPathConfig(PyThreadState *tstate);
+
+PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject();
+
 
 /* --- Function used for testing ---------------------------------- */
 
diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h
index a258aab239766..b8deaa0c3eb06 100644
--- a/Include/internal/pycore_pathconfig.h
+++ b/Include/internal/pycore_pathconfig.h
@@ -8,65 +8,15 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-typedef struct _PyPathConfig {
-    /* Full path to the Python program */
-    wchar_t *program_full_path;
-    wchar_t *prefix;
-    wchar_t *exec_prefix;
-    wchar_t *stdlib_dir;
-    /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */
-    wchar_t *module_search_path;
-    /* Python program name */
-    wchar_t *program_name;
-    /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */
-    wchar_t *home;
-#ifdef MS_WINDOWS
-    /* isolated and site_import are used to set Py_IsolatedFlag and
-       Py_NoSiteFlag flags on Windows in read_pth_file(). These fields
-       are ignored when their value are equal to -1 (unset). */
-    int isolated;
-    int site_import;
-    /* Set when a venv is detected */
-    wchar_t *base_executable;
-#endif
-} _PyPathConfig;
-
-#ifdef MS_WINDOWS
-#  define _PyPathConfig_INIT \
-      {.module_search_path = NULL, \
-       .isolated = -1, \
-       .site_import = -1}
-#else
-#  define _PyPathConfig_INIT \
-      {.module_search_path = NULL}
-#endif
-/* Note: _PyPathConfig_INIT sets other fields to 0/NULL */
-
-PyAPI_DATA(_PyPathConfig) _Py_path_config;
-#ifdef MS_WINDOWS
-PyAPI_DATA(wchar_t*) _Py_dll_path;
-#endif
-
-extern void _PyPathConfig_ClearGlobal(void);
+PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void);
+extern PyStatus _PyPathConfig_ReadGlobal(PyConfig *config);
+extern PyStatus _PyPathConfig_UpdateGlobal(const PyConfig *config);
+extern const wchar_t * _PyPathConfig_GetGlobalModuleSearchPath(void);
 
-extern PyStatus _PyPathConfig_Calculate(
-    _PyPathConfig *pathconfig,
-    const PyConfig *config);
 extern int _PyPathConfig_ComputeSysPath0(
     const PyWideStringList *argv,
     PyObject **path0);
-extern PyStatus _Py_FindEnvConfigValue(
-    FILE *env_file,
-    const wchar_t *key,
-    wchar_t **value_p);
-
-#ifdef MS_WINDOWS
-extern wchar_t* _Py_GetDLLPath(void);
-#endif
 
-extern PyStatus _PyConfig_WritePathConfig(const PyConfig *config);
-extern void _Py_DumpPathConfig(PyThreadState *tstate);
-extern PyObject* _PyPathConfig_AsDict(void);
 
 #ifdef __cplusplus
 }
diff --git a/Lib/ntpath.py b/Lib/ntpath.py
index 527c7ae1938fb..58483a0c0a98b 100644
--- a/Lib/ntpath.py
+++ b/Lib/ntpath.py
@@ -70,7 +70,7 @@ def isabs(s):
         if s.replace('/', '\\').startswith('\\\\?\\'):
             return True
     s = splitdrive(s)[1]
-    return len(s) > 0 and s[0] in _get_bothseps(s)
+    return len(s) > 0 and s[0] and s[0] in _get_bothseps(s)
 
 
 # Join two (or more) paths.
@@ -268,11 +268,13 @@ def ismount(path):
     root, rest = splitdrive(path)
     if root and root[0] in seps:
         return (not rest) or (rest in seps)
-    if rest in seps:
+    if rest and rest in seps:
         return True
 
     if _getvolumepathname:
-        return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps)
+        x = path.rstrip(seps)
+        y =_getvolumepathname(path).rstrip(seps)
+        return x.casefold() == y.casefold()
     else:
         return False
 
@@ -459,56 +461,68 @@ def expandvars(path):
 # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B.
 # Previously, this function also truncated pathnames to 8+3 format,
 # but as this module is called "ntpath", that's obviously wrong!
+try:
+    from nt import _path_normpath
 
-def normpath(path):
-    """Normalize path, eliminating double slashes, etc."""
-    path = os.fspath(path)
-    if isinstance(path, bytes):
-        sep = b'\\'
-        altsep = b'/'
-        curdir = b'.'
-        pardir = b'..'
-        special_prefixes = (b'\\\\.\\', b'\\\\?\\')
-    else:
-        sep = '\\'
-        altsep = '/'
-        curdir = '.'
-        pardir = '..'
-        special_prefixes = ('\\\\.\\', '\\\\?\\')
-    if path.startswith(special_prefixes):
-        # in the case of paths with these prefixes:
-        # \\.\ -> device names
-        # \\?\ -> literal paths
-        # do not do any normalization, but return the path
-        # unchanged apart from the call to os.fspath()
-        return path
-    path = path.replace(altsep, sep)
-    prefix, path = splitdrive(path)
-
-    # collapse initial backslashes
-    if path.startswith(sep):
-        prefix += sep
-        path = path.lstrip(sep)
-
-    comps = path.split(sep)
-    i = 0
-    while i < len(comps):
-        if not comps[i] or comps[i] == curdir:
-            del comps[i]
-        elif comps[i] == pardir:
-            if i > 0 and comps[i-1] != pardir:
-                del comps[i-1:i+1]
-                i -= 1
-            elif i == 0 and prefix.endswith(sep):
+except ImportError:
+    def normpath(path):
+        """Normalize path, eliminating double slashes, etc."""
+        path = os.fspath(path)
+        if isinstance(path, bytes):
+            sep = b'\\'
+            altsep = b'/'
+            curdir = b'.'
+            pardir = b'..'
+            special_prefixes = (b'\\\\.\\', b'\\\\?\\')
+        else:
+            sep = '\\'
+            altsep = '/'
+            curdir = '.'
+            pardir = '..'
+            special_prefixes = ('\\\\.\\', '\\\\?\\')
+        if path.startswith(special_prefixes):
+            # in the case of paths with these prefixes:
+            # \\.\ -> device names
+            # \\?\ -> literal paths
+            # do not do any normalization, but return the path
+            # unchanged apart from the call to os.fspath()
+            return path
+        path = path.replace(altsep, sep)
+        prefix, path = splitdrive(path)
+
+        # collapse initial backslashes
+        if path.startswith(sep):
+            prefix += sep
+            path = path.lstrip(sep)
+
+        comps = path.split(sep)
+        i = 0
+        while i < len(comps):
+            if not comps[i] or comps[i] == curdir:
                 del comps[i]
+            elif comps[i] == pardir:
+                if i > 0 and comps[i-1] != pardir:
+                    del comps[i-1:i+1]
+                    i -= 1
+                elif i == 0 and prefix.endswith(sep):
+                    del comps[i]
+                else:
+                    i += 1
             else:
                 i += 1
-        else:
-            i += 1
-    # If the path is now empty, substitute '.'
-    if not prefix and not comps:
-        comps.append(curdir)
-    return prefix + sep.join(comps)
+        # If the path is now empty, substitute '.'
+        if not prefix and not comps:
+            comps.append(curdir)
+        return prefix + sep.join(comps)
+
+else:
+    def normpath(path):
+        """Normalize path, eliminating double slashes, etc."""
+        path = os.fspath(path)
+        if isinstance(path, bytes):
+            return os.fsencode(_path_normpath(os.fsdecode(path))) or b"."
+        return _path_normpath(path) or "."
+
 
 def _abspath_fallback(path):
     """Return the absolute version of a path as a fallback function in case
diff --git a/Lib/posixpath.py b/Lib/posixpath.py
index 195374613a779..a46c667db5633 100644
--- a/Lib/posixpath.py
+++ b/Lib/posixpath.py
@@ -334,43 +334,55 @@ def expandvars(path):
 # It should be understood that this may change the meaning of the path
 # if it contains symbolic links!
 
-def normpath(path):
-    """Normalize path, eliminating double slashes, etc."""
-    path = os.fspath(path)
-    if isinstance(path, bytes):
-        sep = b'/'
-        empty = b''
-        dot = b'.'
-        dotdot = b'..'
-    else:
-        sep = '/'
-        empty = ''
-        dot = '.'
-        dotdot = '..'
-    if path == empty:
-        return dot
-    initial_slashes = path.startswith(sep)
-    # POSIX allows one or two initial slashes, but treats three or more
-    # as single slash.
-    # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13)
-    if (initial_slashes and
-        path.startswith(sep*2) and not path.startswith(sep*3)):
-        initial_slashes = 2
-    comps = path.split(sep)
-    new_comps = []
-    for comp in comps:
-        if comp in (empty, dot):
-            continue
-        if (comp != dotdot or (not initial_slashes and not new_comps) or
-             (new_comps and new_comps[-1] == dotdot)):
-            new_comps.append(comp)
-        elif new_comps:
-            new_comps.pop()
-    comps = new_comps
-    path = sep.join(comps)
-    if initial_slashes:
-        path = sep*initial_slashes + path
-    return path or dot
+try:
+    from posix import _path_normpath
+
+except ImportError:
+    def normpath(path):
+        """Normalize path, eliminating double slashes, etc."""
+        path = os.fspath(path)
+        if isinstance(path, bytes):
+            sep = b'/'
+            empty = b''
+            dot = b'.'
+            dotdot = b'..'
+        else:
+            sep = '/'
+            empty = ''
+            dot = '.'
+            dotdot = '..'
+        if path == empty:
+            return dot
+        initial_slashes = path.startswith(sep)
+        # POSIX allows one or two initial slashes, but treats three or more
+        # as single slash.
+        # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13)
+        if (initial_slashes and
+            path.startswith(sep*2) and not path.startswith(sep*3)):
+            initial_slashes = 2
+        comps = path.split(sep)
+        new_comps = []
+        for comp in comps:
+            if comp in (empty, dot):
+                continue
+            if (comp != dotdot or (not initial_slashes and not new_comps) or
+                 (new_comps and new_comps[-1] == dotdot)):
+                new_comps.append(comp)
+            elif new_comps:
+                new_comps.pop()
+        comps = new_comps
+        path = sep.join(comps)
+        if initial_slashes:
+            path = sep*initial_slashes + path
+        return path or dot
+
+else:
+    def normpath(path):
+        """Normalize path, eliminating double slashes, etc."""
+        path = os.fspath(path)
+        if isinstance(path, bytes):
+            return os.fsencode(_path_normpath(os.fsdecode(path))) or b"."
+        return _path_normpath(path) or "."
 
 
 def abspath(path):
diff --git a/Lib/site.py b/Lib/site.py
index e129f3b4851f3..b11cd48e69e9a 100644
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -361,11 +361,11 @@ def getsitepackages(prefixes=None):
             continue
         seen.add(prefix)
 
-        libdirs = [sys.platlibdir]
-        if sys.platlibdir != "lib":
-            libdirs.append("lib")
-
         if os.sep == '/':
+            libdirs = [sys.platlibdir]
+            if sys.platlibdir != "lib":
+                libdirs.append("lib")
+
             for libdir in libdirs:
                 path = os.path.join(prefix, libdir,
                                     "python%d.%d" % sys.version_info[:2],
@@ -373,10 +373,7 @@ def getsitepackages(prefixes=None):
                 sitepackages.append(path)
         else:
             sitepackages.append(prefix)
-
-            for libdir in libdirs:
-                path = os.path.join(prefix, libdir, "site-packages")
-                sitepackages.append(path)
+            sitepackages.append(os.path.join(prefix, "Lib", "site-packages"))
     return sitepackages
 
 def addsitepackages(known_paths, prefixes=None):
diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
index daf9f000060a3..da918b7ba1904 100644
--- a/Lib/sysconfig.py
+++ b/Lib/sysconfig.py
@@ -216,6 +216,11 @@ def _expand_vars(scheme, vars):
     if vars is None:
         vars = {}
     _extend_dict(vars, get_config_vars())
+    if os.name == 'nt':
+        # On Windows we want to substitute 'lib' for schemes rather
+        # than the native value (without modifying vars, in case it
+        # was passed in)
+        vars = vars | {'platlibdir': 'lib'}
 
     for key, value in _INSTALL_SCHEMES[scheme].items():
         if os.name in ('posix', 'nt'):
diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py
index d05907ecfe744..e7b7106e17851 100644
--- a/Lib/test/_test_embed_set_config.py
+++ b/Lib/test/_test_embed_set_config.py
@@ -20,6 +20,7 @@ def setUp(self):
         self.sys_copy = dict(sys.__dict__)
 
     def tearDown(self):
+        _testinternalcapi.reset_path_config()
         _testinternalcapi.set_config(self.old_config)
         sys.__dict__.clear()
         sys.__dict__.update(self.sys_copy)
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 4781c7f2a9d12..3620a7619601d 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -13,6 +13,7 @@
 import shutil
 import subprocess
 import sys
+import sysconfig
 import tempfile
 import textwrap
 
@@ -445,6 +446,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         '_init_main': 1,
         '_isolated_interpreter': 0,
         'use_frozen_modules': 1,
+        '_is_python_build': IGNORE_CONFIG,
     }
     if MS_WINDOWS:
         CONFIG_COMPAT.update({
@@ -508,32 +510,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'),
         ))
 
-    # path config
-    if MS_WINDOWS:
-        PATH_CONFIG = {
-            'isolated': -1,
-            'site_import': -1,
-            'python3_dll': GET_DEFAULT_CONFIG,
-        }
-    else:
-        PATH_CONFIG = {}
-    # other keys are copied by COPY_PATH_CONFIG
-
-    COPY_PATH_CONFIG = [
-        # Copy core config to global config for expected values
-        'prefix',
-        'exec_prefix',
-        'program_name',
-        'home',
-        'stdlib_dir',
-        # program_full_path and module_search_path are copied indirectly from
-        # the core configuration in check_path_config().
-    ]
-    if MS_WINDOWS:
-        COPY_PATH_CONFIG.extend((
-            'base_executable',
-        ))
-
     EXPECTED_CONFIG = None
 
     @classmethod
@@ -599,8 +575,7 @@ def _get_expected_config(self):
         return configs
 
     def get_expected_config(self, expected_preconfig, expected,
-                            expected_pathconfig, env, api,
-                            modify_path_cb=None):
+                            env, api, modify_path_cb=None):
         configs = self._get_expected_config()
 
         pre_config = configs['pre_config']
@@ -608,11 +583,6 @@ def get_expected_config(self, expected_preconfig, expected,
             if value is self.GET_DEFAULT_CONFIG:
                 expected_preconfig[key] = pre_config[key]
 
-        path_config = configs['path_config']
-        for key, value in expected_pathconfig.items():
-            if value is self.GET_DEFAULT_CONFIG:
-                expected_pathconfig[key] = path_config[key]
-
         if not expected_preconfig['configure_locale'] or api == API_COMPAT:
             # there is no easy way to get the locale encoding before
             # setlocale(LC_CTYPE, "") is called: don't test encodings
@@ -705,18 +675,8 @@ def check_global_config(self, configs):
 
         self.assertEqual(configs['global_config'], expected)
 
-    def check_path_config(self, configs, expected):
-        config = configs['config']
-
-        for key in self.COPY_PATH_CONFIG:
-            expected[key] = config[key]
-        expected['module_search_path'] = os.path.pathsep.join(config['module_search_paths'])
-        expected['program_full_path'] = config['executable']
-
-        self.assertEqual(configs['path_config'], expected)
-
     def check_all_configs(self, testname, expected_config=None,
-                          expected_preconfig=None, expected_pathconfig=None,
+                          expected_preconfig=None,
                           modify_path_cb=None,
                           stderr=None, *, api, preconfig_api=None,
                           env=None, ignore_stderr=False, cwd=None):
@@ -740,10 +700,6 @@ def check_all_configs(self, testname, expected_config=None,
         if expected_config is None:
             expected_config = {}
 
-        if expected_pathconfig is None:
-            expected_pathconfig = {}
-        expected_pathconfig = dict(self.PATH_CONFIG, **expected_pathconfig)
-
         if api == API_PYTHON:
             default_config = self.CONFIG_PYTHON
         elif api == API_ISOLATED:
@@ -754,7 +710,6 @@ def check_all_configs(self, testname, expected_config=None,
 
         self.get_expected_config(expected_preconfig,
                                  expected_config,
-                                 expected_pathconfig,
                                  env,
                                  api, modify_path_cb)
 
@@ -772,7 +727,6 @@ def check_all_configs(self, testname, expected_config=None,
         self.check_pre_config(configs, expected_preconfig)
         self.check_config(configs, expected_config)
         self.check_global_config(configs)
-        self.check_path_config(configs, expected_pathconfig)
         return configs
 
     def test_init_default_config(self):
@@ -1039,10 +993,13 @@ def test_init_dont_configure_locale(self):
         self.check_all_configs("test_init_dont_configure_locale", {}, preconfig,
                                api=API_PYTHON)
 
+    @unittest.skip('as of 3.11 this test no longer works because '
+                   'path calculations do not occur on read')
     def test_init_read_set(self):
         config = {
             'program_name': './init_read_set',
             'executable': 'my_executable',
+            'base_executable': 'my_executable',
         }
         def modify_path(path):
             path.insert(1, "test_path_insert1")
@@ -1156,7 +1113,6 @@ def test_init_setpath(self):
              # The current getpath.c doesn't determine the stdlib dir
              # in this case.
             'stdlib_dir': '',
-            'use_frozen_modules': -1,
         }
         self.default_program_name(config)
         env = {'TESTPATH': os.path.pathsep.join(paths)}
@@ -1180,7 +1136,7 @@ def test_init_setpath_config(self):
              # The current getpath.c doesn't determine the stdlib dir
              # in this case.
             'stdlib_dir': '',
-            'use_frozen_modules': -1,
+            'use_frozen_modules': 1,
             # overridden by PyConfig
             'program_name': 'conf_program_name',
             'base_executable': 'conf_executable',
@@ -1210,13 +1166,16 @@ def module_search_paths(self, prefix=None, exec_prefix=None):
             ]
 
     @contextlib.contextmanager
-    def tmpdir_with_python(self):
+    def tmpdir_with_python(self, subdir=None):
         # Temporary directory with a copy of the Python program
         with tempfile.TemporaryDirectory() as tmpdir:
             # bpo-38234: On macOS and FreeBSD, the temporary directory
             # can be symbolic link. For example, /tmp can be a symbolic link
             # to /var/tmp. Call realpath() to resolve all symbolic links.
             tmpdir = os.path.realpath(tmpdir)
+            if subdir:
+                tmpdir = os.path.normpath(os.path.join(tmpdir, subdir))
+                os.makedirs(tmpdir)
 
             if MS_WINDOWS:
                 # Copy pythonXY.dll (or pythonXY_d.dll)
@@ -1260,11 +1219,14 @@ def test_init_setpythonhome(self):
 
         prefix = exec_prefix = home
         if MS_WINDOWS:
-            stdlib = os.path.join(home, sys.platlibdir)
+            stdlib = os.path.join(home, "Lib")
+            # Because we are specifying 'home', module search paths
+            # are fairly static
+            expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')]
         else:
             version = f'{sys.version_info.major}.{sys.version_info.minor}'
             stdlib = os.path.join(home, sys.platlibdir, f'python{version}')
-        expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)
+            expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)
 
         config = {
             'home': home,
@@ -1291,7 +1253,7 @@ def copy_paths_by_env(self, config):
         env = {'PYTHONPATH': paths_str}
         return env
 
-    @unittest.skipIf(MS_WINDOWS, 'Windows does not use pybuilddir.txt')
+    @unittest.skipIf(MS_WINDOWS, 'See test_init_pybuilddir_win32')
     def test_init_pybuilddir(self):
         # Test path configuration with pybuilddir.txt configuration file
 
@@ -1300,6 +1262,8 @@ def test_init_pybuilddir(self):
             # directory (tmpdir)
             subdir = 'libdir'
             libdir = os.path.join(tmpdir, subdir)
+            # The stdlib dir is dirname(executable) + VPATH + 'Lib'
+            stdlibdir = os.path.join(tmpdir, 'Lib')
             os.mkdir(libdir)
 
             filename = os.path.join(tmpdir, 'pybuilddir.txt')
@@ -1307,21 +1271,58 @@ def test_init_pybuilddir(self):
                 fp.write(subdir)
 
             module_search_paths = self.module_search_paths()
+            module_search_paths[-2] = stdlibdir
             module_search_paths[-1] = libdir
 
             executable = self.test_exe
             config = {
+                'base_exec_prefix': sysconfig.get_config_var("exec_prefix"),
+                'base_prefix': sysconfig.get_config_var("prefix"),
                 'base_executable': executable,
                 'executable': executable,
                 'module_search_paths': module_search_paths,
-                'stdlib_dir': STDLIB_INSTALL,
-                'use_frozen_modules': 1 if STDLIB_INSTALL else -1,
+                'stdlib_dir': stdlibdir,
             }
             env = self.copy_paths_by_env(config)
             self.check_all_configs("test_init_compat_config", config,
                                    api=API_COMPAT, env=env,
                                    ignore_stderr=True, cwd=tmpdir)
 
+    @unittest.skipUnless(MS_WINDOWS, 'See test_init_pybuilddir')
+    def test_init_pybuilddir_win32(self):
+        # Test path configuration with pybuilddir.txt configuration file
+
+        with self.tmpdir_with_python(r'PCbuild\arch') as tmpdir:
+            # The prefix is dirname(executable) + VPATH
+            prefix = os.path.normpath(os.path.join(tmpdir, r'..\..'))
+            # The stdlib dir is dirname(executable) + VPATH + 'Lib'
+            stdlibdir = os.path.normpath(os.path.join(tmpdir, r'..\..\Lib'))
+
+            filename = os.path.join(tmpdir, 'pybuilddir.txt')
+            with open(filename, "w", encoding="utf8") as fp:
+                fp.write(tmpdir)
+
+            module_search_paths = self.module_search_paths()
+            module_search_paths[-3] = os.path.join(tmpdir, os.path.basename(module_search_paths[-3]))
+            module_search_paths[-2] = stdlibdir
+            module_search_paths[-1] = tmpdir
+
+            executable = self.test_exe
+            config = {
+                'base_exec_prefix': prefix,
+                'base_prefix': prefix,
+                'base_executable': executable,
+                'executable': executable,
+                'prefix': prefix,
+                'exec_prefix': prefix,
+                'module_search_paths': module_search_paths,
+                'stdlib_dir': stdlibdir,
+            }
+            env = self.copy_paths_by_env(config)
+            self.check_all_configs("test_init_compat_config", config,
+                                   api=API_COMPAT, env=env,
+                                   ignore_stderr=False, cwd=tmpdir)
+
     def test_init_pyvenv_cfg(self):
         # Test path configuration with pyvenv.cfg configuration file
 
@@ -1336,12 +1337,12 @@ def test_init_pyvenv_cfg(self):
                                            'lib-dynload')
                 os.makedirs(lib_dynload)
             else:
-                lib_dynload = os.path.join(pyvenv_home, 'lib')
-                os.makedirs(lib_dynload)
-                # getpathp.c uses Lib\os.py as the LANDMARK
+                lib_folder = os.path.join(pyvenv_home, 'Lib')
+                os.makedirs(lib_folder)
+                # getpath.py uses Lib\os.py as the LANDMARK
                 shutil.copyfile(
                     os.path.join(support.STDLIB_DIR, 'os.py'),
-                    os.path.join(lib_dynload, 'os.py'),
+                    os.path.join(lib_folder, 'os.py'),
                 )
 
             filename = os.path.join(tmpdir, 'pyvenv.cfg')
@@ -1355,43 +1356,38 @@ def test_init_pyvenv_cfg(self):
             else:
                 for index, path in enumerate(paths):
                     if index == 0:
+                        # Because we copy the DLLs into tmpdir as well, the zip file
+                        # entry in sys.path will be there. For a regular venv, it will
+                        # usually be in the home directory.
                         paths[index] = os.path.join(tmpdir, os.path.basename(path))
                     else:
                         paths[index] = os.path.join(pyvenv_home, os.path.basename(path))
                 paths[-1] = pyvenv_home
 
             executable = self.test_exe
+            base_executable = os.path.join(pyvenv_home, os.path.basename(executable))
             exec_prefix = pyvenv_home
             config = {
+                'base_prefix': sysconfig.get_config_var("prefix"),
                 'base_exec_prefix': exec_prefix,
                 'exec_prefix': exec_prefix,
-                'base_executable': executable,
+                'base_executable': base_executable,
                 'executable': executable,
                 'module_search_paths': paths,
             }
-            path_config = {}
             if MS_WINDOWS:
                 config['base_prefix'] = pyvenv_home
                 config['prefix'] = pyvenv_home
-                config['stdlib_dir'] = os.path.join(pyvenv_home, 'lib')
+                config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib')
                 config['use_frozen_modules'] = 1
-
-                ver = sys.version_info
-                dll = f'python{ver.major}'
-                if debug_build(executable):
-                    dll += '_d'
-                dll += '.DLL'
-                dll = os.path.join(os.path.dirname(executable), dll)
-                path_config['python3_dll'] = dll
             else:
-                # The current getpath.c doesn't determine the stdlib dir
-                # in this case.
-                config['stdlib_dir'] = STDLIB_INSTALL
-                config['use_frozen_modules'] = 1 if STDLIB_INSTALL else -1
+                # cannot reliably assume stdlib_dir here because it
+                # depends too much on our build. But it ought to be found
+                config['stdlib_dir'] = self.IGNORE_CONFIG
+                config['use_frozen_modules'] = 1
 
             env = self.copy_paths_by_env(config)
             self.check_all_configs("test_init_compat_config", config,
-                                   expected_pathconfig=path_config,
                                    api=API_COMPAT, env=env,
                                    ignore_stderr=True, cwd=tmpdir)
 
@@ -1502,10 +1498,14 @@ class SetConfigTests(unittest.TestCase):
     def test_set_config(self):
         # bpo-42260: Test _PyInterpreterState_SetConfig()
         import_helper.import_module('_testcapi')
-        cmd = [sys.executable, '-I', '-m', 'test._test_embed_set_config']
+        cmd = [sys.executable, '-X', 'utf8', '-I', '-m', 'test._test_embed_set_config']
         proc = subprocess.run(cmd,
                               stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE)
+                              stderr=subprocess.PIPE,
+                              encoding='utf-8', errors='backslashreplace')
+        if proc.returncode and support.verbose:
+            print(proc.stdout)
+            print(proc.stderr)
         self.assertEqual(proc.returncode, 0,
                          (proc.returncode, proc.stdout, proc.stderr))
 
diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py
new file mode 100644
index 0000000000000..c18689c0590d4
--- /dev/null
+++ b/Lib/test/test_getpath.py
@@ -0,0 +1,879 @@
+import copy
+import ntpath
+import pathlib
+import posixpath
+import sys
+import unittest
+
+from test.support import verbose
+
+try:
+    # If we are in a source tree, use the original source file for tests
+    SOURCE = (pathlib.Path(__file__).absolute().parent.parent.parent / "Modules/getpath.py").read_bytes()
+except FileNotFoundError:
+    # Try from _testcapimodule instead
+    from _testinternalcapi import get_getpath_codeobject
+    SOURCE = get_getpath_codeobject()
+
+
+class MockGetPathTests(unittest.TestCase):
+    def __init__(self, *a, **kw):
+        super().__init__(*a, **kw)
+        self.maxDiff = None
+
+    def test_normal_win32(self):
+        "Test a 'standard' install layout on Windows."
+        ns = MockNTNamespace(
+            argv0=r"C:\Python\python.exe",
+            real_executable=r"C:\Python\python.exe",
+        )
+        ns.add_known_xfile(r"C:\Python\python.exe")
+        ns.add_known_file(r"C:\Python\Lib\os.py")
+        ns.add_known_dir(r"C:\Python\DLLs")
+        expected = dict(
+            executable=r"C:\Python\python.exe",
+            base_executable=r"C:\Python\python.exe",
+            prefix=r"C:\Python",
+            exec_prefix=r"C:\Python",
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\Python\python98.zip",
+                r"C:\Python\Lib",
+                r"C:\Python\DLLs",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_buildtree_win32(self):
+        "Test an in-build-tree layout on Windows."
+        ns = MockNTNamespace(
+            argv0=r"C:\CPython\PCbuild\amd64\python.exe",
+            real_executable=r"C:\CPython\PCbuild\amd64\python.exe",
+        )
+        ns.add_known_xfile(r"C:\CPython\PCbuild\amd64\python.exe")
+        ns.add_known_file(r"C:\CPython\Lib\os.py")
+        ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""])
+        expected = dict(
+            executable=r"C:\CPython\PCbuild\amd64\python.exe",
+            base_executable=r"C:\CPython\PCbuild\amd64\python.exe",
+            prefix=r"C:\CPython",
+            exec_prefix=r"C:\CPython",
+            build_prefix=r"C:\CPython",
+            _is_python_build=1,
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\CPython\PCbuild\amd64\python98.zip",
+                r"C:\CPython\Lib",
+                r"C:\CPython\PCbuild\amd64",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_venv_win32(self):
+        """Test a venv layout on Windows.
+
+        This layout is discovered by the presence of %__PYVENV_LAUNCHER__%,
+        specifying the original launcher executable. site.py is responsible
+        for updating prefix and exec_prefix.
+        """
+        ns = MockNTNamespace(
+            argv0=r"C:\Python\python.exe",
+            ENV___PYVENV_LAUNCHER__=r"C:\venv\Scripts\python.exe",
+            real_executable=r"C:\Python\python.exe",
+        )
+        ns.add_known_xfile(r"C:\Python\python.exe")
+        ns.add_known_xfile(r"C:\venv\Scripts\python.exe")
+        ns.add_known_file(r"C:\Python\Lib\os.py")
+        ns.add_known_dir(r"C:\Python\DLLs")
+        ns.add_known_file(r"C:\venv\pyvenv.cfg", [
+            r"home = C:\Python"
+        ])
+        expected = dict(
+            executable=r"C:\venv\Scripts\python.exe",
+            prefix=r"C:\Python",
+            exec_prefix=r"C:\Python",
+            base_executable=r"C:\Python\python.exe",
+            base_prefix=r"C:\Python",
+            base_exec_prefix=r"C:\Python",
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\Python\python98.zip",
+                r"C:\Python\Lib",
+                r"C:\Python",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_registry_win32(self):
+        """Test registry lookup on Windows.
+
+        On Windows there are registry entries that are intended for other
+        applications to register search paths.
+        """
+        hkey = rf"HKLM\Software\Python\PythonCore\9.8-XY\PythonPath"
+        winreg = MockWinreg({
+            hkey: None,
+            f"{hkey}\\Path1": "path1-dir",
+            f"{hkey}\\Path1\\Subdir": "not-subdirs",
+        })
+        ns = MockNTNamespace(
+            argv0=r"C:\Python\python.exe",
+            real_executable=r"C:\Python\python.exe",
+            winreg=winreg,
+        )
+        ns.add_known_xfile(r"C:\Python\python.exe")
+        ns.add_known_file(r"C:\Python\Lib\os.py")
+        ns.add_known_dir(r"C:\Python\DLLs")
+        expected = dict(
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\Python\python98.zip",
+                "path1-dir",
+                # should not contain not-subdirs
+                r"C:\Python\Lib",
+                r"C:\Python\DLLs",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+        ns["config"]["use_environment"] = 0
+        ns["config"]["module_search_paths_set"] = 0
+        ns["config"]["module_search_paths"] = None
+        expected = dict(
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\Python\python98.zip",
+                r"C:\Python\Lib",
+                r"C:\Python\DLLs",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_normal_win32(self):
+        "Test a 'standard' install layout via symlink on Windows."
+        ns = MockNTNamespace(
+            argv0=r"C:\LinkedFrom\python.exe",
+            real_executable=r"C:\Python\python.exe",
+        )
+        ns.add_known_xfile(r"C:\LinkedFrom\python.exe")
+        ns.add_known_xfile(r"C:\Python\python.exe")
+        ns.add_known_link(r"C:\LinkedFrom\python.exe", r"C:\Python\python.exe")
+        ns.add_known_file(r"C:\Python\Lib\os.py")
+        ns.add_known_dir(r"C:\Python\DLLs")
+        expected = dict(
+            executable=r"C:\LinkedFrom\python.exe",
+            base_executable=r"C:\LinkedFrom\python.exe",
+            prefix=r"C:\Python",
+            exec_prefix=r"C:\Python",
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\Python\python98.zip",
+                r"C:\Python\Lib",
+                r"C:\Python\DLLs",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_buildtree_win32(self):
+        "Test an in-build-tree layout via symlink on Windows."
+        ns = MockNTNamespace(
+            argv0=r"C:\LinkedFrom\python.exe",
+            real_executable=r"C:\CPython\PCbuild\amd64\python.exe",
+        )
+        ns.add_known_xfile(r"C:\LinkedFrom\python.exe")
+        ns.add_known_xfile(r"C:\CPython\PCbuild\amd64\python.exe")
+        ns.add_known_link(r"C:\LinkedFrom\python.exe", r"C:\CPython\PCbuild\amd64\python.exe")
+        ns.add_known_file(r"C:\CPython\Lib\os.py")
+        ns.add_known_file(r"C:\CPython\PCbuild\amd64\pybuilddir.txt", [""])
+        expected = dict(
+            executable=r"C:\LinkedFrom\python.exe",
+            base_executable=r"C:\LinkedFrom\python.exe",
+            prefix=r"C:\CPython",
+            exec_prefix=r"C:\CPython",
+            build_prefix=r"C:\CPython",
+            _is_python_build=1,
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\CPython\PCbuild\amd64\python98.zip",
+                r"C:\CPython\Lib",
+                r"C:\CPython\PCbuild\amd64",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_normal_posix(self):
+        "Test a 'standard' install layout on *nix"
+        ns = MockPosixNamespace(
+            PREFIX="/usr",
+            argv0="python",
+            ENV_PATH="/usr/bin",
+        )
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_file("/usr/lib/python9.8/os.py")
+        ns.add_known_dir("/usr/lib/python9.8/lib-dynload")
+        expected = dict(
+            executable="/usr/bin/python",
+            base_executable="/usr/bin/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib/python98.zip",
+                "/usr/lib/python9.8",
+                "/usr/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_buildpath_posix(self):
+        """Test an in-build-tree layout on POSIX.
+
+        This layout is discovered from the presence of pybuilddir.txt, which
+        contains the relative path from the executable's directory to the
+        platstdlib path.
+        """
+        ns = MockPosixNamespace(
+            argv0=r"/home/cpython/python",
+            PREFIX="/usr/local",
+        )
+        ns.add_known_xfile("/home/cpython/python")
+        ns.add_known_xfile("/usr/local/bin/python")
+        ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.linux-x86_64-9.8"])
+        ns.add_known_file("/home/cpython/Lib/os.py")
+        ns.add_known_dir("/home/cpython/lib-dynload")
+        expected = dict(
+            executable="/home/cpython/python",
+            prefix="/usr/local",
+            exec_prefix="/usr/local",
+            base_executable="/home/cpython/python",
+            build_prefix="/home/cpython",
+            _is_python_build=1,
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/local/lib/python98.zip",
+                "/home/cpython/Lib",
+                "/home/cpython/build/lib.linux-x86_64-9.8",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_venv_posix(self):
+        "Test a venv layout on *nix."
+        ns = MockPosixNamespace(
+            argv0="python",
+            PREFIX="/usr",
+            ENV_PATH="/venv/bin:/usr/bin",
+        )
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_xfile("/venv/bin/python")
+        ns.add_known_file("/usr/lib/python9.8/os.py")
+        ns.add_known_dir("/usr/lib/python9.8/lib-dynload")
+        ns.add_known_file("/venv/pyvenv.cfg", [
+            r"home = /usr/bin"
+        ])
+        expected = dict(
+            executable="/venv/bin/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            base_executable="/usr/bin/python",
+            base_prefix="/usr",
+            base_exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib/python98.zip",
+                "/usr/lib/python9.8",
+                "/usr/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_normal_posix(self):
+        "Test a 'standard' install layout via symlink on *nix"
+        ns = MockPosixNamespace(
+            PREFIX="/usr",
+            argv0="/linkfrom/python",
+        )
+        ns.add_known_xfile("/linkfrom/python")
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_link("/linkfrom/python", "/usr/bin/python")
+        ns.add_known_file("/usr/lib/python9.8/os.py")
+        ns.add_known_dir("/usr/lib/python9.8/lib-dynload")
+        expected = dict(
+            executable="/linkfrom/python",
+            base_executable="/linkfrom/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib/python98.zip",
+                "/usr/lib/python9.8",
+                "/usr/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_buildpath_posix(self):
+        """Test an in-build-tree layout on POSIX.
+
+        This layout is discovered from the presence of pybuilddir.txt, which
+        contains the relative path from the executable's directory to the
+        platstdlib path.
+        """
+        ns = MockPosixNamespace(
+            argv0=r"/linkfrom/python",
+            PREFIX="/usr/local",
+        )
+        ns.add_known_xfile("/linkfrom/python")
+        ns.add_known_xfile("/home/cpython/python")
+        ns.add_known_link("/linkfrom/python", "/home/cpython/python")
+        ns.add_known_xfile("/usr/local/bin/python")
+        ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.linux-x86_64-9.8"])
+        ns.add_known_file("/home/cpython/Lib/os.py")
+        ns.add_known_dir("/home/cpython/lib-dynload")
+        expected = dict(
+            executable="/linkfrom/python",
+            prefix="/usr/local",
+            exec_prefix="/usr/local",
+            base_executable="/linkfrom/python",
+            build_prefix="/home/cpython",
+            _is_python_build=1,
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/local/lib/python98.zip",
+                "/home/cpython/Lib",
+                "/home/cpython/build/lib.linux-x86_64-9.8",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_custom_platlibdir_posix(self):
+        "Test an install with custom platlibdir on *nix"
+        ns = MockPosixNamespace(
+            PREFIX="/usr",
+            argv0="/linkfrom/python",
+            PLATLIBDIR="lib64",
+        )
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_file("/usr/lib64/python9.8/os.py")
+        ns.add_known_dir("/usr/lib64/python9.8/lib-dynload")
+        expected = dict(
+            executable="/linkfrom/python",
+            base_executable="/linkfrom/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib64/python98.zip",
+                "/usr/lib64/python9.8",
+                "/usr/lib64/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_venv_macos(self):
+        """Test a venv layout on macOS.
+
+        This layout is discovered when 'executable' and 'real_executable' match,
+        but $__PYVENV_LAUNCHER__ has been set to the original process.
+        """
+        ns = MockPosixNamespace(
+            os_name="darwin",
+            argv0="/usr/bin/python",
+            PREFIX="/usr",
+            ENV___PYVENV_LAUNCHER__="/framework/Python9.8/python",
+            real_executable="/usr/bin/python",
+        )
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_xfile("/framework/Python9.8/python")
+        ns.add_known_file("/usr/lib/python9.8/os.py")
+        ns.add_known_dir("/usr/lib/python9.8/lib-dynload")
+        ns.add_known_file("/framework/Python9.8/pyvenv.cfg", [
+            "home = /usr/bin"
+        ])
+        expected = dict(
+            executable="/framework/Python9.8/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            base_executable="/usr/bin/python",
+            base_prefix="/usr",
+            base_exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib/python98.zip",
+                "/usr/lib/python9.8",
+                "/usr/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_normal_macos(self):
+        "Test a 'standard' install layout via symlink on macOS"
+        ns = MockPosixNamespace(
+            os_name="darwin",
+            PREFIX="/usr",
+            argv0="python",
+            ENV_PATH="/linkfrom:/usr/bin",
+            # real_executable on macOS matches the invocation path
+            real_executable="/linkfrom/python",
+        )
+        ns.add_known_xfile("/linkfrom/python")
+        ns.add_known_xfile("/usr/bin/python")
+        ns.add_known_link("/linkfrom/python", "/usr/bin/python")
+        ns.add_known_file("/usr/lib/python9.8/os.py")
+        ns.add_known_dir("/usr/lib/python9.8/lib-dynload")
+        expected = dict(
+            executable="/linkfrom/python",
+            base_executable="/linkfrom/python",
+            prefix="/usr",
+            exec_prefix="/usr",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/lib/python98.zip",
+                "/usr/lib/python9.8",
+                "/usr/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_symlink_buildpath_macos(self):
+        """Test an in-build-tree layout via symlink on macOS.
+
+        This layout is discovered from the presence of pybuilddir.txt, which
+        contains the relative path from the executable's directory to the
+        platstdlib path.
+        """
+        ns = MockPosixNamespace(
+            os_name="darwin",
+            argv0=r"python",
+            ENV_PATH="/linkfrom:/usr/bin",
+            PREFIX="/usr/local",
+            # real_executable on macOS matches the invocation path
+            real_executable="/linkfrom/python",
+        )
+        ns.add_known_xfile("/linkfrom/python")
+        ns.add_known_xfile("/home/cpython/python")
+        ns.add_known_link("/linkfrom/python", "/home/cpython/python")
+        ns.add_known_xfile("/usr/local/bin/python")
+        ns.add_known_file("/home/cpython/pybuilddir.txt", ["build/lib.macos-9.8"])
+        ns.add_known_file("/home/cpython/Lib/os.py")
+        ns.add_known_dir("/home/cpython/lib-dynload")
+        expected = dict(
+            executable="/linkfrom/python",
+            prefix="/usr/local",
+            exec_prefix="/usr/local",
+            base_executable="/linkfrom/python",
+            build_prefix="/home/cpython",
+            _is_python_build=1,
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/usr/local/lib/python98.zip",
+                "/home/cpython/Lib",
+                "/home/cpython/build/lib.macos-9.8",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+
+# ******************************************************************************
+
+DEFAULT_NAMESPACE = dict(
+    PREFIX="",
+    EXEC_PREFIX="",
+    PYTHONPATH="",
+    VPATH="",
+    PLATLIBDIR="",
+    PYDEBUGEXT="",
+    VERSION_MAJOR=9,    # fixed version number for ease
+    VERSION_MINOR=8,    # of testing
+    PYWINVER=None,
+    EXE_SUFFIX=None,
+    
+    ENV_PATH="",
+    ENV_PYTHONHOME="",
+    ENV_PYTHONEXECUTABLE="",
+    ENV___PYVENV_LAUNCHER__="",
+    argv0="",
+    py_setpath="",
+    real_executable="",
+    executable_dir="",
+    library="",
+    winreg=None,
+    build_prefix=None,
+    venv_prefix=None,
+)
+
+DEFAULT_CONFIG = dict(
+    home=None,
+    platlibdir=None,
+    pythonpath=None,
+    program_name=None,
+    prefix=None,
+    exec_prefix=None,
+    base_prefix=None,
+    base_exec_prefix=None,
+    executable=None,
+    base_executable="",
+    stdlib_dir=None,
+    platstdlib_dir=None,
+    module_search_paths=None,
+    module_search_paths_set=0,
+    pythonpath_env=None,
+    argv=None,
+    orig_argv=None,
+
+    isolated=0,
+    use_environment=1,
+    use_site=1,
+)
+
+class MockNTNamespace(dict):
+    def __init__(self, *a, argv0=None, config=None, **kw):
+        self.update(DEFAULT_NAMESPACE)
+        self["config"] = DEFAULT_CONFIG.copy()
+        self["os_name"] = "nt"
+        self["PLATLIBDIR"] = "DLLs"
+        self["PYWINVER"] = "9.8-XY"
+        self["VPATH"] = r"..\.."
+        super().__init__(*a, **kw)
+        if argv0:
+            self["config"]["orig_argv"] = [argv0]
+        if config:
+            self["config"].update(config)
+        self._files = {}
+        self._links = {}
+        self._dirs = set()
+        self._warnings = []
+
+    def add_known_file(self, path, lines=None):
+        self._files[path.casefold()] = list(lines or ())
+        self.add_known_dir(path.rpartition("\\")[0])
+
+    def add_known_xfile(self, path):
+        self.add_known_file(path)
+
+    def add_known_link(self, path, target):
+        self._links[path.casefold()] = target
+
+    def add_known_dir(self, path):
+        p = path.rstrip("\\").casefold()
+        while p:
+            self._dirs.add(p)
+            p = p.rpartition("\\")[0]
+
+    def __missing__(self, key):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            raise KeyError(key) from None
+
+    def abspath(self, path):
+        if self.isabs(path):
+            return path
+        return self.joinpath("C:\\Absolute", path)
+
+    def basename(self, path):
+        return path.rpartition("\\")[2]
+
+    def dirname(self, path):
+        name = path.rstrip("\\").rpartition("\\")[0]
+        if name[1:] == ":":
+            return name + "\\"
+        return name
+
+    def hassuffix(self, path, suffix):
+        return path.casefold().endswith(suffix.casefold())
+
+    def isabs(self, path):
+        return path[1:3] == ":\\"
+
+    def isdir(self, path):
+        if verbose:
+            print("Check if", path, "is a dir")
+        return path.casefold() in self._dirs
+
+    def isfile(self, path):
+        if verbose:
+            print("Check if", path, "is a file")
+        return path.casefold() in self._files
+
+    def ismodule(self, path):
+        if verbose:
+            print("Check if", path, "is a module")
+        path = path.casefold()
+        return path in self._files and path.rpartition(".")[2] == "py".casefold()
+
+    def isxfile(self, path):
+        if verbose:
+            print("Check if", path, "is a executable")
+        path = path.casefold()
+        return path in self._files and path.rpartition(".")[2] == "exe".casefold()
+
+    def joinpath(self, *path):
+        return ntpath.normpath(ntpath.join(*path))
+
+    def readlines(self, path):
+        try:
+            return self._files[path.casefold()]
+        except KeyError:
+            raise FileNotFoundError(path) from None
+
+    def realpath(self, path, _trail=None):
+        if verbose:
+            print("Read link from", path)
+        try:
+            link = self._links[path.casefold()]
+        except KeyError:
+            return path
+        if _trail is None:
+            _trail = set()
+        elif link.casefold() in _trail:
+            raise OSError("circular link")
+        _trail.add(link.casefold())
+        return self.realpath(link, _trail)
+
+    def warn(self, message):
+        self._warnings.append(message)
+        if verbose:
+            print(message)
+
+
+class MockWinreg:
+    HKEY_LOCAL_MACHINE = "HKLM"
+    HKEY_CURRENT_USER = "HKCU"
+
+    def __init__(self, keys):
+        self.keys = {k.casefold(): v for k, v in keys.items()}
+        self.open = {}
+
+    def __repr__(self):
+        return "<MockWinreg>"
+
+    def __eq__(self, other):
+        return isinstance(other, type(self))
+
+    def open_keys(self):
+        return list(self.open)
+
+    def OpenKeyEx(self, hkey, subkey):
+        if verbose:
+            print(f"OpenKeyEx({hkey}, {subkey})")
+        key = f"{hkey}\\{subkey}".casefold()
+        if key in self.keys:
+            self.open[key] = self.open.get(key, 0) + 1
+            return key
+        raise FileNotFoundError()
+
+    def CloseKey(self, hkey):
+        if verbose:
+            print(f"CloseKey({hkey})")
+        hkey = hkey.casefold()
+        if hkey not in self.open:
+            raise RuntimeError("key is not open")
+        self.open[hkey] -= 1
+        if not self.open[hkey]:
+            del self.open[hkey]
+
+    def EnumKey(self, hkey, i):
+        if verbose:
+            print(f"EnumKey({hkey}, {i})")
+        hkey = hkey.casefold()
+        if hkey not in self.open:
+            raise RuntimeError("key is not open")
+        prefix = f'{hkey}\\'
+        subkeys = [k[len(prefix):] for k in sorted(self.keys) if k.startswith(prefix)]
+        subkeys[:] = [k for k in subkeys if '\\' not in k]
+        for j, n in enumerate(subkeys):
+            if j == i:
+                return n.removeprefix(prefix)
+        raise OSError("end of enumeration")
+
+    def QueryValue(self, hkey):
+        if verbose:
+            print(f"QueryValue({hkey})")
+        hkey = hkey.casefold()
+        if hkey not in self.open:
+            raise RuntimeError("key is not open")
+        try:
+            return self.keys[hkey]
+        except KeyError:
+            raise OSError()
+
+
+class MockPosixNamespace(dict):
+    def __init__(self, *a, argv0=None, config=None, **kw):
+        self.update(DEFAULT_NAMESPACE)
+        self["config"] = DEFAULT_CONFIG.copy()
+        self["os_name"] = "posix"
+        self["PLATLIBDIR"] = "lib"
+        super().__init__(*a, **kw)
+        if argv0:
+            self["config"]["orig_argv"] = [argv0]
+        if config:
+            self["config"].update(config)
+        self._files = {}
+        self._xfiles = set()
+        self._links = {}
+        self._dirs = set()
+        self._warnings = []
+
+    def add_known_file(self, path, lines=None):
+        self._files[path] = list(lines or ())
+        self.add_known_dir(path.rpartition("/")[0])
+
+    def add_known_xfile(self, path):
+        self.add_known_file(path)
+        self._xfiles.add(path)
+
+    def add_known_link(self, path, target):
+        self._links[path] = target
+
+    def add_known_dir(self, path):
+        p = path.rstrip("/")
+        while p:
+            self._dirs.add(p)
+            p = p.rpartition("/")[0]
+
+    def __missing__(self, key):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            raise KeyError(key) from None
+
+    def abspath(self, path):
+        if self.isabs(path):
+            return path
+        return self.joinpath("/Absolute", path)
+
+    def basename(self, path):
+        return path.rpartition("/")[2]
+
+    def dirname(self, path):
+        return path.rstrip("/").rpartition("/")[0]
+
+    def hassuffix(self, path, suffix):
+        return path.endswith(suffix)
+
+    def isabs(self, path):
+        return path[0:1] == "/"
+
+    def isdir(self, path):
+        if verbose:
+            print("Check if", path, "is a dir")
+        return path in self._dirs
+
+    def isfile(self, path):
+        if verbose:
+            print("Check if", path, "is a file")
+        return path in self._files
+
+    def ismodule(self, path):
+        if verbose:
+            print("Check if", path, "is a module")
+        return path in self._files and path.rpartition(".")[2] == "py"
+
+    def isxfile(self, path):
+        if verbose:
+            print("Check if", path, "is an xfile")
+        return path in self._xfiles
+
+    def joinpath(self, *path):
+        return posixpath.normpath(posixpath.join(*path))
+
+    def readlines(self, path):
+        try:
+            return self._files[path]
+        except KeyError:
+            raise FileNotFoundError(path) from None
+
+    def realpath(self, path, _trail=None):
+        if verbose:
+            print("Read link from", path)
+        try:
+            link = self._links[path]
+        except KeyError:
+            return path
+        if _trail is None:
+            _trail = set()
+        elif link in _trail:
+            raise OSError("circular link")
+        _trail.add(link)
+        return self.realpath(link, _trail)
+
+    def warn(self, message):
+        self._warnings.append(message)
+        if verbose:
+            print(message)
+
+
+def diff_dict(before, after, prefix="global"):
+    diff = []
+    for k in sorted(before):
+        if k[:2] == "__":
+            continue
+        if k == "config":
+            diff_dict(before[k], after[k], prefix="config")
+            continue
+        if k in after and after[k] != before[k]:
+            diff.append((k, before[k], after[k]))
+    if not diff:
+        return
+    max_k = max(len(k) for k, _, _ in diff)
+    indent = " " * (len(prefix) + 1 + max_k)
+    if verbose:
+        for k, b, a in diff:
+            if b:
+                print("{}.{} -{!r}\n{} +{!r}".format(prefix, k.ljust(max_k), b, indent, a))
+            else:
+                print("{}.{} +{!r}".format(prefix, k.ljust(max_k), a))
+
+
+def dump_dict(before, after, prefix="global"):
+    if not verbose or not after:
+        return
+    max_k = max(len(k) for k in after)
+    for k, v in sorted(after.items(), key=lambda i: i[0]):
+        if k[:2] == "__":
+            continue
+        if k == "config":
+            dump_dict(before[k], after[k], prefix="config")
+            continue
+        try:
+            if v != before[k]:
+                print("{}.{} {!r} (was {!r})".format(prefix, k.ljust(max_k), v, before[k]))
+                continue
+        except KeyError:
+            pass
+        print("{}.{} {!r}".format(prefix, k.ljust(max_k), v))
+
+
+def getpath(ns, keys):
+    before = copy.deepcopy(ns)
+    failed = True
+    try:
+        exec(SOURCE, ns)
+        failed = False
+    finally:
+        if failed:
+            dump_dict(before, ns)
+        else:
+            diff_dict(before, ns)
+    return {
+        k: ns['config'].get(k, ns.get(k, ...))
+        for k in keys
+    }
diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index 661c59d617163..a8d87e53db9e9 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -741,8 +741,9 @@ def test_ismount(self):
             # (or any other volume root). The drive-relative
             # locations below cannot then refer to mount points
             #
-            drive, path = ntpath.splitdrive(sys.executable)
-            with os_helper.change_cwd(ntpath.dirname(sys.executable)):
+            test_cwd = os.getenv("SystemRoot")
+            drive, path = ntpath.splitdrive(test_cwd)
+            with os_helper.change_cwd(test_cwd):
                 self.assertFalse(ntpath.ismount(drive.lower()))
                 self.assertFalse(ntpath.ismount(drive.upper()))
 
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
index 5f06a0d4b0372..76d35daed0b80 100644
--- a/Lib/test/test_site.py
+++ b/Lib/test/test_site.py
@@ -11,6 +11,7 @@
 from test.support import socket_helper
 from test.support import captured_stderr
 from test.support.os_helper import TESTFN, EnvironmentVarGuard, change_cwd
+import ast
 import builtins
 import encodings
 import glob
@@ -300,7 +301,8 @@ def test_getsitepackages(self):
             self.assertEqual(len(dirs), 2)
             self.assertEqual(dirs[0], 'xoxo')
             wanted = os.path.join('xoxo', 'lib', 'site-packages')
-            self.assertEqual(dirs[1], wanted)
+            self.assertEqual(os.path.normcase(dirs[1]),
+                             os.path.normcase(wanted))
 
     @unittest.skipUnless(HAS_USER_SITE, 'need user site')
     def test_no_home_directory(self):
@@ -497,13 +499,14 @@ class StartupImportTests(unittest.TestCase):
 
     def test_startup_imports(self):
         # Get sys.path in isolated mode (python3 -I)
-        popen = subprocess.Popen([sys.executable, '-I', '-c',
-                                  'import sys; print(repr(sys.path))'],
+        popen = subprocess.Popen([sys.executable, '-X', 'utf8', '-I',
+                                  '-c', 'import sys; print(repr(sys.path))'],
                                  stdout=subprocess.PIPE,
-                                 encoding='utf-8')
+                                 encoding='utf-8',
+                                 errors='surrogateescape')
         stdout = popen.communicate()[0]
         self.assertEqual(popen.returncode, 0, repr(stdout))
-        isolated_paths = eval(stdout)
+        isolated_paths = ast.literal_eval(stdout)
 
         # bpo-27807: Even with -I, the site module executes all .pth files
         # found in sys.path (see site.addpackage()). Skip the test if at least
@@ -515,14 +518,15 @@ def test_startup_imports(self):
 
         # This tests checks which modules are loaded by Python when it
         # initially starts upon startup.
-        popen = subprocess.Popen([sys.executable, '-I', '-v', '-c',
-                                  'import sys; print(set(sys.modules))'],
+        popen = subprocess.Popen([sys.executable, '-X', 'utf8', '-I', '-v',
+                                  '-c', 'import sys; print(set(sys.modules))'],
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE,
-                                 encoding='utf-8')
+                                 encoding='utf-8',
+                                 errors='surrogateescape')
         stdout, stderr = popen.communicate()
         self.assertEqual(popen.returncode, 0, (stdout, stderr))
-        modules = eval(stdout)
+        modules = ast.literal_eval(stdout)
 
         self.assertIn('site', modules)
 
diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py
index 9408657c91886..506266d08185d 100644
--- a/Lib/test/test_sysconfig.py
+++ b/Lib/test/test_sysconfig.py
@@ -102,6 +102,10 @@ def test_get_paths(self):
 
     def test_get_path(self):
         config_vars = get_config_vars()
+        if os.name == 'nt':
+            # On Windows, we replace the native platlibdir name with the
+            # default so that POSIX schemes resolve correctly
+            config_vars = config_vars | {'platlibdir': 'lib'}
         for scheme in _INSTALL_SCHEMES:
             for name in _INSTALL_SCHEMES[scheme]:
                 expected = _INSTALL_SCHEMES[scheme][name].format(**config_vars)
diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
index 94d626598bac3..de714de2a2049 100644
--- a/Lib/test/test_venv.py
+++ b/Lib/test/test_venv.py
@@ -15,7 +15,7 @@
 import sys
 import tempfile
 from test.support import (captured_stdout, captured_stderr, requires_zlib,
-                          skip_if_broken_multiprocessing_synchronize)
+                          skip_if_broken_multiprocessing_synchronize, verbose)
 from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree)
 import unittest
 import venv
@@ -40,6 +40,8 @@ def check_output(cmd, encoding=None):
         encoding=encoding)
     out, err = p.communicate()
     if p.returncode:
+        if verbose and err:
+            print(err.decode('utf-8', 'backslashreplace'))
         raise subprocess.CalledProcessError(
             p.returncode, cmd, out, err)
     return out, err
@@ -194,7 +196,7 @@ def test_prefixes(self):
             ('base_exec_prefix', sys.base_exec_prefix)):
             cmd[2] = 'import sys; print(sys.%s)' % prefix
             out, err = check_output(cmd)
-            self.assertEqual(out.strip(), expected.encode())
+            self.assertEqual(out.strip(), expected.encode(), prefix)
 
     if sys.platform == 'win32':
         ENV_SUBDIRS = (
diff --git a/Makefile.pre.in b/Makefile.pre.in
index d5be6edf784e2..3dc131b6c9651 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -308,7 +308,6 @@ COVERAGE_REPORT_OPTIONS=--no-branch-coverage --title "CPython lcov report"
 # Modules
 MODULE_OBJS=	\
 		Modules/config.o \
-		Modules/getpath.o \
 		Modules/main.o \
 		Modules/gcmodule.o
 
@@ -505,6 +504,7 @@ LIBRARY_OBJS_OMIT_FROZEN=	\
 LIBRARY_OBJS=	\
 		$(LIBRARY_OBJS_OMIT_FROZEN) \
 		$(DEEPFREEZE_OBJS) \
+		Modules/getpath.o \
 		Python/frozen.o
 
 ##########################################################################
@@ -1061,8 +1061,11 @@ FROZEN_FILES_OUT = \
 
 Programs/_freeze_module.o: Programs/_freeze_module.c Makefile
 
-Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN)
-	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
+Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile
+	$(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Modules/getpath_noop.c
+
+Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Modules/getpath_noop.o
+	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Modules/getpath_noop.o $(LIBS) $(MODLIBS) $(SYSLIBS)
 
 # BEGIN: freezing modules
 
@@ -1130,6 +1133,10 @@ Python/frozen_modules/frozen_only.h: $(FREEZE_MODULE) Tools/freeze/flag.py
 
 Tools/scripts/freeze_modules.py: $(FREEZE_MODULE)
 
+# We manually freeze getpath.py rather than through freeze_modules
+Modules/getpath.h: Programs/_freeze_module Modules/getpath.py
+	Programs/_freeze_module getpath $(srcdir)/Modules/getpath.py $(srcdir)/Modules/getpath.h
+
 .PHONY: regen-frozen
 regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN)
 	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py
@@ -1170,12 +1177,13 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \
 	      -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \
 	      -o $@ $(srcdir)/Modules/getbuildinfo.c
 
-Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile
+Modules/getpath.o: $(srcdir)/Modules/getpath.c Modules/getpath.h Makefile
 	$(CC) -c $(PY_CORE_CFLAGS) -DPYTHONPATH='"$(PYTHONPATH)"' \
 		-DPREFIX='"$(prefix)"' \
 		-DEXEC_PREFIX='"$(exec_prefix)"' \
 		-DVERSION='"$(VERSION)"' \
 		-DVPATH='"$(VPATH)"' \
+		-DPLATLIBDIR='"$(PLATLIBDIR)"' \
 		-o $@ $(srcdir)/Modules/getpath.c
 
 Programs/python.o: $(srcdir)/Programs/python.c
@@ -1210,11 +1218,6 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydt
 		$(MULTIARCH_CPPFLAGS) \
 		-o $@ $(srcdir)/Python/sysmodule.c
 
-Python/initconfig.o: $(srcdir)/Python/initconfig.c
-	$(CC) -c $(PY_CORE_CFLAGS) \
-		-DPLATLIBDIR='"$(PLATLIBDIR)"' \
-		-o $@ $(srcdir)/Python/initconfig.c
-
 $(IO_OBJS): $(IO_H)
 
 .PHONY: regen-pegen-metaparser
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-23-00-39-31.bpo-45582.YONPuo.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-23-00-39-31.bpo-45582.YONPuo.rst
new file mode 100644
index 0000000000000..45fa75e870d78
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-23-00-39-31.bpo-45582.YONPuo.rst	
@@ -0,0 +1,3 @@
+Path calculation (known as ``getpath``) has been reimplemented as a frozen
+Python module. This should have no visible impact, but may affect
+calculation of all paths referenced in :mod:`sys` and :mod:`sysconfig`.
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index c230ba28d6111..19babb06f5635 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -14,10 +14,11 @@
 #include "Python.h"
 #include "pycore_atomic_funcs.h" // _Py_atomic_int_get()
 #include "pycore_bitutils.h"     // _Py_bswap32()
-#include "pycore_fileutils.h"    // _Py_normalize_path
+#include "pycore_fileutils.h"    // _Py_normpath
 #include "pycore_gc.h"           // PyGC_Head
 #include "pycore_hashtable.h"    // _Py_hashtable_new()
 #include "pycore_initconfig.h"   // _Py_GetConfigsAsDict()
+#include "pycore_pathconfig.h"   // _PyPathConfig_ClearGlobal()
 #include "pycore_interp.h"       // _PyInterpreterState_GetConfigCopy()
 #include "pycore_pyerrors.h"     // _Py_UTF8_Edit_Cost()
 #include "pycore_pystate.h"      // _PyThreadState_GET()
@@ -273,6 +274,14 @@ test_set_config(PyObject *Py_UNUSED(self), PyObject *dict)
 }
 
 
+static PyObject *
+test_reset_path_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(arg))
+{
+    _PyPathConfig_ClearGlobal();
+    Py_RETURN_NONE;
+}
+
+
 static PyObject*
 test_atomic_funcs(PyObject *self, PyObject *Py_UNUSED(args))
 {
@@ -378,15 +387,15 @@ normalize_path(PyObject *self, PyObject *filename)
         return NULL;
     }
 
-    wchar_t buf[MAXPATHLEN + 1];
-    int res = _Py_normalize_path(encoded, buf, Py_ARRAY_LENGTH(buf));
+    PyObject *result = PyUnicode_FromWideChar(_Py_normpath(encoded, size), -1);
     PyMem_Free(encoded);
-    if (res != 0) {
-        PyErr_SetString(PyExc_ValueError, "string too long");
-        return NULL;
-    }
 
-    return PyUnicode_FromWideChar(buf, -1);
+    return result;
+}
+
+static PyObject *
+get_getpath_codeobject(PyObject *self, PyObject *Py_UNUSED(args)) {
+    return _Py_Get_Getpath_CodeObject();
 }
 
 
@@ -399,9 +408,11 @@ static PyMethodDef TestMethods[] = {
     {"test_hashtable", test_hashtable, METH_NOARGS},
     {"get_config", test_get_config, METH_NOARGS},
     {"set_config", test_set_config, METH_O},
+    {"reset_path_config", test_reset_path_config, METH_NOARGS},
     {"test_atomic_funcs", test_atomic_funcs, METH_NOARGS},
     {"test_edit_cost", test_edit_cost, METH_NOARGS},
     {"normalize_path", normalize_path, METH_O, NULL},
+    {"get_getpath_codeobject", get_getpath_codeobject, METH_NOARGS, NULL},
     {NULL, NULL} /* sentinel */
 };
 
diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h
index 7921c222b9009..86da08711fd44 100644
--- a/Modules/clinic/posixmodule.c.h
+++ b/Modules/clinic/posixmodule.c.h
@@ -1317,6 +1317,38 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py
 
 #endif /* defined(MS_WINDOWS) */
 
+PyDoc_STRVAR(os__path_normpath__doc__,
+"_path_normpath($module, /, path)\n"
+"--\n"
+"\n"
+"Basic path normalization.");
+
+#define OS__PATH_NORMPATH_METHODDEF    \
+    {"_path_normpath", (PyCFunction)(void(*)(void))os__path_normpath, METH_FASTCALL|METH_KEYWORDS, os__path_normpath__doc__},
+
+static PyObject *
+os__path_normpath_impl(PyObject *module, PyObject *path);
+
+static PyObject *
+os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"path", NULL};
+    static _PyArg_Parser _parser = {NULL, _keywords, "_path_normpath", 0};
+    PyObject *argsbuf[1];
+    PyObject *path;
+
+    args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    path = args[0];
+    return_value = os__path_normpath_impl(module, path);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(os_mkdir__doc__,
 "mkdir($module, /, path, mode=511, *, dir_fd=None)\n"
 "--\n"
@@ -9263,4 +9295,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na
 #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF
     #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF
 #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */
-/*[clinic end generated code: output=65a85d7d3f2c487e input=a9049054013a1b77]*/
+/*[clinic end generated code: output=05505f171cdcff72 input=a9049054013a1b77]*/
diff --git a/Modules/getpath.c b/Modules/getpath.c
index 4dbd502ddcf04..32d5db9d2c4dc 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -1,1617 +1,935 @@
 /* Return the initial module search path. */
 
 #include "Python.h"
-#include "pycore_fileutils.h"
+#include "marshal.h"              // PyMarshal_ReadObjectFromString
+#include "osdefs.h"               // DELIM
 #include "pycore_initconfig.h"
+#include "pycore_fileutils.h"
 #include "pycore_pathconfig.h"
-#include "osdefs.h"               // DELIM
+#include "pycore_pymem.h"         // _PyMem_SetDefaultAllocator()
+#include <wchar.h>
 
-#include <stdlib.h>               // getenv()
-#include <string.h>
-#include <sys/types.h>
+#ifdef MS_WINDOWS
+#  include <windows.h>            // GetFullPathNameW(), MAX_PATH
+#  include <pathcch.h>
+#endif
 
 #ifdef __APPLE__
 #  include <mach-o/dyld.h>
 #endif
 
-/* Search in some common locations for the associated Python libraries.
- *
- * Two directories must be found, the platform independent directory
- * (prefix), containing the common .py and .pyc files, and the platform
- * dependent directory (exec_prefix), containing the shared library
- * modules.  Note that prefix and exec_prefix can be the same directory,
- * but for some installations, they are different.
- *
- * Py_GetPath() carries out separate searches for prefix and exec_prefix.
- * Each search tries a number of different locations until a ``landmark''
- * file or directory is found.  If no prefix or exec_prefix is found, a
- * warning message is issued and the preprocessor defined PREFIX and
- * EXEC_PREFIX are used (even though they will not work); python carries on
- * as best as is possible, but most imports will fail.
- *
- * Before any searches are done, the location of the executable is
- * determined.  If argv[0] has one or more slashes in it, it is used
- * unchanged.  Otherwise, it must have been invoked from the shell's path,
- * so we search $PATH for the named executable and use that.  If the
- * executable was not found on $PATH (or there was no $PATH environment
- * variable), the original argv[0] string is used.
- *
- * Next, the executable location is examined to see if it is a symbolic
- * link.  If so, the link is chased (correctly interpreting a relative
- * pathname if one is found) and the directory of the link target is used.
- *
- * Finally, argv0_path is set to the directory containing the executable
- * (i.e. the last component is stripped).
- *
- * With argv0_path in hand, we perform a number of steps.  The same steps
- * are performed for prefix and for exec_prefix, but with a different
- * landmark.
- *
- * Step 1. Are we running python out of the build directory?  This is
- * checked by looking for a different kind of landmark relative to
- * argv0_path.  For prefix, the landmark's path is derived from the VPATH
- * preprocessor variable (taking into account that its value is almost, but
- * not quite, what we need).  For exec_prefix, the landmark is
- * pybuilddir.txt.  If the landmark is found, we're done.
- *
- * For the remaining steps, the prefix landmark will always be
- * lib/python$VERSION/os.py and the exec_prefix will always be
- * lib/python$VERSION/lib-dynload, where $VERSION is Python's version
- * number as supplied by the Makefile.  Note that this means that no more
- * build directory checking is performed; if the first step did not find
- * the landmarks, the assumption is that python is running from an
- * installed setup.
- *
- * Step 2. See if the $PYTHONHOME environment variable points to the
- * installed location of the Python libraries.  If $PYTHONHOME is set, then
- * it points to prefix and exec_prefix.  $PYTHONHOME can be a single
- * directory, which is used for both, or the prefix and exec_prefix
- * directories separated by a colon.
- *
- * Step 3. Try to find prefix and exec_prefix relative to argv0_path,
- * backtracking up the path until it is exhausted.  This is the most common
- * step to succeed.  Note that if prefix and exec_prefix are different,
- * exec_prefix is more likely to be found; however if exec_prefix is a
- * subdirectory of prefix, both will be found.
- *
- * Step 4. Search the directories pointed to by the preprocessor variables
- * PREFIX and EXEC_PREFIX.  These are supplied by the Makefile but can be
- * passed in as options to the configure script.
- *
- * That's it!
- *
- * Well, almost.  Once we have determined prefix and exec_prefix, the
- * preprocessor variable PYTHONPATH is used to construct a path.  Each
- * relative path on PYTHONPATH is prefixed with prefix.  Then the directory
- * containing the shared library modules is appended.  The environment
- * variable $PYTHONPATH is inserted in front of it all.  Finally, the
- * prefix and exec_prefix globals are tweaked so they reflect the values
- * expected by other code, by stripping the "lib/python$VERSION/..." stuff
- * off.  If either points to the build directory, the globals are reset to
- * the corresponding preprocessor variables (so sys.prefix will reflect the
- * installation location, even though sys.path points into the build
- * directory).  This seems to make more sense given that currently the only
- * known use of sys.prefix and sys.exec_prefix is for the ILU installation
- * process to find the installed Python tree.
- *
- * An embedding application can use Py_SetPath() to override all of
- * these automatic path computations.
- *
- * NOTE: Windows MSVC builds use PC/getpathp.c instead!
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
+/* Reference the precompiled getpath.py */
+#include "getpath.h"
 
 #if (!defined(PREFIX) || !defined(EXEC_PREFIX) \
-        || !defined(VERSION) || !defined(VPATH))
-#error "PREFIX, EXEC_PREFIX, VERSION and VPATH macros must be defined"
+        || !defined(VERSION) || !defined(VPATH) \
+        || !defined(PLATLIBDIR))
+#error "PREFIX, EXEC_PREFIX, VERSION, VPATH and PLATLIBDIR macros must be defined"
 #endif
 
-#ifndef LANDMARK
-#define LANDMARK L"os.py"
+#if !defined(PYTHONPATH)
+#define PYTHONPATH NULL
 #endif
 
-#define BUILD_LANDMARK L"Modules/Setup.local"
-
-#define PATHLEN_ERR() _PyStatus_ERR("path configuration: path too long")
-
-typedef struct {
-    wchar_t *path_env;                 /* PATH environment variable */
-
-    wchar_t *pythonpath_macro;         /* PYTHONPATH macro */
-    wchar_t *prefix_macro;             /* PREFIX macro */
-    wchar_t *exec_prefix_macro;        /* EXEC_PREFIX macro */
-    wchar_t *vpath_macro;              /* VPATH macro */
-
-    wchar_t *lib_python;               /* <platlibdir> / "pythonX.Y" */
-
-    int prefix_found;         /* found platform independent libraries? */
-    int exec_prefix_found;    /* found the platform dependent libraries? */
-
-    int warnings;
-    const wchar_t *pythonpath_env;
-    const wchar_t *platlibdir;
-
-    wchar_t *argv0_path;
-    wchar_t *zip_path;
-    wchar_t *prefix;
-    wchar_t *exec_prefix;
-} PyCalculatePath;
-
-static const wchar_t delimiter[2] = {DELIM, '\0'};
-static const wchar_t separator[2] = {SEP, '\0'};
-
-
-static void
-reduce(wchar_t *dir)
-{
-    size_t i = wcslen(dir);
-    while (i > 0 && dir[i] != SEP) {
-        --i;
-    }
-    dir[i] = '\0';
-}
-
+#if !defined(PYDEBUGEXT)
+#define PYDEBUGEXT NULL
+#endif
 
-/* Is file, not directory */
-static int
-isfile(const wchar_t *filename)
-{
-    struct stat buf;
-    if (_Py_wstat(filename, &buf) != 0) {
-        return 0;
-    }
-    if (!S_ISREG(buf.st_mode)) {
-        return 0;
-    }
-    return 1;
-}
+#if !defined(PYWINVER)
+#ifdef MS_DLL_ID
+#define PYWINVER MS_DLL_ID
+#else
+#define PYWINVER NULL
+#endif
+#endif
 
+#if !defined(EXE_SUFFIX)
+#if defined(MS_WINDOWS) || defined(__CYGWIN__) || defined(__MINGW32__)
+#define EXE_SUFFIX L".exe"
+#else
+#define EXE_SUFFIX NULL
+#endif
+#endif
 
-/* Is executable file */
-static int
-isxfile(const wchar_t *filename)
-{
-    struct stat buf;
-    if (_Py_wstat(filename, &buf) != 0) {
-        return 0;
-    }
-    if (!S_ISREG(buf.st_mode)) {
-        return 0;
-    }
-    if ((buf.st_mode & 0111) == 0) {
-        return 0;
-    }
-    return 1;
-}
 
+/* HELPER FUNCTIONS for getpath.py */
 
-/* Is directory */
-static int
-isdir(const wchar_t *filename)
+static PyObject *
+getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    struct stat buf;
-    if (_Py_wstat(filename, &buf) != 0) {
-        return 0;
+    PyObject *r = NULL;
+    PyObject *pathobj;
+    const wchar_t *path;
+    if (!PyArg_ParseTuple(args, "U", &pathobj)) {
+        return NULL;
     }
-    if (!S_ISDIR(buf.st_mode)) {
-        return 0;
+    Py_ssize_t len;
+    path = PyUnicode_AsWideCharString(pathobj, &len);
+    if (path) {
+        wchar_t *abs;
+        if (_Py_abspath(path, &abs) == 0 && abs) {
+            r = PyUnicode_FromWideChar(_Py_normpath(abs, -1), -1);
+            PyMem_RawFree((void *)abs);
+        } else {
+            PyErr_SetString(PyExc_OSError, "failed to make path absolute");
+        }
+        PyMem_Free((void *)path);
     }
-    return 1;
+    return r;
 }
 
 
-/* Add a path component, by appending stuff to buffer.
-   buflen: 'buffer' length in characters including trailing NUL.
-
-   If path2 is empty:
-
-   - if path doesn't end with SEP and is not empty, add SEP to path
-   - otherwise, do nothing. */
-static PyStatus
-joinpath(wchar_t *path, const wchar_t *path2, size_t path_len)
+static PyObject *
+getpath_basename(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    if (_Py_isabs(path2)) {
-        if (wcslen(path2) >= path_len) {
-            return PATHLEN_ERR();
-        }
-        wcscpy(path, path2);
-    }
-    else {
-        if (_Py_add_relfile(path, path2, path_len) < 0) {
-            return PATHLEN_ERR();
-        }
-        return _PyStatus_OK();
+    const char *path;
+    if (!PyArg_ParseTuple(args, "s", &path)) {
+        return NULL;
     }
-    return _PyStatus_OK();
+    const char *name = strrchr(path, SEP);
+    return PyUnicode_FromString(name ? name + 1 : path);
 }
 
 
-static wchar_t*
-substring(const wchar_t *str, size_t len)
+static PyObject *
+getpath_dirname(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    wchar_t *substr = PyMem_RawMalloc((len + 1) * sizeof(wchar_t));
-    if (substr == NULL) {
+    const char *path;
+    if (!PyArg_ParseTuple(args, "s", &path)) {
         return NULL;
     }
-
-    if (len) {
-        memcpy(substr, str, len * sizeof(wchar_t));
+    const char *name = strrchr(path, SEP);
+    if (!name) {
+        return PyUnicode_FromStringAndSize(NULL, 0);
     }
-    substr[len] = L'\0';
-    return substr;
+    return PyUnicode_FromStringAndSize(path, (name - path));
 }
 
 
-static wchar_t*
-joinpath2(const wchar_t *path, const wchar_t *path2)
+static PyObject *
+getpath_isabs(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    if (_Py_isabs(path2)) {
-        return _PyMem_RawWcsdup(path2);
+    PyObject *r = NULL;
+    PyObject *pathobj;
+    const wchar_t *path;
+    if (!PyArg_ParseTuple(args, "U", &pathobj)) {
+        return NULL;
     }
-    return _Py_join_relfile(path, path2);
-}
-
-
-static inline int
-safe_wcscpy(wchar_t *dst, const wchar_t *src, size_t n)
-{
-    size_t srclen = wcslen(src);
-    if (n <= srclen) {
-        dst[0] = L'\0';
-        return -1;
+    path = PyUnicode_AsWideCharString(pathobj, NULL);
+    if (path) {
+        r = _Py_isabs(path) ? Py_True : Py_False;
+        PyMem_Free((void *)path);
     }
-    memcpy(dst, src, (srclen + 1) * sizeof(wchar_t));
-    return 0;
+    Py_XINCREF(r);
+    return r;
 }
 
 
-/* copy_absolute requires that path be allocated at least
-   'abs_path_len' characters (including trailing NUL). */
-static PyStatus
-copy_absolute(wchar_t *abs_path, const wchar_t *path, size_t abs_path_len)
+static PyObject *
+getpath_hassuffix(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    if (_Py_isabs(path)) {
-        if (safe_wcscpy(abs_path, path, abs_path_len) < 0) {
-            return PATHLEN_ERR();
-        }
+    PyObject *r = NULL;
+    PyObject *pathobj;
+    PyObject *suffixobj;
+    const wchar_t *path;
+    const wchar_t *suffix;
+    if (!PyArg_ParseTuple(args, "UU", &pathobj, &suffixobj)) {
+        return NULL;
     }
-    else {
-        if (!_Py_wgetcwd(abs_path, abs_path_len)) {
-            /* unable to get the current directory */
-            if (safe_wcscpy(abs_path, path, abs_path_len) < 0) {
-                return PATHLEN_ERR();
+    Py_ssize_t len, suffixLen;
+    path = PyUnicode_AsWideCharString(pathobj, &len);
+    if (path) {
+        suffix = PyUnicode_AsWideCharString(suffixobj, &suffixLen);
+        if (suffix) {
+            if (suffixLen < len ||
+#ifdef MS_WINDOWS
+                wcsicmp(&path[len - suffixLen], suffix) != 0
+#else
+                wcscmp(&path[len - suffixLen], suffix) != 0
+#endif
+            ) {
+                r = Py_False;
+            } else {
+                r = Py_True;
             }
-            return _PyStatus_OK();
-        }
-        if (path[0] == '.' && path[1] == SEP) {
-            path += 2;
-        }
-        PyStatus status = joinpath(abs_path, path, abs_path_len);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
+            Py_INCREF(r);
+            PyMem_Free((void *)suffix);
         }
+        PyMem_Free((void *)path);
     }
-    return _PyStatus_OK();
-}
-
-
-/* path_len: path length in characters including trailing NUL */
-static PyStatus
-absolutize(wchar_t **path_p)
-{
-    assert(!_Py_isabs(*path_p));
-
-    wchar_t abs_path[MAXPATHLEN+1];
-    wchar_t *path = *path_p;
-
-    PyStatus status = copy_absolute(abs_path, path, Py_ARRAY_LENGTH(abs_path));
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-
-    PyMem_RawFree(*path_p);
-    *path_p = _PyMem_RawWcsdup(abs_path);
-    if (*path_p == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-    return _PyStatus_OK();
+    return r;
 }
 
 
-/* Is module -- check for .pyc too */
-static PyStatus
-ismodule(const wchar_t *path, int *result)
+static PyObject *
+getpath_isdir(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    wchar_t *filename = joinpath2(path, LANDMARK);
-    if (filename == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-
-    if (isfile(filename)) {
-        PyMem_RawFree(filename);
-        *result = 1;
-        return _PyStatus_OK();
-    }
-
-    /* Check for the compiled version of prefix. */
-    size_t len = wcslen(filename);
-    wchar_t *pyc = PyMem_RawMalloc((len + 2) * sizeof(wchar_t));
-    if (pyc == NULL) {
-        PyMem_RawFree(filename);
-        return _PyStatus_NO_MEMORY();
+    PyObject *r = NULL;
+    PyObject *pathobj;
+    const wchar_t *path;
+    if (!PyArg_ParseTuple(args, "U", &pathobj)) {
+        return NULL;
     }
-
-    memcpy(pyc, filename, len * sizeof(wchar_t));
-    pyc[len] = L'c';
-    pyc[len + 1] = L'\0';
-    *result = isfile(pyc);
-
-    PyMem_RawFree(filename);
-    PyMem_RawFree(pyc);
-
-    return _PyStatus_OK();
-}
-
-
-#if defined(__CYGWIN__) || defined(__MINGW32__)
-#ifndef EXE_SUFFIX
-#define EXE_SUFFIX L".exe"
+    path = PyUnicode_AsWideCharString(pathobj, NULL);
+    if (path) {
+#ifdef MS_WINDOWS
+        r = (GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) ? Py_True : Py_False;
+#else
+        struct stat st;
+        r = (_Py_wstat(path, &st) == 0) && S_ISDIR(st.st_mode) ? Py_True : Py_False;
 #endif
-
-/* pathlen: 'path' length in characters including trailing NUL */
-static PyStatus
-add_exe_suffix(wchar_t **progpath_p)
-{
-    wchar_t *progpath = *progpath_p;
-
-    /* Check for already have an executable suffix */
-    size_t n = wcslen(progpath);
-    size_t s = wcslen(EXE_SUFFIX);
-    if (wcsncasecmp(EXE_SUFFIX, progpath + n - s, s) == 0) {
-        return _PyStatus_OK();
-    }
-
-    wchar_t *progpath2 = PyMem_RawMalloc((n + s + 1) * sizeof(wchar_t));
-    if (progpath2 == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-
-    memcpy(progpath2, progpath, n * sizeof(wchar_t));
-    memcpy(progpath2 + n, EXE_SUFFIX, s * sizeof(wchar_t));
-    progpath2[n+s] = L'\0';
-
-    if (isxfile(progpath2)) {
-        PyMem_RawFree(*progpath_p);
-        *progpath_p = progpath2;
-    }
-    else {
-        PyMem_RawFree(progpath2);
+        PyMem_Free((void *)path);
     }
-    return _PyStatus_OK();
+    Py_XINCREF(r);
+    return r;
 }
-#endif
 
 
-/* search_for_prefix requires that argv0_path be no more than MAXPATHLEN
-   bytes long.
-*/
-static PyStatus
-search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
-                  wchar_t *prefix, size_t prefix_len, int *found)
+static PyObject *
+getpath_isfile(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    PyStatus status;
-
-    /* If PYTHONHOME is set, we believe it unconditionally */
-    if (pathconfig->home) {
-        /* Path: <home> / <lib_python> */
-        if (safe_wcscpy(prefix, pathconfig->home, prefix_len) < 0) {
-            return PATHLEN_ERR();
-        }
-        wchar_t *delim = wcschr(prefix, DELIM);
-        if (delim) {
-            *delim = L'\0';
-        }
-        status = joinpath(prefix, calculate->lib_python, prefix_len);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-        *found = 1;
-        return _PyStatus_OK();
-    }
-
-    /* Check to see if argv0_path is in the build directory
-
-       Path: <argv0_path> / <BUILD_LANDMARK define> */
-    wchar_t *path = joinpath2(calculate->argv0_path, BUILD_LANDMARK);
-    if (path == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-
-    int is_build_dir = isfile(path);
-    PyMem_RawFree(path);
-
-    if (is_build_dir) {
-        /* argv0_path is the build directory (BUILD_LANDMARK exists),
-           now also check LANDMARK using ismodule(). */
-
-        /* Path: <argv0_path> / <VPATH macro> / Lib */
-        /* or if VPATH is empty: <argv0_path> / Lib */
-        if (safe_wcscpy(prefix, calculate->argv0_path, prefix_len) < 0) {
-            return PATHLEN_ERR();
-        }
-
-        status = joinpath(prefix, calculate->vpath_macro, prefix_len);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-
-        status = joinpath(prefix, L"Lib", prefix_len);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-
-        int module;
-        status = ismodule(prefix, &module);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-        if (module) {
-            /* BUILD_LANDMARK and LANDMARK found */
-            *found = -1;
-            return _PyStatus_OK();
-        }
-    }
-
-    /* Search from argv0_path, until root is found */
-    status = copy_absolute(prefix, calculate->argv0_path, prefix_len);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-
-    do {
-        /* Path: <argv0_path or substring> / <lib_python> / LANDMARK */
-        size_t n = wcslen(prefix);
-        status = joinpath(prefix, calculate->lib_python, prefix_len);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-
-        int module;
-        status = ismodule(prefix, &module);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-        if (module) {
-            *found = 1;
-            return _PyStatus_OK();
-        }
-        prefix[n] = L'\0';
-        reduce(prefix);
-    } while (prefix[0]);
-
-    /* Look at configure's PREFIX.
-       Path: <PREFIX macro> / <lib_python> / LANDMARK */
-    if (safe_wcscpy(prefix, calculate->prefix_macro, prefix_len) < 0) {
-        return PATHLEN_ERR();
-    }
-    status = joinpath(prefix, calculate->lib_python, prefix_len);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-
-    int module;
-    status = ismodule(prefix, &module);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+    PyObject *r = NULL;
+    PyObject *pathobj;
+    const wchar_t *path;
+    if (!PyArg_ParseTuple(args, "U", &pathobj)) {
+        return NULL;
     }
-    if (module) {
-        *found = 1;
-        return _PyStatus_OK();
+    path = PyUnicode_AsWideCharString(pathobj, NULL);
+    if (path) {
+#ifdef MS_WINDOWS
+        r = !(GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) ? Py_True : Py_False;
+#else
+        struct stat st;
+        r = (_Py_wstat(path, &st) == 0) && S_ISREG(st.st_mode) ? Py_True : Py_False;
+#endif
+        PyMem_Free((void *)path);
     }
-
-    /* Fail */
-    *found = 0;
-    return _PyStatus_OK();
+    Py_XINCREF(r);
+    return r;
 }
 
 
-static PyStatus
-calculate_set_stdlib_dir(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+static PyObject *
+getpath_isxfile(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    // Note that, unlike calculate_set_prefix(), here we allow a negative
-    // prefix_found.  That means the source tree Lib dir gets used.
-    if (!calculate->prefix_found) {
-        return _PyStatus_OK();
-    }
-    PyStatus status;
-    wchar_t *prefix = calculate->prefix;
-    if (!_Py_isabs(prefix)) {
-        prefix = _PyMem_RawWcsdup(prefix);
-        if (prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-        status = absolutize(&prefix);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-    wchar_t buf[MAXPATHLEN + 1];
-    int res = _Py_normalize_path(prefix, buf, Py_ARRAY_LENGTH(buf));
-    if (prefix != calculate->prefix) {
-        PyMem_RawFree(prefix);
-    }
-    if (res < 0) {
-        return PATHLEN_ERR();
+    PyObject *r = NULL;
+    PyObject *pathobj;
+    const wchar_t *path;
+    Py_ssize_t cchPath;
+    if (!PyArg_ParseTuple(args, "U", &pathobj)) {
+        return NULL;
     }
-    pathconfig->stdlib_dir = _PyMem_RawWcsdup(buf);
-    if (pathconfig->stdlib_dir == NULL) {
-        return _PyStatus_NO_MEMORY();
+    path = PyUnicode_AsWideCharString(pathobj, &cchPath);
+    if (path) {
+#ifdef MS_WINDOWS
+        const wchar_t *ext;
+        r = (GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) &&
+            SUCCEEDED(PathCchFindExtension(path, cchPath, &ext)) &&
+            (CompareStringOrdinal(ext, -1, L".exe", -1, 1 /* ignore case */) == CSTR_EQUAL)
+            ? Py_True : Py_False;
+#else
+        struct stat st;
+        r = (_Py_wstat(path, &st) == 0) &&
+            S_ISREG(st.st_mode) &&
+            (st.st_mode & 0111)
+            ? Py_True : Py_False;
+#endif
+        PyMem_Free((void *)path);
     }
-    return _PyStatus_OK();
+    Py_XINCREF(r);
+    return r;
 }
 
 
-static PyStatus
-calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+static PyObject *
+getpath_joinpath(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    wchar_t prefix[MAXPATHLEN+1];
-    memset(prefix, 0, sizeof(prefix));
-    size_t prefix_len = Py_ARRAY_LENGTH(prefix);
-
-    PyStatus status;
-    status = search_for_prefix(calculate, pathconfig,
-                               prefix, prefix_len,
-                               &calculate->prefix_found);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-
-    if (!calculate->prefix_found) {
-        if (calculate->warnings) {
-            fprintf(stderr,
-                "Could not find platform independent libraries <prefix>\n");
-        }
-
-        calculate->prefix = joinpath2(calculate->prefix_macro,
-                                      calculate->lib_python);
-    }
-    else {
-        calculate->prefix = _PyMem_RawWcsdup(prefix);
-    }
-
-    if (calculate->prefix == NULL) {
-        return _PyStatus_NO_MEMORY();
+    if (!PyTuple_Check(args)) {
+        PyErr_SetString(PyExc_TypeError, "requires tuple of arguments");
+        return NULL;
     }
-    return _PyStatus_OK();
-}
-
-
-static PyStatus
-calculate_set_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
-{
-    /* Reduce prefix and exec_prefix to their essence,
-     * e.g. /usr/local/lib/python1.5 is reduced to /usr/local.
-     * If we're loading relative to the build directory,
-     * return the compiled-in defaults instead.
-     */
-    if (calculate->prefix_found > 0) {
-        wchar_t *prefix = _PyMem_RawWcsdup(calculate->prefix);
-        if (prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-
-        reduce(prefix);
-        reduce(prefix);
-        if (prefix[0]) {
-            pathconfig->prefix = prefix;
-        }
-        else {
-            PyMem_RawFree(prefix);
-
-            /* The prefix is the root directory, but reduce() chopped
-               off the "/". */
-            pathconfig->prefix = _PyMem_RawWcsdup(separator);
-            if (pathconfig->prefix == NULL) {
-                return _PyStatus_NO_MEMORY();
+    Py_ssize_t n = PyTuple_GET_SIZE(args);
+    if (n == 0) {
+        return PyUnicode_FromString(NULL);
+    }
+    /* Convert all parts to wchar and accumulate max final length */
+    wchar_t **parts = (wchar_t **)PyMem_Malloc(n * sizeof(wchar_t *));
+    memset(parts, 0, n * sizeof(wchar_t *));
+    Py_ssize_t cchFinal = 0;
+    Py_ssize_t first = 0;
+
+    for (Py_ssize_t i = 0; i < n; ++i) {
+        PyObject *s = PyTuple_GET_ITEM(args, i);
+        Py_ssize_t cch;
+        if (s == Py_None) {
+            cch = 0;
+        } else if (PyUnicode_Check(s)) {
+            parts[i] = PyUnicode_AsWideCharString(s, &cch);
+            if (!parts[i]) {
+                cchFinal = -1;
+                break;
             }
+            if (_Py_isabs(parts[i])) {
+                first = i;
+            }
+        } else {
+            PyErr_SetString(PyExc_TypeError, "all arguments to joinpath() must be str or None");
+            cchFinal = -1;
+            break;
         }
+        cchFinal += cch + 1;
     }
-    else {
-        pathconfig->prefix = _PyMem_RawWcsdup(calculate->prefix_macro);
-        if (pathconfig->prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-    }
-    return _PyStatus_OK();
-}
-
-
-static PyStatus
-calculate_pybuilddir(const wchar_t *argv0_path,
-                     wchar_t *exec_prefix, size_t exec_prefix_len,
-                     int *found)
-{
-    PyStatus status;
-
-    /* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
-       is written by setup.py and contains the relative path to the location
-       of shared library modules.
-
-       Filename: <argv0_path> / "pybuilddir.txt" */
-    wchar_t *filename = joinpath2(argv0_path, L"pybuilddir.txt");
-    if (filename == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-
-    FILE *fp = _Py_wfopen(filename, L"rb");
-    PyMem_RawFree(filename);
-    if (fp == NULL) {
-        errno = 0;
-        return _PyStatus_OK();
-    }
-
-    char buf[MAXPATHLEN + 1];
-    size_t n = fread(buf, 1, Py_ARRAY_LENGTH(buf) - 1, fp);
-    buf[n] = '\0';
-    fclose(fp);
-
-    size_t dec_len;
-    wchar_t *pybuilddir = _Py_DecodeUTF8_surrogateescape(buf, n, &dec_len);
-    if (!pybuilddir) {
-        return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len);
-    }
-
-    /* Path: <argv0_path> / <pybuilddir content> */
-    if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
-        PyMem_RawFree(pybuilddir);
-        return PATHLEN_ERR();
-    }
-    status = joinpath(exec_prefix, pybuilddir, exec_prefix_len);
-    PyMem_RawFree(pybuilddir);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-
-    *found = -1;
-    return _PyStatus_OK();
-}
 
-
-/* search_for_exec_prefix requires that argv0_path be no more than
-   MAXPATHLEN bytes long.
-*/
-static PyStatus
-search_for_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
-                       wchar_t *exec_prefix, size_t exec_prefix_len,
-                       int *found)
-{
-    PyStatus status;
-
-    /* If PYTHONHOME is set, we believe it unconditionally */
-    if (pathconfig->home) {
-        /* Path: <home> / <lib_python> / "lib-dynload" */
-        wchar_t *delim = wcschr(pathconfig->home, DELIM);
-        if (delim) {
-            if (safe_wcscpy(exec_prefix, delim+1, exec_prefix_len) < 0) {
-                return PATHLEN_ERR();
-            }
+    wchar_t *final = cchFinal > 0 ? (wchar_t *)PyMem_Malloc(cchFinal * sizeof(wchar_t)) : NULL;
+    if (!final) {
+        for (Py_ssize_t i = 0; i < n; ++i) {
+            PyMem_Free(parts[i]);
         }
-        else {
-            if (safe_wcscpy(exec_prefix, pathconfig->home, exec_prefix_len) < 0) {
-                return PATHLEN_ERR();
-            }
-        }
-        status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
+        PyMem_Free(parts);
+        if (cchFinal) {
+            PyErr_NoMemory();
+            return NULL;
         }
-        status = joinpath(exec_prefix, L"lib-dynload", exec_prefix_len);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-        *found = 1;
-        return _PyStatus_OK();
-    }
-
-    /* Check for pybuilddir.txt */
-    assert(*found == 0);
-    status = calculate_pybuilddir(calculate->argv0_path,
-                                  exec_prefix, exec_prefix_len, found);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-    if (*found) {
-        return _PyStatus_OK();
+        return PyUnicode_FromStringAndSize(NULL, 0);
     }
 
-    /* Search from argv0_path, until root is found */
-    status = copy_absolute(exec_prefix, calculate->argv0_path, exec_prefix_len);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-
-    do {
-        /* Path: <argv0_path or substring> / <lib_python> / "lib-dynload" */
-        size_t n = wcslen(exec_prefix);
-        status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-        status = joinpath(exec_prefix, L"lib-dynload", exec_prefix_len);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
+    final[0] = '\0';
+    /* Now join all the paths. The final result should be shorter than the buffer */
+    for (Py_ssize_t i = 0; i < n; ++i) {
+        if (!parts[i]) {
+            continue;
         }
-        if (isdir(exec_prefix)) {
-            *found = 1;
-            return _PyStatus_OK();
+        if (i >= first && final) {
+            if (!final[0]) {
+                /* final is definitely long enough to fit any individual part */
+                wcscpy(final, parts[i]);
+            } else if (_Py_add_relfile(final, parts[i], cchFinal) < 0) {
+                /* if we fail, keep iterating to free memory, but stop adding parts */
+                PyMem_Free(final);
+                final = NULL;
+            }
         }
-        exec_prefix[n] = L'\0';
-        reduce(exec_prefix);
-    } while (exec_prefix[0]);
-
-    /* Look at configure's EXEC_PREFIX.
-
-       Path: <EXEC_PREFIX macro> / <lib_python> / "lib-dynload" */
-    if (safe_wcscpy(exec_prefix, calculate->exec_prefix_macro, exec_prefix_len) < 0) {
-        return PATHLEN_ERR();
+        PyMem_Free(parts[i]);
     }
-    status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-    status = joinpath(exec_prefix, L"lib-dynload", exec_prefix_len);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-    if (isdir(exec_prefix)) {
-        *found = 1;
-        return _PyStatus_OK();
+    PyMem_Free(parts);
+    if (!final) {
+        PyErr_SetString(PyExc_SystemError, "failed to join paths");
+        return NULL;
     }
-
-    /* Fail */
-    *found = 0;
-    return _PyStatus_OK();
+    PyObject *r = PyUnicode_FromWideChar(_Py_normpath(final, -1), -1);
+    PyMem_Free(final);
+    return r;
 }
 
 
-static PyStatus
-calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+static PyObject *
+getpath_readlines(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    PyStatus status;
-    wchar_t exec_prefix[MAXPATHLEN+1];
-    memset(exec_prefix, 0, sizeof(exec_prefix));
-    size_t exec_prefix_len = Py_ARRAY_LENGTH(exec_prefix);
-
-    status = search_for_exec_prefix(calculate, pathconfig,
-                                    exec_prefix, exec_prefix_len,
-                                    &calculate->exec_prefix_found);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+    PyObject *r = NULL;
+    PyObject *pathobj;
+    const wchar_t *path;
+    if (!PyArg_ParseTuple(args, "U", &pathobj)) {
+        return NULL;
     }
-
-    if (!calculate->exec_prefix_found) {
-        if (calculate->warnings) {
-            fprintf(stderr,
-                "Could not find platform dependent libraries <exec_prefix>\n");
-        }
-
-        /* <platlibdir> / "lib-dynload" */
-        wchar_t *lib_dynload = joinpath2(calculate->platlibdir,
-                                         L"lib-dynload");
-        if (lib_dynload == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-
-        calculate->exec_prefix = joinpath2(calculate->exec_prefix_macro,
-                                           lib_dynload);
-        PyMem_RawFree(lib_dynload);
-
-        if (calculate->exec_prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
+    path = PyUnicode_AsWideCharString(pathobj, NULL);
+    if (!path) {
+        return NULL;
     }
-    else {
-        /* If we found EXEC_PREFIX do *not* reduce it!  (Yet.) */
-        calculate->exec_prefix = _PyMem_RawWcsdup(exec_prefix);
-        if (calculate->exec_prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
+    FILE *fp = _Py_wfopen(path, L"rb");
+    PyMem_Free((void *)path);
+    if (!fp) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
     }
-    return _PyStatus_OK();
-}
-
-
-static PyStatus
-calculate_set_exec_prefix(PyCalculatePath *calculate,
-                          _PyPathConfig *pathconfig)
-{
-    if (calculate->exec_prefix_found > 0) {
-        wchar_t *exec_prefix = _PyMem_RawWcsdup(calculate->exec_prefix);
-        if (exec_prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
 
-        reduce(exec_prefix);
-        reduce(exec_prefix);
-        reduce(exec_prefix);
-
-        if (exec_prefix[0]) {
-            pathconfig->exec_prefix = exec_prefix;
-        }
-        else {
-            /* empty string: use SEP instead */
-            PyMem_RawFree(exec_prefix);
-
-            /* The exec_prefix is the root directory, but reduce() chopped
-               off the "/". */
-            pathconfig->exec_prefix = _PyMem_RawWcsdup(separator);
-            if (pathconfig->exec_prefix == NULL) {
-                return _PyStatus_NO_MEMORY();
-            }
-        }
+    r = PyList_New(0);
+    if (!r) {
+        fclose(fp);
+        return NULL;
     }
-    else {
-        pathconfig->exec_prefix = _PyMem_RawWcsdup(calculate->exec_prefix_macro);
-        if (pathconfig->exec_prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
+    const size_t MAX_FILE = 32 * 1024;
+    char *buffer = (char *)PyMem_Malloc(MAX_FILE);
+    if (!buffer) {
+        Py_DECREF(r);
+        fclose(fp);
+        return NULL;
     }
-    return _PyStatus_OK();
-}
-
-
-/* Similar to shutil.which().
-   If found, write the path into *abs_path_p. */
-static PyStatus
-calculate_which(const wchar_t *path_env, wchar_t *program_name,
-                wchar_t **abs_path_p)
-{
-    while (1) {
-        wchar_t *delim = wcschr(path_env, DELIM);
-        wchar_t *abs_path;
-
-        if (delim) {
-            wchar_t *path = substring(path_env, delim - path_env);
-            if (path == NULL) {
-                return _PyStatus_NO_MEMORY();
-            }
-            abs_path = joinpath2(path, program_name);
-            PyMem_RawFree(path);
-        }
-        else {
-            abs_path = joinpath2(path_env, program_name);
-        }
-
-        if (abs_path == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-
-        if (isxfile(abs_path)) {
-            *abs_path_p = abs_path;
-            return _PyStatus_OK();
-        }
-        PyMem_RawFree(abs_path);
 
-        if (!delim) {
-            break;
-        }
-        path_env = delim + 1;
+    size_t cb = fread(buffer, 1, MAX_FILE, fp);
+    fclose(fp);
+    if (!cb) {
+        return r;
     }
-
-    /* not found */
-    return _PyStatus_OK();
-}
-
-
-#ifdef __APPLE__
-static PyStatus
-calculate_program_macos(wchar_t **abs_path_p)
-{
-    char execpath[MAXPATHLEN + 1];
-    uint32_t nsexeclength = Py_ARRAY_LENGTH(execpath) - 1;
-
-    /* On Mac OS X, if a script uses an interpreter of the form
-       "#!/opt/python2.3/bin/python", the kernel only passes "python"
-       as argv[0], which falls through to the $PATH search below.
-       If /opt/python2.3/bin isn't in your path, or is near the end,
-       this algorithm may incorrectly find /usr/bin/python. To work
-       around this, we can use _NSGetExecutablePath to get a better
-       hint of what the intended interpreter was, although this
-       will fail if a relative path was used. but in that case,
-       absolutize() should help us out below
-     */
-    if (_NSGetExecutablePath(execpath, &nsexeclength) != 0
-        || (wchar_t)execpath[0] != SEP)
-    {
-        /* _NSGetExecutablePath() failed or the path is relative */
-        return _PyStatus_OK();
+    if (cb >= MAX_FILE) {
+        Py_DECREF(r);
+        PyErr_SetString(PyExc_MemoryError,
+            "cannot read file larger than 32KB during initialization");
+        return NULL;
     }
+    buffer[cb] = '\0';
 
     size_t len;
-    *abs_path_p = Py_DecodeLocale(execpath, &len);
-    if (*abs_path_p == NULL) {
-        return DECODE_LOCALE_ERR("executable path", len);
-    }
-    return _PyStatus_OK();
-}
-#endif  /* __APPLE__ */
-
-
-static PyStatus
-calculate_program_impl(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
-{
-    assert(pathconfig->program_full_path == NULL);
-
-    PyStatus status;
-
-    /* If there is no slash in the argv0 path, then we have to
-     * assume python is on the user's $PATH, since there's no
-     * other way to find a directory to start the search from.  If
-     * $PATH isn't exported, you lose.
-     */
-    if (wcschr(pathconfig->program_name, SEP)) {
-        pathconfig->program_full_path = _PyMem_RawWcsdup(pathconfig->program_name);
-        if (pathconfig->program_full_path == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-        return _PyStatus_OK();
-    }
-
-#ifdef __APPLE__
-    wchar_t *abs_path = NULL;
-    status = calculate_program_macos(&abs_path);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-    if (abs_path) {
-        pathconfig->program_full_path = abs_path;
-        return _PyStatus_OK();
+    wchar_t *wbuffer = _Py_DecodeUTF8_surrogateescape(buffer, cb, &len);
+    PyMem_Free((void *)buffer);
+    if (!wbuffer) {
+        Py_DECREF(r);
+        PyErr_NoMemory();
+        return NULL;
     }
-#endif /* __APPLE__ */
 
-    if (calculate->path_env) {
-        wchar_t *abs_path = NULL;
-        status = calculate_which(calculate->path_env, pathconfig->program_name,
-                                 &abs_path);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
+    wchar_t *p1 = wbuffer;
+    wchar_t *p2 = p1;
+    while ((p2 = wcschr(p1, L'\n')) != NULL) {
+        size_t cb = p2 - p1;
+        while (cb && (p1[cb] == L'\n' || p1[cb] == L'\r')) {
+            --cb;
         }
-        if (abs_path) {
-            pathconfig->program_full_path = abs_path;
-            return _PyStatus_OK();
+        PyObject *u = PyUnicode_FromWideChar(p1, cb + 1);
+        if (!u || PyList_Append(r, u) < 0) {
+            Py_XDECREF(u);
+            Py_CLEAR(r);
+            break;
         }
+        Py_DECREF(u);
+        p1 = p2 + 1;
     }
-
-    /* In the last resort, use an empty string */
-    pathconfig->program_full_path = _PyMem_RawWcsdup(L"");
-    if (pathconfig->program_full_path == NULL) {
-        return _PyStatus_NO_MEMORY();
+    if (r && p1 && *p1) {
+        PyObject *u = PyUnicode_FromWideChar(p1, -1);
+        if (!u || PyList_Append(r, u) < 0) {
+            Py_CLEAR(r);
+        }
+        Py_XDECREF(u);
     }
-    return _PyStatus_OK();
+    PyMem_RawFree(wbuffer);
+    return r;
 }
 
 
-/* Calculate pathconfig->program_full_path */
-static PyStatus
-calculate_program(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
+static PyObject *
+getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args)
 {
-    PyStatus status;
-
-    status = calculate_program_impl(calculate, pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+    PyObject *pathobj;
+    if (!PyArg_ParseTuple(args, "U", &pathobj)) {
+        return NULL;
     }
-
-    if (pathconfig->program_full_path[0] != '\0') {
-        /* program_full_path is not empty */
-
-        /* Make sure that program_full_path is an absolute path */
-        if (!_Py_isabs(pathconfig->program_full_path)) {
-            status = absolutize(&pathconfig->program_full_path);
-            if (_PyStatus_EXCEPTION(status)) {
-                return status;
-            }
-        }
-
-#if defined(__CYGWIN__) || defined(__MINGW32__)
-        /* For these platforms it is necessary to ensure that the .exe suffix
-         * is appended to the filename, otherwise there is potential for
-         * sys.executable to return the name of a directory under the same
-         * path (bpo-28441).
-         */
-        status = add_exe_suffix(&pathconfig->program_full_path);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-#endif
+#if defined(HAVE_READLINK)
+    /* This readlink calculation only resolves a symlinked file, and
+       does not resolve any path segments. This is consistent with
+       prior releases, however, the realpath implementation below is
+       potentially correct in more cases. */
+    PyObject *r = NULL;
+    int nlink = 0;
+    wchar_t *path = PyUnicode_AsWideCharString(pathobj, NULL);
+    if (!path) {
+        goto done;
     }
-    return _PyStatus_OK();
-}
-
-
-#if HAVE_READLINK
-static PyStatus
-resolve_symlinks(wchar_t **path_p)
-{
-    wchar_t new_path[MAXPATHLEN + 1];
-    const size_t new_path_len = Py_ARRAY_LENGTH(new_path);
-    unsigned int nlink = 0;
-
-    while (1) {
-        int linklen = _Py_wreadlink(*path_p, new_path, new_path_len);
+    wchar_t *path2 = _PyMem_RawWcsdup(path);
+    PyMem_Free((void *)path);
+    path = path2;
+    while (path) {
+        wchar_t resolved[MAXPATHLEN + 1];
+        int linklen = _Py_wreadlink(path, resolved, Py_ARRAY_LENGTH(resolved));
         if (linklen == -1) {
-            /* not a symbolic link: we are done */
+            r = PyUnicode_FromWideChar(path, -1);
             break;
         }
-
-        if (_Py_isabs(new_path)) {
-            PyMem_RawFree(*path_p);
-            *path_p = _PyMem_RawWcsdup(new_path);
-            if (*path_p == NULL) {
-                return _PyStatus_NO_MEMORY();
+        if (_Py_isabs(resolved)) {
+            PyMem_RawFree((void *)path);
+            path = _PyMem_RawWcsdup(resolved);
+        } else {
+            wchar_t *s = wcsrchr(path, SEP);
+            if (s) {
+                *s = L'\0';
             }
+            path2 = _Py_normpath(_Py_join_relfile(path, resolved), -1);
+            PyMem_RawFree((void *)path);
+            path = path2;
         }
-        else {
-            /* new_path is relative to path */
-            reduce(*path_p);
-
-            wchar_t *abs_path = joinpath2(*path_p, new_path);
-            if (abs_path == NULL) {
-                return _PyStatus_NO_MEMORY();
-            }
-
-            PyMem_RawFree(*path_p);
-            *path_p = abs_path;
-        }
-
         nlink++;
         /* 40 is the Linux kernel 4.2 limit */
         if (nlink >= 40) {
-            return _PyStatus_ERR("maximum number of symbolic links reached");
+            PyErr_SetString(PyExc_OSError, "maximum number of symbolic links reached");
+            break;
         }
     }
-    return _PyStatus_OK();
-}
-#endif /* HAVE_READLINK */
-
-
-#ifdef WITH_NEXT_FRAMEWORK
-static PyStatus
-calculate_argv0_path_framework(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
-{
-    NSModule pythonModule;
-
-    /* On Mac OS X we have a special case if we're running from a framework.
-       This is because the python home should be set relative to the library,
-       which is in the framework, not relative to the executable, which may
-       be outside of the framework. Except when we're in the build
-       directory... */
-    pythonModule = NSModuleForSymbol(NSLookupAndBindSymbol("_Py_Initialize"));
-
-    /* Use dylib functions to find out where the framework was loaded from */
-    const char* modPath = NSLibraryNameForModule(pythonModule);
-    if (modPath == NULL) {
-        return _PyStatus_OK();
-    }
-
-    /* We're in a framework.
-       See if we might be in the build directory. The framework in the
-       build directory is incomplete, it only has the .dylib and a few
-       needed symlinks, it doesn't have the Lib directories and such.
-       If we're running with the framework from the build directory we must
-       be running the interpreter in the build directory, so we use the
-       build-directory-specific logic to find Lib and such. */
-    size_t len;
-    wchar_t* wbuf = Py_DecodeLocale(modPath, &len);
-    if (wbuf == NULL) {
-        return DECODE_LOCALE_ERR("framework location", len);
+    if (!path) {
+        PyErr_NoMemory();
     }
-
-    /* Path: reduce(modPath) / lib_python / LANDMARK */
-    PyStatus status;
-
-    wchar_t *parent = _PyMem_RawWcsdup(wbuf);
-    if (parent == NULL) {
-        status = _PyStatus_NO_MEMORY();
+done:
+    PyMem_RawFree((void *)path);
+    return r;
+
+#elif defined(HAVE_REALPATH)
+    PyObject *r = NULL;
+    struct stat st;
+    const char *narrow = NULL;
+    wchar_t *path = PyUnicode_AsWideCharString(pathobj, NULL);
+    if (!path) {
         goto done;
     }
-
-    reduce(parent);
-    wchar_t *lib_python = joinpath2(parent, calculate->lib_python);
-    PyMem_RawFree(parent);
-
-    if (lib_python == NULL) {
-        status = _PyStatus_NO_MEMORY();
+    narrow = Py_EncodeLocale(path, NULL);
+    if (!narrow) {
+        PyErr_NoMemory();
         goto done;
     }
-
-    int module;
-    status = ismodule(lib_python, &module);
-    PyMem_RawFree(lib_python);
-
-    if (_PyStatus_EXCEPTION(status)) {
+    if (lstat(narrow, &st)) {
+        PyErr_SetFromErrno(PyExc_OSError);
         goto done;
     }
-    if (!module) {
-        /* We are in the build directory so use the name of the
-           executable - we know that the absolute path is passed */
-        PyMem_RawFree(calculate->argv0_path);
-        calculate->argv0_path = _PyMem_RawWcsdup(pathconfig->program_full_path);
-        if (calculate->argv0_path == NULL) {
-            status = _PyStatus_NO_MEMORY();
-            goto done;
-        }
-
-        status = _PyStatus_OK();
+    if (!S_ISLNK(st.st_mode)) {
+        Py_INCREF(pathobj);
+        r = pathobj;
         goto done;
     }
-
-    /* Use the location of the library as argv0_path */
-    PyMem_RawFree(calculate->argv0_path);
-    calculate->argv0_path = wbuf;
-    return _PyStatus_OK();
-
+    wchar_t resolved[MAXPATHLEN+1];
+    if (_Py_wrealpath(path, resolved, MAXPATHLEN) == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+    } else {
+        r = PyUnicode_FromWideChar(resolved, -1);
+    }
 done:
-    PyMem_RawFree(wbuf);
-    return status;
-}
+    PyMem_Free((void *)path);
+    PyMem_Free((void *)narrow);
+    return r;
 #endif
 
+    Py_INCREF(pathobj);
+    return pathobj;
+}
 
-static PyStatus
-calculate_argv0_path(PyCalculatePath *calculate,
-                     _PyPathConfig *pathconfig)
-{
-    PyStatus status;
-
-    calculate->argv0_path = _PyMem_RawWcsdup(pathconfig->program_full_path);
-    if (calculate->argv0_path == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-
-#ifdef WITH_NEXT_FRAMEWORK
-    status = calculate_argv0_path_framework(calculate, pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-#endif
-
-    status = resolve_symlinks(&calculate->argv0_path);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
 
-    reduce(calculate->argv0_path);
+static PyMethodDef getpath_methods[] = {
+    {"abspath", getpath_abspath, METH_VARARGS, NULL},
+    {"basename", getpath_basename, METH_VARARGS, NULL},
+    {"dirname", getpath_dirname, METH_VARARGS, NULL},
+    {"hassuffix", getpath_hassuffix, METH_VARARGS, NULL},
+    {"isabs", getpath_isabs, METH_VARARGS, NULL},
+    {"isdir", getpath_isdir, METH_VARARGS, NULL},
+    {"isfile", getpath_isfile, METH_VARARGS, NULL},
+    {"isxfile", getpath_isxfile, METH_VARARGS, NULL},
+    {"joinpath", getpath_joinpath, METH_VARARGS, NULL},
+    {"readlines", getpath_readlines, METH_VARARGS, NULL},
+    {"realpath", getpath_realpath, METH_VARARGS, NULL},
+    {NULL, NULL, 0, NULL}
+};
 
-    return _PyStatus_OK();
-}
 
+/* Two implementations of warn() to use depending on whether warnings
+   are enabled or not. */
 
-static PyStatus
-calculate_open_pyenv(PyCalculatePath *calculate, FILE **env_file_p)
+static PyObject *
+getpath_warn(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    *env_file_p = NULL;
-
-    const wchar_t *env_cfg = L"pyvenv.cfg";
-
-    /* Filename: <argv0_path> / "pyvenv.cfg" */
-    wchar_t *filename = joinpath2(calculate->argv0_path, env_cfg);
-    if (filename == NULL) {
-        return _PyStatus_NO_MEMORY();
+    PyObject *msgobj;
+    if (!PyArg_ParseTuple(args, "U", &msgobj)) {
+        return NULL;
     }
+    fprintf(stderr, "%s\n", PyUnicode_AsUTF8(msgobj));
+    Py_RETURN_NONE;
+}
 
-    *env_file_p = _Py_wfopen(filename, L"r");
-    PyMem_RawFree(filename);
 
-    if (*env_file_p != NULL) {
-        return _PyStatus_OK();
+static PyObject *
+getpath_nowarn(PyObject *Py_UNUSED(self), PyObject *args)
+{
+    Py_RETURN_NONE;
+}
 
-    }
 
-    /* fopen() failed: reset errno */
-    errno = 0;
+static PyMethodDef getpath_warn_method = {"warn", getpath_warn, METH_VARARGS, NULL};
+static PyMethodDef getpath_nowarn_method = {"warn", getpath_nowarn, METH_VARARGS, NULL};
 
-    /* Path: <basename(argv0_path)> / "pyvenv.cfg" */
-    wchar_t *parent = _PyMem_RawWcsdup(calculate->argv0_path);
-    if (parent == NULL) {
-        return _PyStatus_NO_MEMORY();
+/* Add the helper functions to the dict */
+static int
+funcs_to_dict(PyObject *dict, int warnings)
+{
+    for (PyMethodDef *m = getpath_methods; m->ml_name; ++m) {
+        PyObject *f = PyCFunction_NewEx(m, NULL, NULL);
+        if (!f) {
+            return 0;
+        }
+        if (PyDict_SetItemString(dict, m->ml_name, f) < 0) {
+            Py_DECREF(f);
+            return 0;
+        }
+        Py_DECREF(f);
     }
-    reduce(parent);
-
-    filename = joinpath2(parent, env_cfg);
-    PyMem_RawFree(parent);
-    if (filename == NULL) {
-        return _PyStatus_NO_MEMORY();
+    PyMethodDef *m2 = warnings ? &getpath_warn_method : &getpath_nowarn_method;
+    PyObject *f = PyCFunction_NewEx(m2, NULL, NULL);
+    if (!f) {
+        return 0;
     }
-
-    *env_file_p = _Py_wfopen(filename, L"r");
-    PyMem_RawFree(filename);
-
-    if (*env_file_p == NULL) {
-        /* fopen() failed: reset errno */
-        errno = 0;
+    if (PyDict_SetItemString(dict, m2->ml_name, f) < 0) {
+        Py_DECREF(f);
+        return 0;
     }
-    return _PyStatus_OK();
+    Py_DECREF(f);
+    return 1;
 }
 
 
-/* Search for an "pyvenv.cfg" environment configuration file, first in the
-   executable's directory and then in the parent directory.
-   If found, open it for use when searching for prefixes.
-
-   Write the 'home' variable of pyvenv.cfg into calculate->argv0_path. */
-static PyStatus
-calculate_read_pyenv(PyCalculatePath *calculate)
+/* Add a wide-character string constant to the dict */
+static int
+wchar_to_dict(PyObject *dict, const char *key, const wchar_t *s)
 {
-    PyStatus status;
-    FILE *env_file = NULL;
-
-    status = calculate_open_pyenv(calculate, &env_file);
-    if (_PyStatus_EXCEPTION(status)) {
-        assert(env_file == NULL);
-        return status;
-    }
-    if (env_file == NULL) {
-        /* pyvenv.cfg not found */
-        return _PyStatus_OK();
-    }
-
-    /* Look for a 'home' variable and set argv0_path to it, if found */
-    wchar_t *home = NULL;
-    status = _Py_FindEnvConfigValue(env_file, L"home", &home);
-    if (_PyStatus_EXCEPTION(status)) {
-        fclose(env_file);
-        return status;
-    }
-
-    if (home) {
-        PyMem_RawFree(calculate->argv0_path);
-        calculate->argv0_path = home;
+    PyObject *u;
+    int r;
+    if (s && s[0]) {
+        u = PyUnicode_FromWideChar(s, -1);
+        if (!u) {
+            return 0;
+        }
+    } else {
+        u = Py_None;
+        Py_INCREF(u);
     }
-    fclose(env_file);
-    return _PyStatus_OK();
+    r = PyDict_SetItemString(dict, key, u) == 0;
+    Py_DECREF(u);
+    return r;
 }
 
 
-static PyStatus
-calculate_zip_path(PyCalculatePath *calculate)
+/* Add a narrow string constant to the dict, using default locale decoding */
+static int
+decode_to_dict(PyObject *dict, const char *key, const char *s)
 {
-    PyStatus res;
-
-    /* Path: <platlibdir> / "pythonXY.zip" */
-    wchar_t *path = joinpath2(calculate->platlibdir,
-                              L"python" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION)
-                              L".zip");
-    if (path == NULL) {
-        return _PyStatus_NO_MEMORY();
+    PyObject *u = NULL;
+    int r;
+    if (s && s[0]) {
+        size_t len;
+        const wchar_t *w = Py_DecodeLocale(s, &len);
+        if (w) {
+            u = PyUnicode_FromWideChar(w, len);
+            PyMem_RawFree((void *)w);
+        }
+        if (!u) {
+            return 0;
+        }
+    } else {
+        u = Py_None;
+        Py_INCREF(u);
     }
+    r = PyDict_SetItemString(dict, key, u) == 0;
+    Py_DECREF(u);
+    return r;
+}
 
-    if (calculate->prefix_found > 0) {
-        /* Use the reduced prefix returned by Py_GetPrefix()
-
-           Path: <basename(basename(prefix))> / <platlibdir> / "pythonXY.zip" */
-        wchar_t *parent = _PyMem_RawWcsdup(calculate->prefix);
-        if (parent == NULL) {
-            res = _PyStatus_NO_MEMORY();
-            goto done;
+/* Add an environment variable to the dict, optionally clearing it afterwards */
+static int
+env_to_dict(PyObject *dict, const char *key, int and_clear)
+{
+    PyObject *u = NULL;
+    int r = 0;
+    assert(strncmp(key, "ENV_", 4) == 0);
+    assert(strlen(key) < 64);
+#ifdef MS_WINDOWS
+    wchar_t wkey[64];
+    // Quick convert to wchar_t, since we know key is ASCII
+    wchar_t *wp = wkey;
+    for (const char *p = &key[4]; *p; ++p) {
+        assert(*p < 128);
+        *wp++ = *p;
+    }
+    *wp = L'\0';
+    const wchar_t *v = _wgetenv(wkey);
+    if (v) {
+        u = PyUnicode_FromWideChar(v, -1);
+        if (!u) {
+            PyErr_Clear();
+        }
+    }
+#else
+    const char *v = getenv(&key[4]);
+    if (v) {
+        size_t len;
+        const wchar_t *w = Py_DecodeLocale(v, &len);
+        if (w) {
+            u = PyUnicode_FromWideChar(w, len);
+            if (!u) {
+                PyErr_Clear();
+            }
+            PyMem_RawFree((void *)w);
         }
-        reduce(parent);
-        reduce(parent);
-        calculate->zip_path = joinpath2(parent, path);
-        PyMem_RawFree(parent);
     }
-    else {
-        calculate->zip_path = joinpath2(calculate->prefix_macro, path);
-    }
-
-    if (calculate->zip_path == NULL) {
-        res = _PyStatus_NO_MEMORY();
-        goto done;
+#endif
+    if (u) {
+        r = PyDict_SetItemString(dict, key, u) == 0;
+        Py_DECREF(u);
+    } else {
+        r = PyDict_SetItemString(dict, key, Py_None) == 0;
+    }
+    if (r && and_clear) {
+#ifdef MS_WINDOWS
+        _wputenv_s(wkey, L"");
+#else
+        unsetenv(&key[4]);
+#endif
     }
-
-    res = _PyStatus_OK();
-
-done:
-    PyMem_RawFree(path);
-    return res;
+    return r;
 }
 
 
-static PyStatus
-calculate_module_search_path(PyCalculatePath *calculate,
-                             _PyPathConfig *pathconfig)
+/* Add an integer constant to the dict */
+static int
+int_to_dict(PyObject *dict, const char *key, int v)
 {
-    /* Calculate size of return buffer */
-    size_t bufsz = 0;
-    if (calculate->pythonpath_env != NULL) {
-        bufsz += wcslen(calculate->pythonpath_env) + 1;
+    PyObject *o;
+    int r;
+    o = PyLong_FromLong(v);
+    if (!o) {
+        return 0;
     }
+    r = PyDict_SetItemString(dict, key, o) == 0;
+    Py_DECREF(o);
+    return r;
+}
 
-    wchar_t *defpath = calculate->pythonpath_macro;
-    size_t prefixsz = wcslen(calculate->prefix) + 1;
-    while (1) {
-        wchar_t *delim = wcschr(defpath, DELIM);
-
-        if (!_Py_isabs(defpath)) {
-            /* Paths are relative to prefix */
-            bufsz += prefixsz;
-        }
 
-        if (delim) {
-            bufsz += delim - defpath + 1;
-        }
-        else {
-            bufsz += wcslen(defpath) + 1;
-            break;
+#ifdef MS_WINDOWS
+static int
+winmodule_to_dict(PyObject *dict, const char *key, HMODULE mod)
+{
+    wchar_t *buffer = NULL;
+    for (DWORD cch = 256; buffer == NULL && cch < (1024 * 1024); cch *= 2) {
+        buffer = (wchar_t*)PyMem_RawMalloc(cch * sizeof(wchar_t));
+        if (buffer) {
+            if (GetModuleFileNameW(mod, buffer, cch) == cch) {
+                PyMem_RawFree(buffer);
+                buffer = NULL;
+            }
         }
-        defpath = delim + 1;
-    }
-
-    bufsz += wcslen(calculate->zip_path) + 1;
-    bufsz += wcslen(calculate->exec_prefix) + 1;
-
-    /* Allocate the buffer */
-    wchar_t *buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t));
-    if (buf == NULL) {
-        return _PyStatus_NO_MEMORY();
     }
-    buf[0] = '\0';
-
-    /* Run-time value of $PYTHONPATH goes first */
-    if (calculate->pythonpath_env) {
-        wcscpy(buf, calculate->pythonpath_env);
-        wcscat(buf, delimiter);
-    }
-
-    /* Next is the default zip path */
-    wcscat(buf, calculate->zip_path);
-    wcscat(buf, delimiter);
-
-    /* Next goes merge of compile-time $PYTHONPATH with
-     * dynamically located prefix.
-     */
-    defpath = calculate->pythonpath_macro;
-    while (1) {
-        wchar_t *delim = wcschr(defpath, DELIM);
+    int r = wchar_to_dict(dict, key, buffer);
+    PyMem_RawFree(buffer);
+    return r;
+}
+#endif
 
-        if (!_Py_isabs(defpath)) {
-            wcscat(buf, calculate->prefix);
-            if (prefixsz >= 2 && calculate->prefix[prefixsz - 2] != SEP &&
-                defpath[0] != (delim ? DELIM : L'\0'))
-            {
-                /* not empty */
-                wcscat(buf, separator);
-            }
-        }
 
-        if (delim) {
-            size_t len = delim - defpath + 1;
-            size_t end = wcslen(buf) + len;
-            wcsncat(buf, defpath, len);
-            buf[end] = '\0';
+/* Add the current executable's path to the dict */
+static int
+progname_to_dict(PyObject *dict, const char *key)
+{
+#ifdef MS_WINDOWS
+    return winmodule_to_dict(dict, key, NULL);
+#elif defined(__APPLE__)
+    char *path;
+    uint32_t pathLen = 256;
+    while (pathLen) {
+        path = PyMem_RawMalloc((pathLen + 1) * sizeof(char));
+        if (!path) {
+            return 0;
+        }
+        if (_NSGetExecutablePath(path, &pathLen) != 0) {
+            PyMem_RawFree(path);
+            continue;
         }
-        else {
-            wcscat(buf, defpath);
-            break;
+        // Only keep if the path is absolute
+        if (path[0] == SEP) {
+            int r = decode_to_dict(dict, key, path);
+            PyMem_RawFree(path);
+            return r;
         }
-        defpath = delim + 1;
+        // Fall back and store None
+        PyMem_RawFree(path);
+        break;
     }
-    wcscat(buf, delimiter);
-
-    /* Finally, on goes the directory for dynamic-load modules */
-    wcscat(buf, calculate->exec_prefix);
-
-    pathconfig->module_search_path = buf;
-    return _PyStatus_OK();
+#endif
+    return PyDict_SetItemString(dict, key, Py_None) == 0;
 }
 
 
-static PyStatus
-calculate_init(PyCalculatePath *calculate, const PyConfig *config)
+/* Add the runtime library's path to the dict */
+static int
+library_to_dict(PyObject *dict, const char *key)
 {
-    size_t len;
-
-    calculate->warnings = config->pathconfig_warnings;
-    calculate->pythonpath_env = config->pythonpath_env;
-    calculate->platlibdir = config->platlibdir;
-
-    const char *path = getenv("PATH");
-    if (path) {
-        calculate->path_env = Py_DecodeLocale(path, &len);
-        if (!calculate->path_env) {
-            return DECODE_LOCALE_ERR("PATH environment variable", len);
-        }
+#ifdef MS_WINDOWS
+    extern HMODULE PyWin_DLLhModule;
+    if (PyWin_DLLhModule) {
+        return winmodule_to_dict(dict, key, PyWin_DLLhModule);
     }
+#elif defined(WITH_NEXT_FRAMEWORK)
+    static const char modPath[MAXPATHLEN + 1];
+    static int modPathInitialized = -1;
+    if (modPathInitialized < 0) {
+        NSModule pythonModule;
+        modPathInitialized = 0;
 
-    /* Decode macros */
-    calculate->pythonpath_macro = Py_DecodeLocale(PYTHONPATH, &len);
-    if (!calculate->pythonpath_macro) {
-        return DECODE_LOCALE_ERR("PYTHONPATH macro", len);
-    }
-    calculate->prefix_macro = Py_DecodeLocale(PREFIX, &len);
-    if (!calculate->prefix_macro) {
-        return DECODE_LOCALE_ERR("PREFIX macro", len);
-    }
-    calculate->exec_prefix_macro = Py_DecodeLocale(EXEC_PREFIX, &len);
-    if (!calculate->exec_prefix_macro) {
-        return DECODE_LOCALE_ERR("EXEC_PREFIX macro", len);
-    }
-    calculate->vpath_macro = Py_DecodeLocale(VPATH, &len);
-    if (!calculate->vpath_macro) {
-        return DECODE_LOCALE_ERR("VPATH macro", len);
-    }
+        /* On Mac OS X we have a special case if we're running from a framework.
+           This is because the python home should be set relative to the library,
+           which is in the framework, not relative to the executable, which may
+           be outside of the framework. Except when we're in the build
+           directory... */
+        pythonModule = NSModuleForSymbol(NSLookupAndBindSymbol("_Py_Initialize"));
 
-    // <platlibdir> / "pythonX.Y"
-    wchar_t *pyversion = Py_DecodeLocale("python" VERSION, &len);
-    if (!pyversion) {
-        return DECODE_LOCALE_ERR("VERSION macro", len);
+        /* Use dylib functions to find out where the framework was loaded from */
+        const char *path = NSLibraryNameForModule(pythonModule);
+        if (path) {
+            strncpy(modPath, path, MAXPATHLEN);
+        }
     }
-    calculate->lib_python = joinpath2(config->platlibdir, pyversion);
-    PyMem_RawFree(pyversion);
-    if (calculate->lib_python == NULL) {
-        return _PyStatus_NO_MEMORY();
+    if (modPathInitialized > 0) {
+        return decode_to_dict(dict, key, modPath);
     }
-
-    return _PyStatus_OK();
+#endif
+    return PyDict_SetItemString(dict, key, Py_None) == 0;
 }
 
 
-static void
-calculate_free(PyCalculatePath *calculate)
+PyObject *
+_Py_Get_Getpath_CodeObject()
 {
-    PyMem_RawFree(calculate->pythonpath_macro);
-    PyMem_RawFree(calculate->prefix_macro);
-    PyMem_RawFree(calculate->exec_prefix_macro);
-    PyMem_RawFree(calculate->vpath_macro);
-    PyMem_RawFree(calculate->lib_python);
-    PyMem_RawFree(calculate->path_env);
-    PyMem_RawFree(calculate->zip_path);
-    PyMem_RawFree(calculate->argv0_path);
-    PyMem_RawFree(calculate->prefix);
-    PyMem_RawFree(calculate->exec_prefix);
+    return PyMarshal_ReadObjectFromString(
+        (const char*)_Py_M__getpath, sizeof(_Py_M__getpath));
 }
 
 
-static PyStatus
-calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
-{
-    PyStatus status;
-
-    if (pathconfig->program_full_path == NULL) {
-        status = calculate_program(calculate, pathconfig);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
+/* Perform the actual path calculation.
 
-    status = calculate_argv0_path(calculate, pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
+   When compute_path_config is 0, this only reads any initialised path
+   config values into the PyConfig struct. For example, Py_SetHome() or
+   Py_SetPath(). The only error should be due to failed memory allocation.
 
-    /* If a pyvenv.cfg configure file is found,
-       argv0_path is overridden with its 'home' variable. */
-    status = calculate_read_pyenv(calculate);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
+   When compute_path_config is 1, full path calculation is performed.
+   The GIL must be held, and there may be filesystem access, side
+   effects, and potential unraisable errors that are reported directly
+   to stderr.
 
-    status = calculate_prefix(calculate, pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
+   Calling this function multiple times on the same PyConfig is only
+   safe because already-configured values are not recalculated. To
+   actually recalculate paths, you need a clean PyConfig.
+*/
+PyStatus
+_PyConfig_InitPathConfig(PyConfig *config, int compute_path_config)
+{
+    PyStatus status = _PyPathConfig_ReadGlobal(config);
 
-    status = calculate_zip_path(calculate);
-    if (_PyStatus_EXCEPTION(status)) {
+    if (_PyStatus_EXCEPTION(status) || !compute_path_config) {
         return status;
     }
 
-    status = calculate_exec_prefix(calculate, pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
+    if (!_PyThreadState_UncheckedGet()) {
+        return PyStatus_Error("cannot calculate path configuration without GIL");
     }
 
-    if ((!calculate->prefix_found || !calculate->exec_prefix_found)
-        && calculate->warnings)
-    {
-        fprintf(stderr,
-                "Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]\n");
+    PyObject *configDict = _PyConfig_AsDict(config);
+    if (!configDict) {
+        PyErr_Clear();
+        return PyStatus_NoMemory();
     }
 
-    if (pathconfig->module_search_path == NULL) {
-        status = calculate_module_search_path(calculate, pathconfig);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
+    PyObject *dict = PyDict_New();
+    if (!dict) {
+        PyErr_Clear();
+        Py_DECREF(configDict);
+        return PyStatus_NoMemory();
     }
 
-    if (pathconfig->stdlib_dir == NULL) {
-        /* This must be done *before* calculate_set_prefix() is called. */
-        status = calculate_set_stdlib_dir(calculate, pathconfig);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
+    if (PyDict_SetItemString(dict, "config", configDict) < 0) {
+        PyErr_Clear();
+        Py_DECREF(configDict);
+        Py_DECREF(dict);
+        return PyStatus_NoMemory();
     }
+    /* reference now held by dict */
+    Py_DECREF(configDict);
 
-    if (pathconfig->prefix == NULL) {
-        status = calculate_set_prefix(calculate, pathconfig);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
+    PyObject *co = _Py_Get_Getpath_CodeObject();
+    if (!co || !PyCode_Check(co)) {
+        PyErr_Clear();
+        Py_XDECREF(co);
+        Py_DECREF(dict);
+        return PyStatus_Error("error reading frozen getpath.py");
     }
 
-    if (pathconfig->exec_prefix == NULL) {
-        status = calculate_set_exec_prefix(calculate, pathconfig);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
+#ifdef MS_WINDOWS
+    PyObject *winreg = PyImport_ImportModule("winreg");
+    if (!winreg || PyDict_SetItemString(dict, "winreg", winreg) < 0) {
+        PyErr_Clear();
+        Py_XDECREF(winreg);
+        if (PyDict_SetItemString(dict, "winreg", Py_None) < 0) {
+            PyErr_Clear();
+            Py_DECREF(co);
+            Py_DECREF(dict);
+            return PyStatus_Error("error importing winreg module");
         }
+    } else {
+        Py_DECREF(winreg);
     }
-    return _PyStatus_OK();
-}
-
-
-/* Calculate the Python path configuration.
-
-   Inputs:
-
-   - PATH environment variable
-   - Macros: PYTHONPATH, PREFIX, EXEC_PREFIX, VERSION (ex: "3.9").
-     PREFIX and EXEC_PREFIX are generated by the configure script.
-     PYTHONPATH macro is the default search path.
-   - pybuilddir.txt file
-   - pyvenv.cfg configuration file
-   - PyConfig fields ('config' function argument):
-
-     - pathconfig_warnings
-     - pythonpath_env (PYTHONPATH environment variable)
-
-   - _PyPathConfig fields ('pathconfig' function argument):
-
-     - program_name: see config_init_program_name()
-     - home: Py_SetPythonHome() or PYTHONHOME environment variable
-
-   - current working directory: see copy_absolute()
-
-   Outputs, 'pathconfig' fields:
-
-   - program_full_path
-   - module_search_path
-   - prefix
-   - exec_prefix
-
-   If a field is already set (non NULL), it is left unchanged. */
-PyStatus
-_PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
-{
-    PyStatus status;
-    PyCalculatePath calculate;
-    memset(&calculate, 0, sizeof(calculate));
+#endif
 
-    status = calculate_init(&calculate, config);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto done;
-    }
+    if (
+#ifdef MS_WINDOWS
+        !decode_to_dict(dict, "os_name", "nt") ||
+#elif defined(__APPLE__)
+        !decode_to_dict(dict, "os_name", "darwin") ||
+#else
+        !decode_to_dict(dict, "os_name", "posix") ||
+#endif
+        !decode_to_dict(dict, "PREFIX", PREFIX) ||
+        !decode_to_dict(dict, "EXEC_PREFIX", EXEC_PREFIX) ||
+        !decode_to_dict(dict, "PYTHONPATH", PYTHONPATH) ||
+        !decode_to_dict(dict, "VPATH", VPATH) ||
+        !decode_to_dict(dict, "PLATLIBDIR", PLATLIBDIR) ||
+        !decode_to_dict(dict, "PYDEBUGEXT", PYDEBUGEXT) ||
+        !int_to_dict(dict, "VERSION_MAJOR", PY_MAJOR_VERSION) ||
+        !int_to_dict(dict, "VERSION_MINOR", PY_MINOR_VERSION) ||
+        !decode_to_dict(dict, "PYWINVER", PYWINVER) ||
+        !wchar_to_dict(dict, "EXE_SUFFIX", EXE_SUFFIX) ||
+        !env_to_dict(dict, "ENV_PATH", 0) ||
+        !env_to_dict(dict, "ENV_PYTHONHOME", 0) ||
+        !env_to_dict(dict, "ENV_PYTHONEXECUTABLE", 0) ||
+        !env_to_dict(dict, "ENV___PYVENV_LAUNCHER__", 1) ||
+        !progname_to_dict(dict, "real_executable") ||
+        !library_to_dict(dict, "library") ||
+        !wchar_to_dict(dict, "executable_dir", NULL) ||
+        !wchar_to_dict(dict, "py_setpath", _PyPathConfig_GetGlobalModuleSearchPath()) ||
+        !funcs_to_dict(dict, config->pathconfig_warnings) ||
+#ifndef MS_WINDOWS
+        PyDict_SetItemString(dict, "winreg", Py_None) < 0 ||
+#endif
+        PyDict_SetItemString(dict, "__builtins__", PyEval_GetBuiltins()) < 0
+    ) {
+        Py_DECREF(co);
+        Py_DECREF(dict);
+        _PyErr_WriteUnraisableMsg("error evaluating initial values", NULL);
+        return PyStatus_Error("error evaluating initial values");
+    }
+
+    PyObject *r = PyEval_EvalCode(co, dict, dict);
+    Py_DECREF(co);
+
+    if (!r) {
+        Py_DECREF(dict);
+        _PyErr_WriteUnraisableMsg("error evaluating path", NULL);
+        return PyStatus_Error("error evaluating path");
+    }
+    Py_DECREF(r);
+
+#if 0
+    PyObject *it = PyObject_GetIter(configDict);
+    for (PyObject *k = PyIter_Next(it); k; k = PyIter_Next(it)) {
+        if (!strcmp("__builtins__", PyUnicode_AsUTF8(k))) {
+            Py_DECREF(k);
+            continue;
+        }
+        fprintf(stderr, "%s = ", PyUnicode_AsUTF8(k));
+        PyObject *o = PyDict_GetItem(configDict, k);
+        o = PyObject_Repr(o);
+        fprintf(stderr, "%s\n", PyUnicode_AsUTF8(o));
+        Py_DECREF(o);
+        Py_DECREF(k);
+    }
+    Py_DECREF(it);
+#endif
 
-    status = calculate_path(&calculate, pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto done;
+    if (_PyConfig_FromDict(config, configDict) < 0) {
+        _PyErr_WriteUnraisableMsg("reading getpath results", NULL);
+        Py_DECREF(dict);
+        return PyStatus_Error("error getting getpath results");
     }
 
-    /* program_full_path must an either an empty string or an absolute path */
-    assert(wcslen(pathconfig->program_full_path) == 0
-           || _Py_isabs(pathconfig->program_full_path));
-
-    status = _PyStatus_OK();
+    Py_DECREF(dict);
 
-done:
-    calculate_free(&calculate);
-    return status;
-}
-
-#ifdef __cplusplus
+    return _PyStatus_OK();
 }
-#endif
diff --git a/Modules/getpath.py b/Modules/getpath.py
new file mode 100644
index 0000000000000..2eadfba1dfda2
--- /dev/null
+++ b/Modules/getpath.py
@@ -0,0 +1,727 @@
+# ******************************************************************************
+# getpath.py
+# ******************************************************************************
+
+# This script is designed to be precompiled to bytecode, frozen into the
+# main binary, and then directly evaluated. It is not an importable module,
+# and does not import any other modules (besides winreg on Windows).
+# Rather, the values listed below must be specified in the globals dict
+# used when evaluating the bytecode.
+
+# See _PyConfig_InitPathConfig in Modules/getpath.c for the execution.
+
+# ******************************************************************************
+# REQUIRED GLOBALS
+# ******************************************************************************
+
+# ** Helper functions **
+# abspath(path)     -- make relative paths absolute against CWD
+# basename(path)    -- the filename of path
+# dirname(path)     -- the directory name of path
+# hassuffix(path, suffix) -- returns True if path has suffix
+# isabs(path)       -- path is absolute or not
+# isdir(path)       -- path exists and is a directory
+# isfile(path)      -- path exists and is a file
+# isxfile(path)     -- path exists and is an executable file
+# joinpath(*paths)  -- combine the paths
+# readlines(path)   -- a list of each line of text in the UTF-8 encoded file
+# realpath(path)    -- resolves symlinks in path
+# warn(message)     -- print a warning (if enabled)
+
+# ** Values known at compile time **
+# os_name           -- [in] one of 'nt', 'posix', 'darwin'
+# PREFIX            -- [in] sysconfig.get_config_var(...)
+# EXEC_PREFIX       -- [in] sysconfig.get_config_var(...)
+# PYTHONPATH        -- [in] sysconfig.get_config_var(...)
+# VPATH             -- [in] sysconfig.get_config_var(...)
+# PLATLIBDIR        -- [in] sysconfig.get_config_var(...)
+# PYDEBUGEXT        -- [in, opt] '_d' on Windows for debug builds
+# EXE_SUFFIX        -- [in, opt] '.exe' on Windows/Cygwin/similar
+# VERSION_MAJOR     -- [in] sys.version_info.major
+# VERSION_MINOR     -- [in] sys.version_info.minor
+# PYWINVER          -- [in] the Windows platform-specific version (e.g. 3.8-32)
+
+# ** Values read from the environment **
+#   There is no need to check the use_environment flag before reading
+#   these, as the flag will be tested in this script.
+#   Also note that ENV_PYTHONPATH is read from config['pythonpath_env']
+#   to allow for embedders who choose to specify it via that struct.
+# ENV_PATH                -- [in] getenv(...)
+# ENV_PYTHONHOME          -- [in] getenv(...)
+# ENV_PYTHONEXECUTABLE    -- [in] getenv(...)
+# ENV___PYVENV_LAUNCHER__ -- [in] getenv(...)
+
+# ** Values calculated at runtime **
+# config            -- [in/out] dict of the PyConfig structure
+# real_executable   -- [in, optional] resolved path to main process
+#   On Windows and macOS, read directly from the running process
+#   Otherwise, leave None and it will be calculated from executable
+# executable_dir    -- [in, optional] real directory containing binary
+#   If None, will be calculated from real_executable or executable
+# py_setpath        -- [in] argument provided to Py_SetPath
+#   If None, 'prefix' and 'exec_prefix' may be updated in config
+# library           -- [in, optional] path of dylib/DLL/so
+#   Only used for locating ._pth files
+# winreg            -- [in, optional] the winreg module (only on Windows)
+
+# ******************************************************************************
+# HIGH-LEVEL ALGORITHM
+# ******************************************************************************
+
+# IMPORTANT: The code is the actual specification at time of writing.
+# This prose description is based on the original comment from the old
+# getpath.c to help capture the intent, but should not be considered
+# a specification.
+
+# Search in some common locations for the associated Python libraries.
+
+# Two directories must be found, the platform independent directory
+# (prefix), containing the common .py and .pyc files, and the platform
+# dependent directory (exec_prefix), containing the shared library
+# modules.  Note that prefix and exec_prefix can be the same directory,
+# but for some installations, they are different.
+
+# This script carries out separate searches for prefix and exec_prefix.
+# Each search tries a number of different locations until a ``landmark''
+# file or directory is found.  If no prefix or exec_prefix is found, a
+# warning message is issued and the preprocessor defined PREFIX and
+# EXEC_PREFIX are used (even though they will not work); python carries on
+# as best as is possible, but most imports will fail.
+
+# Before any searches are done, the location of the executable is
+# determined.  If Py_SetPath() was called, or if we are running on
+# Windows, the 'real_executable' path is used (if known).  Otherwise,
+# we use the config-specified program name or default to argv[0].
+# If this has one or more slashes in it, it is made absolute against
+# the current working directory.  If it only contains a name, it must
+# have been invoked from the shell's path, so we search $PATH for the
+# named executable and use that.  If the executable was not found on
+# $PATH (or there was no $PATH environment variable), the original
+# argv[0] string is used.
+
+# At this point, provided Py_SetPath was not used, the
+# __PYVENV_LAUNCHER__ variable may override the executable (on macOS,
+# the PYTHON_EXECUTABLE variable may also override). This allows
+# certain launchers that run Python as a subprocess to properly
+# specify the executable path. They are not intended for users.
+
+# Next, the executable location is examined to see if it is a symbolic
+# link.  If so, the link is realpath-ed and the directory of the link
+# target is used for the remaining searches.  The same steps are
+# performed for prefix and for exec_prefix, but with different landmarks.
+
+# Step 1. Are we running in a virtual environment? Unless 'home' has
+# been specified another way, check for a pyvenv.cfg and use its 'home'
+# property to override the executable dir used later for prefix searches.
+# We do not activate the venv here - that is performed later by site.py.
+
+# Step 2. Is there a ._pth file? A ._pth file lives adjacent to the
+# runtime library (if any) or the actual executable (not the symlink),
+# and contains precisely the intended contents of sys.path as relative
+# paths (to its own location). Its presence also enables isolated mode
+# and suppresses other environment variable usage. Unless already
+# specified by Py_SetHome(), the directory containing the ._pth file is
+# set as 'home'.
+
+# Step 3. Are we running python out of the build directory?  This is
+# checked by looking for the BUILDDIR_TXT file, which contains the
+# relative path to the platlib dir. The executable_dir value is
+# derived from joining the VPATH preprocessor variable to the
+# directory containing pybuilddir.txt. If it is not found, the 
+# BUILD_LANDMARK file is found, which is part of the source tree.
+# prefix is then found by searching up for a file that should only
+# exist in the source tree, and the stdlib dir is set to prefix/Lib.
+
+# Step 4. If 'home' is set, either by Py_SetHome(), ENV_PYTHONHOME,
+# a pyvenv.cfg file, ._pth file, or by detecting a build directory, it
+# is assumed to point to prefix and exec_prefix. $PYTHONHOME can be a
+# single directory, which is used for both, or the prefix and exec_prefix
+# directories separated by DELIM (colon on POSIX; semicolon on Windows).
+
+# Step 5. Try to find prefix and exec_prefix relative to executable_dir,
+# backtracking up the path until it is exhausted.  This is the most common
+# step to succeed.  Note that if prefix and exec_prefix are different,
+# exec_prefix is more likely to be found; however if exec_prefix is a
+# subdirectory of prefix, both will be found.
+
+# Step 6. Search the directories pointed to by the preprocessor variables
+# PREFIX and EXEC_PREFIX.  These are supplied by the Makefile but can be
+# passed in as options to the configure script.
+
+# That's it!
+
+# Well, almost.  Once we have determined prefix and exec_prefix, the
+# preprocessor variable PYTHONPATH is used to construct a path.  Each
+# relative path on PYTHONPATH is prefixed with prefix.  Then the directory
+# containing the shared library modules is appended.  The environment
+# variable $PYTHONPATH is inserted in front of it all. On POSIX, if we are
+# in a build directory, both prefix and exec_prefix are reset to the
+# corresponding preprocessor variables (so sys.prefix will reflect the
+# installation location, even though sys.path points into the build
+# directory).  This seems to make more sense given that currently the only
+# known use of sys.prefix and sys.exec_prefix is for the ILU installation
+# process to find the installed Python tree.
+
+# An embedding application can use Py_SetPath() to override all of
+# these automatic path computations.
+
+
+# ******************************************************************************
+# PLATFORM CONSTANTS
+# ******************************************************************************
+
+platlibdir = config.get('platlibdir') or PLATLIBDIR
+
+if os_name == 'posix' or os_name == 'darwin':
+    BUILDDIR_TXT = 'pybuilddir.txt'
+    BUILD_LANDMARK = 'Modules/Setup.local'
+    DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}'
+    STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}'
+    STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc']
+    PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload'
+    BUILDSTDLIB_LANDMARKS = ['Lib/os.py']
+    VENV_LANDMARK = 'pyvenv.cfg'
+    ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip'
+    DELIM = ':'
+    SEP = '/'
+
+elif os_name == 'nt':
+    BUILDDIR_TXT = 'pybuilddir.txt'
+    BUILD_LANDMARK = r'..\..\Modules\Setup.local'
+    DEFAULT_PROGRAM_NAME = f'python'
+    STDLIB_SUBDIR = 'Lib'
+    STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}\\os.py', f'{STDLIB_SUBDIR}\\os.pyc']
+    PLATSTDLIB_LANDMARK = f'{platlibdir}'
+    BUILDSTDLIB_LANDMARKS = ['Lib\\os.py']
+    VENV_LANDMARK = 'pyvenv.cfg'
+    ZIP_LANDMARK = f'python{VERSION_MAJOR}{VERSION_MINOR}{PYDEBUGEXT or ""}.zip'
+    WINREG_KEY = f'SOFTWARE\\Python\\PythonCore\\{PYWINVER}\\PythonPath'
+    DELIM = ';'
+    SEP = '\\'
+
+
+# ******************************************************************************
+# HELPER FUNCTIONS (note that we prefer C functions for performance)
+# ******************************************************************************
+
+def search_up(prefix, *landmarks, test=isfile):
+    while prefix:
+        if any(test(joinpath(prefix, f)) for f in landmarks):
+            return prefix
+        prefix = dirname(prefix)
+
+
+# ******************************************************************************
+# READ VARIABLES FROM config
+# ******************************************************************************
+
+program_name = config.get('program_name')
+home = config.get('home')
+executable = config.get('executable')
+base_executable = config.get('base_executable')
+prefix = config.get('prefix')
+exec_prefix = config.get('exec_prefix')
+base_prefix = config.get('base_prefix')
+base_exec_prefix = config.get('base_exec_prefix')
+ENV_PYTHONPATH = config['pythonpath_env']
+use_environment = config.get('use_environment', 1)
+
+pythonpath = config.get('module_search_paths')
+
+real_executable_dir = None
+stdlib_dir = None
+platstdlib_dir = None
+
+# ******************************************************************************
+# CALCULATE program_name
+# ******************************************************************************
+
+program_name_was_set = bool(program_name)
+
+if not program_name:
+    try:
+        program_name = config.get('orig_argv', [])[0]
+    except IndexError:
+        pass
+
+if not program_name:
+    program_name = DEFAULT_PROGRAM_NAME
+
+if EXE_SUFFIX and not hassuffix(program_name, EXE_SUFFIX) and isxfile(program_name + EXE_SUFFIX):
+    program_name = program_name + EXE_SUFFIX
+
+
+# ******************************************************************************
+# CALCULATE executable
+# ******************************************************************************
+
+if py_setpath:
+    # When Py_SetPath has been called, executable defaults to
+    # the real executable path.
+    if not executable:
+        executable = real_executable
+
+if not executable and SEP in program_name:
+    # Resolve partial path program_name against current directory
+    executable = abspath(program_name)
+
+if not executable:
+    # All platforms default to real_executable if known at this
+    # stage. POSIX does not set this value.
+    executable = real_executable
+elif os_name == 'darwin':
+    # QUIRK: On macOS we may know the real executable path, but
+    # if our caller has lied to us about it (e.g. most of
+    # test_embed), we need to use their path in order to detect
+    # whether we are in a build tree. This is true even if the
+    # executable path was provided in the config.
+    real_executable = executable
+
+if not executable and program_name:
+    # Resolve names against PATH.
+    # NOTE: The use_environment value is ignored for this lookup.
+    # To properly isolate, launch Python with a full path.
+    for p in ENV_PATH.split(DELIM):
+        p = joinpath(p, program_name)
+        if isxfile(p):
+            executable = p
+            break
+
+if not executable:
+    executable = ''
+    # When we cannot calculate the executable, subsequent searches
+    # look in the current working directory. Here, we emulate that
+    # (the former getpath.c would do it apparently by accident).
+    executable_dir = abspath('.')
+    # Also need to set this fallback in case we are running from a
+    # build directory with an invalid argv0 (i.e. test_sys.test_executable)
+    real_executable_dir = executable_dir
+
+if ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__:
+    # If set, these variables imply that we should be using them as
+    # sys.executable and when searching for venvs. However, we should
+    # use the argv0 path for prefix calculation
+    base_executable = executable
+    if not real_executable:
+        real_executable = executable
+    executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__
+    executable_dir = dirname(executable)
+
+
+# ******************************************************************************
+# CALCULATE (default) home
+# ******************************************************************************
+
+# Used later to distinguish between Py_SetPythonHome and other
+# ways that it may have been set
+home_was_set = False
+
+if home:
+    home_was_set = True
+elif use_environment and ENV_PYTHONHOME and not py_setpath:
+    home = ENV_PYTHONHOME
+
+
+# ******************************************************************************
+# READ pyvenv.cfg
+# ******************************************************************************
+
+venv_prefix = None
+
+# Calling Py_SetPythonHome(), Py_SetPath() or
+# setting $PYTHONHOME will override venv detection.
+if not home and not py_setpath:
+    try:
+        # prefix2 is just to avoid calculating dirname again later,
+        # as the path in venv_prefix is the more common case.
+        venv_prefix2 = executable_dir or dirname(executable)
+        venv_prefix = dirname(venv_prefix2)
+        try:
+            # Read pyvenv.cfg from one level above executable
+            pyvenvcfg = readlines(joinpath(venv_prefix, VENV_LANDMARK))
+        except FileNotFoundError:
+            # Try the same directory as executable
+            pyvenvcfg = readlines(joinpath(venv_prefix2, VENV_LANDMARK))
+            venv_prefix = venv_prefix2
+    except FileNotFoundError:
+        venv_prefix = None
+        pyvenvcfg = []
+
+    for line in pyvenvcfg:
+        key, had_equ, value = line.partition('=')
+        if had_equ and key.strip().lower() == 'home':
+            executable_dir = real_executable_dir = value.strip()
+            base_executable = joinpath(executable_dir, basename(executable))
+            break
+    else:
+        venv_prefix = None
+
+
+# ******************************************************************************
+# CALCULATE base_executable, real_executable AND executable_dir
+# ******************************************************************************
+
+if not base_executable:
+    base_executable = executable or real_executable or ''
+
+if not real_executable:
+    real_executable = base_executable
+
+try:
+    real_executable = realpath(real_executable)
+except OSError as ex:
+    # Only warn if the file actually exists and was unresolvable
+    # Otherwise users who specify a fake executable may get spurious warnings.
+    if isfile(real_executable):
+        warn(f'Failed to find real location of {base_executable}')
+
+if not executable_dir and os_name == 'darwin' and library:
+    # QUIRK: macOS checks adjacent to its library early
+    library_dir = dirname(library)
+    if any(isfile(joinpath(library_dir, p)) for p in STDLIB_LANDMARKS):
+        # Exceptions here should abort the whole process (to match
+        # previous behavior)
+        executable_dir = realpath(library_dir)
+        real_executable_dir = executable_dir
+
+# If we do not have the executable's directory, we can calculate it.
+# This is the directory used to find prefix/exec_prefix if necessary.
+if not executable_dir:
+    executable_dir = real_executable_dir = dirname(real_executable)
+
+# If we do not have the real executable's directory, we calculate it.
+# This is the directory used to detect build layouts.
+if not real_executable_dir:
+    real_executable_dir = dirname(real_executable)
+
+# ******************************************************************************
+# DETECT _pth FILE
+# ******************************************************************************
+
+# The contents of an optional ._pth file are used to totally override
+# sys.path calcualation. Its presence also implies isolated mode and
+# no-site (unless explicitly requested)
+pth = None
+pth_dir = None
+
+# Calling Py_SetPythonHome() or Py_SetPath() will override ._pth search,
+# but environment variables and command-line options cannot.
+if not py_setpath and not home_was_set:
+    # Check adjacent to the main DLL/dylib/so
+    if library:
+        try:
+            pth = readlines(library.rpartition('.')[0] + '._pth')
+            pth_dir = dirname(library)
+        except FileNotFoundError:
+            pass
+
+    # Check adjacent to the original executable, even if we
+    # redirected to actually launch Python. This may allow a
+    # venv to override the base_executable's ._pth file, but
+    # it cannot override the library's one.
+    if not pth_dir:
+        try:
+            pth = readlines(executable.rpartition('.')[0] + '._pth')
+            pth_dir = dirname(executable)
+        except FileNotFoundError:
+            pass
+
+    # If we found a ._pth file, disable environment and home
+    # detection now. Later, we will do the rest.
+    if pth_dir:
+        use_environment = 0
+        home = pth_dir
+        pythonpath = []
+
+
+# ******************************************************************************
+# CHECK FOR BUILD DIRECTORY
+# ******************************************************************************
+
+build_prefix = None
+
+if not home_was_set and real_executable_dir and not py_setpath:
+    # Detect a build marker and use it to infer prefix, exec_prefix,
+    # stdlib_dir and the platstdlib_dir directories.
+    try:
+        platstdlib_dir = joinpath(
+            real_executable_dir,
+            readlines(joinpath(real_executable_dir, BUILDDIR_TXT))[0],
+        )
+        build_prefix = joinpath(real_executable_dir, VPATH)
+    except FileNotFoundError:
+        if isfile(joinpath(real_executable_dir, BUILD_LANDMARK)):
+            build_prefix = joinpath(real_executable_dir, VPATH)
+            if os_name == 'nt':
+                # QUIRK: Windows builds need platstdlib_dir to be the executable
+                # dir. Normally the builddir marker handles this, but in this
+                # case we need to correct manually.
+                platstdlib_dir = real_executable_dir
+
+    if build_prefix:
+        if os_name == 'nt':
+            # QUIRK: No searching for more landmarks on Windows
+            build_stdlib_prefix = build_prefix
+        else:
+            build_stdlib_prefix = search_up(build_prefix, *BUILDSTDLIB_LANDMARKS)
+        # Always use the build prefix for stdlib
+        if build_stdlib_prefix:
+            stdlib_dir = joinpath(build_stdlib_prefix, 'Lib')
+        else:
+            stdlib_dir = joinpath(build_prefix, 'Lib')
+        # Only use the build prefix for prefix if it hasn't already been set
+        if not prefix:
+            prefix = build_stdlib_prefix
+        # Do not warn, because 'prefix' never equals 'build_prefix' on POSIX
+        #elif not venv_prefix and prefix != build_prefix:
+        #    warn('Detected development environment but prefix is already set')
+        if not exec_prefix:
+            exec_prefix = build_prefix
+        # Do not warn, because 'exec_prefix' never equals 'build_prefix' on POSIX
+        #elif not venv_prefix and exec_prefix != build_prefix:
+        #    warn('Detected development environment but exec_prefix is already set')
+        config['_is_python_build'] = 1
+
+
+# ******************************************************************************
+# CALCULATE prefix AND exec_prefix
+# ******************************************************************************
+
+if py_setpath:
+    # As documented, calling Py_SetPath will force both prefix
+    # and exec_prefix to the empty string.
+    prefix = exec_prefix = ''
+
+else:
+    # Read prefix and exec_prefix from explicitly set home
+    if home:
+        # When multiple paths are listed with ':' or ';' delimiters,
+        # split into prefix:exec_prefix
+        prefix, had_delim, exec_prefix = home.partition(DELIM)
+        if not had_delim:
+            exec_prefix = prefix
+
+
+    # First try to detect prefix by looking alongside our runtime library, if known
+    if library and not prefix:
+        library_dir = dirname(library)
+        if ZIP_LANDMARK:
+            if os_name == 'nt':
+                # QUIRK: Windows does not search up for ZIP file
+                if isfile(joinpath(library_dir, ZIP_LANDMARK)):
+                    prefix = library_dir
+            else:
+                prefix = search_up(library_dir, ZIP_LANDMARK)
+        if STDLIB_SUBDIR and STDLIB_LANDMARKS and not prefix:
+            if any(isfile(joinpath(library_dir, f)) for f in STDLIB_LANDMARKS):
+                prefix = library_dir
+                stdlib_dir = joinpath(prefix, STDLIB_SUBDIR)
+
+
+    # Detect prefix by looking for zip file
+    if ZIP_LANDMARK and executable_dir and not prefix:
+        if os_name == 'nt':
+            # QUIRK: Windows does not search up for ZIP file
+            if isfile(joinpath(executable_dir, ZIP_LANDMARK)):
+                prefix = executable_dir
+        else:
+            prefix = search_up(executable_dir, ZIP_LANDMARK)
+        if prefix:
+            stdlib_dir = joinpath(prefix, STDLIB_SUBDIR)
+            if not isdir(stdlib_dir):
+                stdlib_dir = None
+
+
+    # Detect prefix by searching from our executable location for the stdlib_dir
+    if STDLIB_SUBDIR and STDLIB_LANDMARKS and executable_dir and not prefix:
+        prefix = search_up(executable_dir, *STDLIB_LANDMARKS)
+        if prefix:
+            stdlib_dir = joinpath(prefix, STDLIB_SUBDIR)
+
+    if PREFIX and not prefix:
+        prefix = PREFIX
+        if not any(isfile(joinpath(prefix, f)) for f in STDLIB_LANDMARKS):
+            warn('Could not find platform independent libraries <prefix>')
+
+    if not prefix:
+        prefix = abspath('')
+        warn('Could not find platform independent libraries <prefix>')
+
+
+    # Detect exec_prefix by searching from executable for the platstdlib_dir
+    if PLATSTDLIB_LANDMARK and not exec_prefix:
+        if executable_dir:
+            exec_prefix = search_up(executable_dir, PLATSTDLIB_LANDMARK, test=isdir)
+        if not exec_prefix:
+            if EXEC_PREFIX:
+                exec_prefix = EXEC_PREFIX
+                if not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)):
+                    warn('Could not find platform dependent libraries <exec_prefix>')
+            else:
+                warn('Could not find platform dependent libraries <exec_prefix>')
+
+    # Fallback: assume exec_prefix == prefix
+    if not exec_prefix:
+        exec_prefix = prefix
+
+
+    if not prefix or not exec_prefix:
+        warn('Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]')
+
+
+# If we haven't set [plat]stdlib_dir already, set them now
+if not stdlib_dir:
+    if prefix:
+        stdlib_dir = joinpath(prefix, STDLIB_SUBDIR)
+    else:
+        stdlib_dir = ''
+
+if not platstdlib_dir:
+    if exec_prefix:
+        platstdlib_dir = joinpath(exec_prefix, PLATSTDLIB_LANDMARK)
+    else:
+        platstdlib_dir = ''
+
+
+# For a venv, update the main prefix/exec_prefix but leave the base ones unchanged
+# XXX: We currently do not update prefix here, but it happens in site.py
+#if venv_prefix:
+#    base_prefix = prefix
+#    base_exec_prefix = exec_prefix
+#    prefix = exec_prefix = venv_prefix
+
+
+# ******************************************************************************
+# UPDATE pythonpath (sys.path)
+# ******************************************************************************
+
+if py_setpath:
+    # If Py_SetPath was called then it overrides any existing search path
+    config['module_search_paths'] = py_setpath.split(DELIM)
+    config['module_search_paths_set'] = 1
+
+elif not pythonpath:
+    # If pythonpath was already set, we leave it alone.
+    # This won't matter in normal use, but if an embedded host is trying to
+    # recalculate paths while running then we do not want to change it.
+    pythonpath = []
+
+    # First add entries from the process environment
+    if use_environment and ENV_PYTHONPATH:
+        for p in ENV_PYTHONPATH.split(DELIM):
+            pythonpath.append(abspath(p))
+
+    # Then add the default zip file
+    if os_name == 'nt':
+        # QUIRK: Windows uses the library directory rather than the prefix
+        if library:
+            library_dir = dirname(library)
+        else:
+            library_dir = executable_dir
+        pythonpath.append(joinpath(library_dir, ZIP_LANDMARK))
+    elif build_prefix or venv_prefix:
+        # QUIRK: POSIX uses the default prefix when in the build directory
+        # or a venv
+        pythonpath.append(joinpath(PREFIX, ZIP_LANDMARK))
+    else:
+        pythonpath.append(joinpath(prefix, ZIP_LANDMARK))
+
+    if os_name == 'nt' and use_environment and winreg:
+        # QUIRK: Windows also lists paths in the registry. Paths are stored
+        # as the default value of each subkey of
+        # {HKCU,HKLM}\Software\Python\PythonCore\{winver}\PythonPath
+        # where winver is sys.winver (typically '3.x' or '3.x-32')
+        for hk in (winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE):
+            try:
+                key = winreg.OpenKeyEx(hk, WINREG_KEY)
+                try:
+                    i = 0
+                    while True:
+                        try:
+                            keyname = winreg.EnumKey(key, i)
+                            subkey = winreg.OpenKeyEx(key, keyname)
+                            if not subkey:
+                                continue
+                            try:
+                                v = winreg.QueryValue(subkey)
+                            finally:
+                                winreg.CloseKey(subkey)
+                            if isinstance(v, str):
+                                pythonpath.append(v)
+                            i += 1
+                        except OSError:
+                            break
+                finally:
+                    winreg.CloseKey(key)
+            except OSError:
+                pass
+
+    # Then add any entries compiled into the PYTHONPATH macro.
+    if PYTHONPATH:
+        for p in PYTHONPATH.split(DELIM):
+            pythonpath.append(joinpath(prefix, p))
+
+    # Then add stdlib_dir and platstdlib_dir
+    if stdlib_dir:
+        pythonpath.append(stdlib_dir)
+    if platstdlib_dir:
+        if os_name == 'nt' and venv_prefix:
+            # QUIRK: Windows appends executable_dir instead of platstdlib_dir
+            # when in a venv
+            pythonpath.append(executable_dir)
+        else:
+            pythonpath.append(platstdlib_dir)
+
+    config['module_search_paths'] = pythonpath
+    config['module_search_paths_set'] = 1
+
+
+# ******************************************************************************
+# POSIX prefix/exec_prefix QUIRKS
+# ******************************************************************************
+
+# QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running
+# in build directory. This happens after pythonpath calculation.
+if os_name != 'nt' and build_prefix:
+    prefix = config.get('prefix') or PREFIX
+    exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix
+
+
+# ******************************************************************************
+# SET pythonpath FROM _PTH FILE
+# ******************************************************************************
+
+if pth:
+    config['isolated'] = 1
+    config['use_environment'] = 0
+    config['site_import'] = 0
+    pythonpath = []
+    for line in pth:
+        line = line.partition('#')[0].strip()
+        if not line:
+            pass
+        elif line == 'import site':
+            config['site_import'] = 1
+        elif line.startswith('import '):
+            warn("unsupported 'import' line in ._pth file")
+        else:
+            pythonpath.append(joinpath(pth_dir, line))
+    config['module_search_paths'] = pythonpath
+    config['module_search_paths_set'] = 1
+
+# ******************************************************************************
+# UPDATE config FROM CALCULATED VALUES
+# ******************************************************************************
+
+config['program_name'] = program_name
+config['home'] = home
+config['executable'] = executable
+config['base_executable'] = base_executable
+config['prefix'] = prefix
+config['exec_prefix'] = exec_prefix
+config['base_prefix'] = base_prefix or prefix
+config['base_exec_prefix'] = base_exec_prefix or exec_prefix
+
+config['platlibdir'] = platlibdir
+config['stdlib_dir'] = stdlib_dir
+config['platstdlib_dir'] = platstdlib_dir
diff --git a/Modules/getpath_noop.c b/Modules/getpath_noop.c
new file mode 100644
index 0000000000000..c10e41d07f27c
--- /dev/null
+++ b/Modules/getpath_noop.c
@@ -0,0 +1,10 @@
+/* Implements the getpath API for compiling with no functionality */
+
+#include "Python.h"
+#include "pycore_pathconfig.h"
+
+PyStatus
+_PyConfig_InitPathConfig(PyConfig *config, int compute_path_config)
+{
+    return PyStatus_Error("path configuration is unsupported");
+}
diff --git a/Modules/main.c b/Modules/main.c
index c537e6b678515..b9bcea393abe3 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -534,11 +534,16 @@ pymain_repl(PyConfig *config, int *exitcode)
 static void
 pymain_run_python(int *exitcode)
 {
+    PyObject *main_importer_path = NULL;
     PyInterpreterState *interp = _PyInterpreterState_GET();
     /* pymain_run_stdin() modify the config */
     PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp);
 
-    PyObject *main_importer_path = NULL;
+    /* ensure path config is written into global variables */
+    if (_PyStatus_EXCEPTION(_PyPathConfig_UpdateGlobal(config))) {
+        goto error;
+    }
+
     if (config->run_filename != NULL) {
         /* If filename is a package (ex: directory or ZIP file) which contains
            __main__.py, main_importer_path is set to filename and will be
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index a89cff7295163..b1c2914fb0f1f 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -4444,6 +4444,33 @@ os__path_splitroot_impl(PyObject *module, path_t *path)
 #endif /* MS_WINDOWS */
 
 
+/*[clinic input]
+os._path_normpath
+
+    path: object
+
+Basic path normalization.
+[clinic start generated code]*/
+
+static PyObject *
+os__path_normpath_impl(PyObject *module, PyObject *path)
+/*[clinic end generated code: output=b94d696d828019da input=5e90c39e12549dc0]*/
+{
+    if (!PyUnicode_Check(path)) {
+        PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'",
+            Py_TYPE(path)->tp_name);
+        return NULL;
+    }
+    Py_ssize_t len;
+    wchar_t *buffer = PyUnicode_AsWideCharString(path, &len);
+    if (!buffer) {
+        return NULL;
+    }
+    PyObject *result = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1);
+    PyMem_Free(buffer);
+    return result;
+}
+
 /*[clinic input]
 os.mkdir
 
@@ -14826,6 +14853,7 @@ static PyMethodDef posix_methods[] = {
     OS__GETFINALPATHNAME_METHODDEF
     OS__GETVOLUMEPATHNAME_METHODDEF
     OS__PATH_SPLITROOT_METHODDEF
+    OS__PATH_NORMPATH_METHODDEF
     OS_GETLOADAVG_METHODDEF
     OS_URANDOM_METHODDEF
     OS_SETRESUID_METHODDEF
diff --git a/PC/config_minimal.c b/PC/config_minimal.c
index adb1c44a7242e..928a4efd32e13 100644
--- a/PC/config_minimal.c
+++ b/PC/config_minimal.c
@@ -5,6 +5,10 @@
 
 #include "Python.h"
 
+/* Define extern variables omitted from minimal builds */
+void *PyWin_DLLhModule = NULL;
+
+
 extern PyObject* PyInit_faulthandler(void);
 extern PyObject* PyInit__tracemalloc(void);
 extern PyObject* PyInit_gc(void);
diff --git a/PC/getpathp.c b/PC/getpathp.c
deleted file mode 100644
index a27348024d252..0000000000000
--- a/PC/getpathp.c
+++ /dev/null
@@ -1,1174 +0,0 @@
-
-/* Return the initial module search path. */
-/* Used by DOS, Windows 3.1, Windows 95/98, Windows NT. */
-
-/* ----------------------------------------------------------------
-   PATH RULES FOR WINDOWS:
-   This describes how sys.path is formed on Windows.  It describes the
-   functionality, not the implementation (ie, the order in which these
-   are actually fetched is different). The presence of a python._pth or
-   pythonXY._pth file alongside the program overrides these rules - see
-   below.
-
-   * Python always adds an empty entry at the start, which corresponds
-     to the current directory.
-
-   * If the PYTHONPATH env. var. exists, its entries are added next.
-
-   * We look in the registry for "application paths" - that is, sub-keys
-     under the main PythonPath registry key.  These are added next (the
-     order of sub-key processing is undefined).
-     HKEY_CURRENT_USER is searched and added first.
-     HKEY_LOCAL_MACHINE is searched and added next.
-     (Note that all known installers only use HKLM, so HKCU is typically
-     empty)
-
-   * We attempt to locate the "Python Home" - if the PYTHONHOME env var
-     is set, we believe it.  Otherwise, we use the path of our host .EXE's
-     to try and locate one of our "landmarks" and deduce our home.
-     - If we DO have a Python Home: The relevant sub-directories (Lib,
-       DLLs, etc) are based on the Python Home
-     - If we DO NOT have a Python Home, the core Python Path is
-       loaded from the registry.  (This is the main PythonPath key,
-       and both HKLM and HKCU are combined to form the path)
-
-   * Iff - we can not locate the Python Home, have not had a PYTHONPATH
-     specified, and can't locate any Registry entries (ie, we have _nothing_
-     we can assume is a good path), a default path with relative entries is
-     used (eg. .\Lib;.\DLLs, etc)
-
-
-   If a '._pth' file exists adjacent to the executable with the same base name
-   (e.g. python._pth adjacent to python.exe) or adjacent to the shared library
-   (e.g. python36._pth adjacent to python36.dll), it is used in preference to
-   the above process. The shared library file takes precedence over the
-   executable. The path file must contain a list of paths to add to sys.path,
-   one per line. Each path is relative to the directory containing the file.
-   Blank lines and comments beginning with '#' are permitted.
-
-   In the presence of this ._pth file, no other paths are added to the search
-   path, the registry finder is not enabled, site.py is not imported and
-   isolated mode is enabled. The site package can be enabled by including a
-   line reading "import site"; no other imports are recognized. Any invalid
-   entry (other than directories that do not exist) will result in immediate
-   termination of the program.
-
-
-  The end result of all this is:
-  * When running python.exe, or any other .exe in the main Python directory
-    (either an installed version, or directly from the PCbuild directory),
-    the core path is deduced, and the core paths in the registry are
-    ignored.  Other "application paths" in the registry are always read.
-
-  * When Python is hosted in another exe (different directory, embedded via
-    COM, etc), the Python Home will not be deduced, so the core path from
-    the registry is used.  Other "application paths" in the registry are
-    always read.
-
-  * If Python can't find its home and there is no registry (eg, frozen
-    exe, some very strange installation setup) you get a path with
-    some default, but relative, paths.
-
-  * An embedding application can use Py_SetPath() to override all of
-    these automatic path computations.
-
-  * An install of Python can fully specify the contents of sys.path using
-    either a 'EXENAME._pth' or 'DLLNAME._pth' file, optionally including
-    "import site" to enable the site module.
-
-   ---------------------------------------------------------------- */
-
-
-#include "Python.h"
-#include "pycore_fileutils.h"     // _Py_add_relfile()
-#include "pycore_initconfig.h"    // PyStatus
-#include "pycore_pathconfig.h"    // _PyPathConfig
-#include "osdefs.h"               // SEP, ALTSEP
-#include <wchar.h>
-
-#ifndef MS_WINDOWS
-#error getpathp.c should only be built on Windows
-#endif
-
-#include <windows.h>
-#include <pathcch.h>
-
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif /* HAVE_SYS_TYPES_H */
-
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif /* HAVE_SYS_STAT_H */
-
-#include <string.h>
-
-/* Search in some common locations for the associated Python libraries.
- *
- * Py_GetPath() tries to return a sensible Python module search path.
- *
- * The approach is an adaptation for Windows of the strategy used in
- * ../Modules/getpath.c; it uses the Windows Registry as one of its
- * information sources.
- *
- * Py_SetPath() can be used to override this mechanism.  Call Py_SetPath
- * with a semicolon separated path prior to calling Py_Initialize.
- */
-
-#define STDLIB_SUBDIR L"lib"
-
-#define INIT_ERR_BUFFER_OVERFLOW() _PyStatus_ERR("buffer overflow")
-
-
-typedef struct {
-    const wchar_t *path_env;           /* PATH environment variable */
-    const wchar_t *home;               /* PYTHONHOME environment variable */
-
-    /* Registry key "Software\Python\PythonCore\X.Y\PythonPath"
-       where X.Y is the Python version (major.minor) */
-    wchar_t *machine_path;   /* from HKEY_LOCAL_MACHINE */
-    wchar_t *user_path;      /* from HKEY_CURRENT_USER */
-
-    const wchar_t *pythonpath_env;
-} PyCalculatePath;
-
-
-/* determine if "ch" is a separator character */
-static int
-is_sep(wchar_t ch)
-{
-#ifdef ALTSEP
-    return ch == SEP || ch == ALTSEP;
-#else
-    return ch == SEP;
-#endif
-}
-
-
-/* assumes 'dir' null terminated in bounds.  Never writes
-   beyond existing terminator. */
-static void
-reduce(wchar_t *dir)
-{
-    size_t i = wcsnlen_s(dir, MAXPATHLEN+1);
-    if (i >= MAXPATHLEN+1) {
-        Py_FatalError("buffer overflow in getpathp.c's reduce()");
-    }
-
-    while (i > 0 && !is_sep(dir[i]))
-        --i;
-    dir[i] = '\0';
-}
-
-
-static int
-change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext)
-{
-    if (src && src != dest) {
-        size_t src_len = wcsnlen_s(src, MAXPATHLEN+1);
-        size_t i = src_len;
-        if (i >= MAXPATHLEN+1) {
-            Py_FatalError("buffer overflow in getpathp.c's reduce()");
-        }
-
-        while (i > 0 && src[i] != '.' && !is_sep(src[i]))
-            --i;
-
-        if (i == 0) {
-            dest[0] = '\0';
-            return -1;
-        }
-
-        if (is_sep(src[i])) {
-            i = src_len;
-        }
-
-        if (wcsncpy_s(dest, MAXPATHLEN+1, src, i)) {
-            dest[0] = '\0';
-            return -1;
-        }
-    } else {
-        wchar_t *s = wcsrchr(dest, L'.');
-        if (s) {
-            s[0] = '\0';
-        }
-    }
-
-    if (wcscat_s(dest, MAXPATHLEN+1, ext)) {
-        dest[0] = '\0';
-        return -1;
-    }
-
-    return 0;
-}
-
-
-static int
-exists(const wchar_t *filename)
-{
-    return GetFileAttributesW(filename) != 0xFFFFFFFF;
-}
-
-
-/* Is module -- check for .pyc too.
-   Assumes 'filename' MAXPATHLEN+1 bytes long -
-   may extend 'filename' by one character. */
-static int
-ismodule(wchar_t *filename)
-{
-    size_t n;
-
-    if (exists(filename)) {
-        return 1;
-    }
-
-    /* Check for the compiled version of prefix. */
-    n = wcsnlen_s(filename, MAXPATHLEN+1);
-    if (n < MAXPATHLEN) {
-        int exist = 0;
-        filename[n] = L'c';
-        filename[n + 1] = L'\0';
-        exist = exists(filename);
-        // Drop the 'c' we just added.
-        filename[n] = L'\0';
-        return exist;
-    }
-    return 0;
-}
-
-
-/* Add a path component, by appending stuff to buffer.
-   buffer must have at least MAXPATHLEN + 1 bytes allocated, and contain a
-   NUL-terminated string with no more than MAXPATHLEN characters (not counting
-   the trailing NUL).  It's a fatal error if it contains a string longer than
-   that (callers must be careful!).  If these requirements are met, it's
-   guaranteed that buffer will still be a NUL-terminated string with no more
-   than MAXPATHLEN characters at exit.  If stuff is too long, only as much of
-   stuff as fits will be appended.
-*/
-
-static void
-join(wchar_t *buffer, const wchar_t *stuff)
-{
-    if (_Py_add_relfile(buffer, stuff, MAXPATHLEN+1) < 0) {
-        Py_FatalError("buffer overflow in getpathp.c's join()");
-    }
-}
-
-/* Call PathCchCanonicalizeEx(path): remove navigation elements such as "."
-   and ".." to produce a direct, well-formed path. */
-static PyStatus
-canonicalize(wchar_t *buffer, const wchar_t *path)
-{
-    if (buffer == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-
-    const wchar_t *pathTail;
-    if (FAILED(PathCchSkipRoot(path, &pathTail)) || path == pathTail) {
-        wchar_t buff[MAXPATHLEN + 1];
-        if (!GetCurrentDirectoryW(MAXPATHLEN, buff)) {
-            return _PyStatus_ERR("unable to find current working directory");
-        }
-        if (FAILED(PathCchCombineEx(buff, MAXPATHLEN + 1, buff, path, PATHCCH_ALLOW_LONG_PATHS))) {
-            return INIT_ERR_BUFFER_OVERFLOW();
-        }
-        if (FAILED(PathCchCanonicalizeEx(buffer, MAXPATHLEN + 1, buff, PATHCCH_ALLOW_LONG_PATHS))) {
-            return INIT_ERR_BUFFER_OVERFLOW();
-        }
-        return _PyStatus_OK();
-    }
-
-    if (FAILED(PathCchCanonicalizeEx(buffer, MAXPATHLEN + 1, path, PATHCCH_ALLOW_LONG_PATHS))) {
-        return INIT_ERR_BUFFER_OVERFLOW();
-    }
-    return _PyStatus_OK();
-}
-
-static int
-is_stdlibdir(wchar_t *stdlibdir)
-{
-    wchar_t *filename = stdlibdir;
-#ifndef LANDMARK
-#  define LANDMARK L"os.py"
-#endif
-    /* join() ensures 'landmark' can not overflow prefix if too long. */
-    join(filename, LANDMARK);
-    return ismodule(filename);
-}
-
-/* assumes argv0_path is MAXPATHLEN+1 bytes long, already \0 term'd.
-   assumption provided by only caller, calculate_path() */
-static int
-search_for_prefix(wchar_t *prefix, const wchar_t *argv0_path)
-{
-    /* Search from argv0_path, until LANDMARK is found.
-       We guarantee 'prefix' is null terminated in bounds. */
-    wcscpy_s(prefix, MAXPATHLEN+1, argv0_path);
-    if (!prefix[0]) {
-        return 0;
-    }
-    wchar_t stdlibdir[MAXPATHLEN+1];
-    wcscpy_s(stdlibdir, Py_ARRAY_LENGTH(stdlibdir), prefix);
-    /* We initialize with the longest possible path, in case it doesn't fit.
-       This also gives us an initial SEP at stdlibdir[wcslen(prefix)]. */
-    join(stdlibdir, STDLIB_SUBDIR);
-    do {
-        assert(stdlibdir[wcslen(prefix)] == SEP);
-        /* Due to reduce() and our initial value, this result
-           is guaranteed to fit. */
-        wcscpy(&stdlibdir[wcslen(prefix) + 1], STDLIB_SUBDIR);
-        if (is_stdlibdir(stdlibdir)) {
-            return 1;
-        }
-        reduce(prefix);
-    } while (prefix[0]);
-    return 0;
-}
-
-
-static int
-get_dllpath(wchar_t *dllpath)
-{
-#ifdef Py_ENABLE_SHARED
-    extern HANDLE PyWin_DLLhModule;
-    if (PyWin_DLLhModule && GetModuleFileNameW(PyWin_DLLhModule, dllpath, MAXPATHLEN)) {
-        return 0;
-    }
-#endif
-    return -1;
-}
-
-
-#ifdef Py_ENABLE_SHARED
-
-/* a string loaded from the DLL at startup.*/
-extern const char *PyWin_DLLVersionString;
-
-/* Load a PYTHONPATH value from the registry.
-   Load from either HKEY_LOCAL_MACHINE or HKEY_CURRENT_USER.
-
-   Works in both Unicode and 8bit environments.  Only uses the
-   Ex family of functions so it also works with Windows CE.
-
-   Returns NULL, or a pointer that should be freed.
-
-   XXX - this code is pretty strange, as it used to also
-   work on Win16, where the buffer sizes were not available
-   in advance.  It could be simplied now Win16/Win32s is dead!
-*/
-static wchar_t *
-getpythonregpath(HKEY keyBase, int skipcore)
-{
-    HKEY newKey = 0;
-    DWORD dataSize = 0;
-    DWORD numKeys = 0;
-    LONG rc;
-    wchar_t *retval = NULL;
-    WCHAR *dataBuf = NULL;
-    static const WCHAR keyPrefix[] = L"Software\\Python\\PythonCore\\";
-    static const WCHAR keySuffix[] = L"\\PythonPath";
-    size_t versionLen, keyBufLen;
-    DWORD index;
-    WCHAR *keyBuf = NULL;
-    WCHAR *keyBufPtr;
-    WCHAR **ppPaths = NULL;
-
-    /* Tried to use sysget("winver") but here is too early :-( */
-    versionLen = strlen(PyWin_DLLVersionString);
-    /* Space for all the chars, plus one \0 */
-    keyBufLen = sizeof(keyPrefix) +
-                sizeof(WCHAR)*(versionLen-1) +
-                sizeof(keySuffix);
-    keyBuf = keyBufPtr = PyMem_RawMalloc(keyBufLen);
-    if (keyBuf==NULL) {
-        goto done;
-    }
-
-    memcpy_s(keyBufPtr, keyBufLen, keyPrefix, sizeof(keyPrefix)-sizeof(WCHAR));
-    keyBufPtr += Py_ARRAY_LENGTH(keyPrefix) - 1;
-    mbstowcs(keyBufPtr, PyWin_DLLVersionString, versionLen);
-    keyBufPtr += versionLen;
-    /* NULL comes with this one! */
-    memcpy(keyBufPtr, keySuffix, sizeof(keySuffix));
-    /* Open the root Python key */
-    rc=RegOpenKeyExW(keyBase,
-                    keyBuf, /* subkey */
-            0, /* reserved */
-            KEY_READ,
-            &newKey);
-    if (rc!=ERROR_SUCCESS) {
-        goto done;
-    }
-    /* Find out how big our core buffer is, and how many subkeys we have */
-    rc = RegQueryInfoKeyW(newKey, NULL, NULL, NULL, &numKeys, NULL, NULL,
-                    NULL, NULL, &dataSize, NULL, NULL);
-    if (rc!=ERROR_SUCCESS) {
-        goto done;
-    }
-    if (skipcore) {
-        dataSize = 0; /* Only count core ones if we want them! */
-    }
-    /* Allocate a temp array of char buffers, so we only need to loop
-       reading the registry once
-    */
-    ppPaths = PyMem_RawCalloc(numKeys, sizeof(WCHAR *));
-    if (ppPaths==NULL) {
-        goto done;
-    }
-    /* Loop over all subkeys, allocating a temp sub-buffer. */
-    for(index=0;index<numKeys;index++) {
-        WCHAR keyBuf[MAX_PATH+1];
-        HKEY subKey = 0;
-        DWORD reqdSize = MAX_PATH+1;
-        /* Get the sub-key name */
-        DWORD rc = RegEnumKeyExW(newKey, index, keyBuf, &reqdSize,
-                                 NULL, NULL, NULL, NULL );
-        if (rc!=ERROR_SUCCESS) {
-            goto done;
-        }
-        /* Open the sub-key */
-        rc=RegOpenKeyExW(newKey,
-                                        keyBuf, /* subkey */
-                        0, /* reserved */
-                        KEY_READ,
-                        &subKey);
-        if (rc!=ERROR_SUCCESS) {
-            goto done;
-        }
-        /* Find the value of the buffer size, malloc, then read it */
-        RegQueryValueExW(subKey, NULL, 0, NULL, NULL, &reqdSize);
-        if (reqdSize) {
-            ppPaths[index] = PyMem_RawMalloc(reqdSize);
-            if (ppPaths[index]) {
-                RegQueryValueExW(subKey, NULL, 0, NULL,
-                                (LPBYTE)ppPaths[index],
-                                &reqdSize);
-                dataSize += reqdSize + 1; /* 1 for the ";" */
-            }
-        }
-        RegCloseKey(subKey);
-    }
-
-    /* return null if no path to return */
-    if (dataSize == 0) {
-        goto done;
-    }
-
-    /* original datasize from RegQueryInfo doesn't include the \0 */
-    dataBuf = PyMem_RawMalloc((dataSize+1) * sizeof(WCHAR));
-    if (dataBuf) {
-        WCHAR *szCur = dataBuf;
-        /* Copy our collected strings */
-        for (index=0;index<numKeys;index++) {
-            if (index > 0) {
-                *(szCur++) = L';';
-                dataSize--;
-            }
-            if (ppPaths[index]) {
-                Py_ssize_t len = wcslen(ppPaths[index]);
-                wcsncpy(szCur, ppPaths[index], len);
-                szCur += len;
-                assert(dataSize > (DWORD)len);
-                dataSize -= (DWORD)len;
-            }
-        }
-        if (skipcore) {
-            *szCur = '\0';
-        }
-        else {
-            /* If we have no values, we don't need a ';' */
-            if (numKeys) {
-                *(szCur++) = L';';
-                dataSize--;
-            }
-            /* Now append the core path entries -
-               this will include the NULL
-            */
-            rc = RegQueryValueExW(newKey, NULL, 0, NULL,
-                                  (LPBYTE)szCur, &dataSize);
-            if (rc != ERROR_SUCCESS) {
-                PyMem_RawFree(dataBuf);
-                goto done;
-            }
-        }
-        /* And set the result - caller must free */
-        retval = dataBuf;
-    }
-done:
-    /* Loop freeing my temp buffers */
-    if (ppPaths) {
-        for(index=0; index<numKeys; index++)
-            PyMem_RawFree(ppPaths[index]);
-        PyMem_RawFree(ppPaths);
-    }
-    if (newKey) {
-        RegCloseKey(newKey);
-    }
-    PyMem_RawFree(keyBuf);
-    return retval;
-}
-#endif /* Py_ENABLE_SHARED */
-
-
-static PyStatus
-get_program_full_path(_PyPathConfig *pathconfig)
-{
-    PyStatus status;
-    const wchar_t *pyvenv_launcher;
-    wchar_t program_full_path[MAXPATHLEN+1];
-    memset(program_full_path, 0, sizeof(program_full_path));
-
-    if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) {
-        /* GetModuleFileName should never fail when passed NULL */
-        return _PyStatus_ERR("Cannot determine program path");
-    }
-
-    /* The launcher may need to force the executable path to a
-     * different environment, so override it here. */
-    pyvenv_launcher = _wgetenv(L"__PYVENV_LAUNCHER__");
-    if (pyvenv_launcher && pyvenv_launcher[0]) {
-        /* If overridden, preserve the original full path */
-        if (pathconfig->base_executable == NULL) {
-            pathconfig->base_executable = PyMem_RawMalloc(
-                sizeof(wchar_t) * (MAXPATHLEN + 1));
-            if (pathconfig->base_executable == NULL) {
-                return _PyStatus_NO_MEMORY();
-            }
-
-            status = canonicalize(pathconfig->base_executable,
-                                  program_full_path);
-            if (_PyStatus_EXCEPTION(status)) {
-                return status;
-            }
-        }
-
-        wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher);
-        /* bpo-35873: Clear the environment variable to avoid it being
-        * inherited by child processes. */
-        _wputenv_s(L"__PYVENV_LAUNCHER__", L"");
-    }
-
-    if (pathconfig->program_full_path == NULL) {
-        pathconfig->program_full_path = PyMem_RawMalloc(
-            sizeof(wchar_t) * (MAXPATHLEN + 1));
-        if (pathconfig->program_full_path == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-
-        status = canonicalize(pathconfig->program_full_path,
-                              program_full_path);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-    return _PyStatus_OK();
-}
-
-
-static PyStatus
-read_pth_file(_PyPathConfig *pathconfig, wchar_t *prefix, const wchar_t *path,
-              int *found)
-{
-    PyStatus status;
-    wchar_t *buf = NULL;
-    wchar_t *wline = NULL;
-    FILE *sp_file;
-
-    sp_file = _Py_wfopen(path, L"r");
-    if (sp_file == NULL) {
-        return _PyStatus_OK();
-    }
-
-    wcscpy_s(prefix, MAXPATHLEN+1, path);
-    reduce(prefix);
-    pathconfig->isolated = 1;
-    pathconfig->site_import = 0;
-
-    size_t bufsiz = MAXPATHLEN;
-    size_t prefixlen = wcslen(prefix);
-
-    buf = (wchar_t*)PyMem_RawMalloc(bufsiz * sizeof(wchar_t));
-    if (buf == NULL) {
-        status = _PyStatus_NO_MEMORY();
-        goto done;
-    }
-    buf[0] = '\0';
-
-    while (!feof(sp_file)) {
-        char line[MAXPATHLEN + 1];
-        char *p = fgets(line, Py_ARRAY_LENGTH(line), sp_file);
-        if (!p) {
-            break;
-        }
-        if (*p == '\0' || *p == '\r' || *p == '\n' || *p == '#') {
-            continue;
-        }
-        while (*++p) {
-            if (*p == '\r' || *p == '\n') {
-                *p = '\0';
-                break;
-            }
-        }
-
-        if (strcmp(line, "import site") == 0) {
-            pathconfig->site_import = 1;
-            continue;
-        }
-        else if (strncmp(line, "import ", 7) == 0) {
-            status = _PyStatus_ERR("only 'import site' is supported "
-                                   "in ._pth file");
-            goto done;
-        }
-
-        DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, NULL, 0);
-        wchar_t *wline = (wchar_t*)PyMem_RawMalloc((wn + 1) * sizeof(wchar_t));
-        if (wline == NULL) {
-            status = _PyStatus_NO_MEMORY();
-            goto done;
-        }
-        wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, wline, wn + 1);
-        wline[wn] = '\0';
-
-        size_t usedsiz = wcslen(buf);
-        while (usedsiz + wn + prefixlen + 4 > bufsiz) {
-            bufsiz += MAXPATHLEN;
-            wchar_t *tmp = (wchar_t*)PyMem_RawRealloc(buf, (bufsiz + 1) *
-                                                            sizeof(wchar_t));
-            if (tmp == NULL) {
-                status = _PyStatus_NO_MEMORY();
-                goto done;
-            }
-            buf = tmp;
-        }
-
-        if (usedsiz) {
-            wcscat_s(buf, bufsiz, L";");
-            usedsiz += 1;
-        }
-
-        errno_t result;
-        _Py_BEGIN_SUPPRESS_IPH
-        result = wcscat_s(buf, bufsiz, prefix);
-        _Py_END_SUPPRESS_IPH
-
-        if (result == EINVAL) {
-            status = _PyStatus_ERR("invalid argument during ._pth processing");
-            goto done;
-        } else if (result == ERANGE) {
-            status = _PyStatus_ERR("buffer overflow during ._pth processing");
-            goto done;
-        }
-
-        wchar_t *b = &buf[usedsiz];
-        join(b, wline);
-
-        PyMem_RawFree(wline);
-        wline = NULL;
-    }
-
-    if (pathconfig->module_search_path == NULL) {
-        pathconfig->module_search_path = _PyMem_RawWcsdup(buf);
-        if (pathconfig->module_search_path == NULL) {
-            status = _PyStatus_NO_MEMORY();
-            goto done;
-        }
-    }
-
-    *found = 1;
-    status = _PyStatus_OK();
-    goto done;
-
-done:
-    PyMem_RawFree(buf);
-    PyMem_RawFree(wline);
-    fclose(sp_file);
-    return status;
-}
-
-
-static int
-get_pth_filename(PyCalculatePath *calculate, wchar_t *filename,
-                 const _PyPathConfig *pathconfig)
-{
-    if (!get_dllpath(filename) &&
-        !change_ext(filename, filename, L"._pth") &&
-        exists(filename))
-    {
-        return 1;
-    }
-    if (pathconfig->program_full_path[0] &&
-        !change_ext(filename, pathconfig->program_full_path, L"._pth") &&
-        exists(filename))
-    {
-        return 1;
-    }
-    return 0;
-}
-
-
-static PyStatus
-calculate_pth_file(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
-                   wchar_t *prefix, int *found)
-{
-    wchar_t filename[MAXPATHLEN+1];
-
-    if (!get_pth_filename(calculate, filename, pathconfig)) {
-        return _PyStatus_OK();
-    }
-
-    return read_pth_file(pathconfig, prefix, filename, found);
-}
-
-
-/* Search for an environment configuration file, first in the
-   executable's directory and then in the parent directory.
-   If found, open it for use when searching for prefixes.
-*/
-static PyStatus
-calculate_pyvenv_file(PyCalculatePath *calculate,
-                      wchar_t *argv0_path, size_t argv0_path_len)
-{
-    wchar_t filename[MAXPATHLEN+1];
-    const wchar_t *env_cfg = L"pyvenv.cfg";
-
-    /* Filename: <argv0_path_len> / "pyvenv.cfg" */
-    wcscpy_s(filename, MAXPATHLEN+1, argv0_path);
-    join(filename, env_cfg);
-
-    FILE *env_file = _Py_wfopen(filename, L"r");
-    if (env_file == NULL) {
-        errno = 0;
-
-        /* Filename: <basename(basename(argv0_path_len))> / "pyvenv.cfg" */
-        reduce(filename);
-        reduce(filename);
-        join(filename, env_cfg);
-
-        env_file = _Py_wfopen(filename, L"r");
-        if (env_file == NULL) {
-            errno = 0;
-            return _PyStatus_OK();
-        }
-    }
-
-    /* Look for a 'home' variable and set argv0_path to it, if found */
-    wchar_t *home = NULL;
-    PyStatus status = _Py_FindEnvConfigValue(env_file, L"home", &home);
-    if (_PyStatus_EXCEPTION(status)) {
-        fclose(env_file);
-        return status;
-    }
-    if (home) {
-        wcscpy_s(argv0_path, argv0_path_len, home);
-        PyMem_RawFree(home);
-    }
-    fclose(env_file);
-    return _PyStatus_OK();
-}
-
-
-static void
-calculate_home_prefix(PyCalculatePath *calculate,
-                      const wchar_t *argv0_path,
-                      const wchar_t *zip_path,
-                      wchar_t *prefix)
-{
-    if (calculate->home == NULL || *calculate->home == '\0') {
-        if (zip_path[0] && exists(zip_path)) {
-            wcscpy_s(prefix, MAXPATHLEN+1, zip_path);
-            reduce(prefix);
-            calculate->home = prefix;
-        }
-        else if (search_for_prefix(prefix, argv0_path)) {
-            calculate->home = prefix;
-        }
-        else {
-            calculate->home = NULL;
-        }
-    }
-    else {
-        wcscpy_s(prefix, MAXPATHLEN+1, calculate->home);
-    }
-}
-
-
-static PyStatus
-calculate_module_search_path(PyCalculatePath *calculate,
-                             _PyPathConfig *pathconfig,
-                             const wchar_t *argv0_path,
-                             wchar_t *prefix,
-                             const wchar_t *zip_path)
-{
-    int skiphome = calculate->home==NULL ? 0 : 1;
-#ifdef Py_ENABLE_SHARED
-    if (!Py_IgnoreEnvironmentFlag) {
-        calculate->machine_path = getpythonregpath(HKEY_LOCAL_MACHINE,
-                                                   skiphome);
-        calculate->user_path = getpythonregpath(HKEY_CURRENT_USER, skiphome);
-    }
-#endif
-    /* We only use the default relative PYTHONPATH if we haven't
-       anything better to use! */
-    int skipdefault = (calculate->pythonpath_env != NULL ||
-                       calculate->home != NULL ||
-                       calculate->machine_path != NULL ||
-                       calculate->user_path != NULL);
-
-    /* We need to construct a path from the following parts.
-       (1) the PYTHONPATH environment variable, if set;
-       (2) for Win32, the zip archive file path;
-       (3) for Win32, the machine_path and user_path, if set;
-       (4) the PYTHONPATH config macro, with the leading "."
-           of each component replaced with home, if set;
-       (5) the directory containing the executable (argv0_path).
-       The length calculation calculates #4 first.
-       Extra rules:
-       - If PYTHONHOME is set (in any way) item (3) is ignored.
-       - If registry values are used, (4) and (5) are ignored.
-    */
-
-    /* Calculate size of return buffer */
-    size_t bufsz = 0;
-    if (calculate->home != NULL) {
-        const wchar_t *p;
-        bufsz = 1;
-        for (p = PYTHONPATH; *p; p++) {
-            if (*p == DELIM) {
-                bufsz++; /* number of DELIM plus one */
-            }
-        }
-        bufsz *= wcslen(calculate->home);
-    }
-    bufsz += wcslen(PYTHONPATH) + 1;
-    bufsz += wcslen(argv0_path) + 1;
-    if (calculate->user_path) {
-        bufsz += wcslen(calculate->user_path) + 1;
-    }
-    if (calculate->machine_path) {
-        bufsz += wcslen(calculate->machine_path) + 1;
-    }
-    bufsz += wcslen(zip_path) + 1;
-    if (calculate->pythonpath_env != NULL) {
-        bufsz += wcslen(calculate->pythonpath_env) + 1;
-    }
-
-    wchar_t *buf, *start_buf;
-    buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t));
-    if (buf == NULL) {
-        return _PyStatus_NO_MEMORY();
-    }
-    start_buf = buf;
-
-    if (calculate->pythonpath_env) {
-        if (wcscpy_s(buf, bufsz - (buf - start_buf),
-                     calculate->pythonpath_env)) {
-            return INIT_ERR_BUFFER_OVERFLOW();
-        }
-        buf = wcschr(buf, L'\0');
-        *buf++ = DELIM;
-    }
-    if (zip_path[0]) {
-        if (wcscpy_s(buf, bufsz - (buf - start_buf), zip_path)) {
-            return INIT_ERR_BUFFER_OVERFLOW();
-        }
-        buf = wcschr(buf, L'\0');
-        *buf++ = DELIM;
-    }
-    if (calculate->user_path) {
-        if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->user_path)) {
-            return INIT_ERR_BUFFER_OVERFLOW();
-        }
-        buf = wcschr(buf, L'\0');
-        *buf++ = DELIM;
-    }
-    if (calculate->machine_path) {
-        if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->machine_path)) {
-            return INIT_ERR_BUFFER_OVERFLOW();
-        }
-        buf = wcschr(buf, L'\0');
-        *buf++ = DELIM;
-    }
-    if (calculate->home == NULL) {
-        if (!skipdefault) {
-            if (wcscpy_s(buf, bufsz - (buf - start_buf), PYTHONPATH)) {
-                return INIT_ERR_BUFFER_OVERFLOW();
-            }
-            buf = wcschr(buf, L'\0');
-            *buf++ = DELIM;
-        }
-    } else {
-        const wchar_t *p = PYTHONPATH;
-        const wchar_t *q;
-        size_t n;
-        for (;;) {
-            q = wcschr(p, DELIM);
-            if (q == NULL) {
-                n = wcslen(p);
-            }
-            else {
-                n = q-p;
-            }
-            if (p[0] == '.' && is_sep(p[1])) {
-                if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->home)) {
-                    return INIT_ERR_BUFFER_OVERFLOW();
-                }
-                buf = wcschr(buf, L'\0');
-                p++;
-                n--;
-            }
-            wcsncpy(buf, p, n);
-            buf += n;
-            *buf++ = DELIM;
-            if (q == NULL) {
-                break;
-            }
-            p = q+1;
-        }
-    }
-    if (argv0_path) {
-        wcscpy(buf, argv0_path);
-        buf = wcschr(buf, L'\0');
-        *buf++ = DELIM;
-    }
-    *(buf - 1) = L'\0';
-
-    /* Now to pull one last hack/trick.  If sys.prefix is
-       empty, then try and find it somewhere on the paths
-       we calculated.  We scan backwards, as our general policy
-       is that Python core directories are at the *end* of
-       sys.path.  We assume that our "lib" directory is
-       on the path, and that our 'prefix' directory is
-       the parent of that.
-    */
-    if (prefix[0] == L'\0') {
-        PyStatus status;
-        wchar_t lookBuf[MAXPATHLEN+1];
-        const wchar_t *look = buf - 1; /* 'buf' is at the end of the buffer */
-        while (1) {
-            Py_ssize_t nchars;
-            const wchar_t *lookEnd = look;
-            /* 'look' will end up one character before the
-               start of the path in question - even if this
-               is one character before the start of the buffer
-            */
-            while (look >= start_buf && *look != DELIM)
-                look--;
-            nchars = lookEnd-look;
-            wcsncpy(lookBuf, look+1, nchars);
-            lookBuf[nchars] = L'\0';
-            status = canonicalize(lookBuf, lookBuf);
-            if (_PyStatus_EXCEPTION(status)) {
-                return status;
-            }
-            /* Up one level to the parent */
-            reduce(lookBuf);
-            if (search_for_prefix(prefix, lookBuf)) {
-                break;
-            }
-            /* If we are out of paths to search - give up */
-            if (look < start_buf) {
-                break;
-            }
-            look--;
-        }
-    }
-
-    pathconfig->module_search_path = start_buf;
-    return _PyStatus_OK();
-}
-
-
-static PyStatus
-calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
-{
-    PyStatus status;
-
-    status = get_program_full_path(pathconfig);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-
-    /* program_full_path guaranteed \0 terminated in MAXPATH+1 bytes. */
-    wchar_t argv0_path[MAXPATHLEN+1];
-    memset(argv0_path, 0, sizeof(argv0_path));
-
-    wcscpy_s(argv0_path, MAXPATHLEN+1, pathconfig->program_full_path);
-    reduce(argv0_path);
-
-    wchar_t prefix[MAXPATHLEN+1];
-    memset(prefix, 0, sizeof(prefix));
-
-    /* Search for a sys.path file */
-    int pth_found = 0;
-    status = calculate_pth_file(calculate, pathconfig, prefix, &pth_found);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-    if (pth_found) {
-        goto done;
-    }
-
-    status = calculate_pyvenv_file(calculate,
-                                   argv0_path, Py_ARRAY_LENGTH(argv0_path));
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-
-    /* Calculate zip archive path from DLL or exe path */
-    wchar_t zip_path[MAXPATHLEN+1];
-    memset(zip_path, 0, sizeof(zip_path));
-
-    if (get_dllpath(zip_path) || change_ext(zip_path, zip_path, L".zip"))
-    {
-        if (change_ext(zip_path, pathconfig->program_full_path, L".zip")) {
-            zip_path[0] = L'\0';
-        }
-    }
-
-    calculate_home_prefix(calculate, argv0_path, zip_path, prefix);
-
-    if (pathconfig->module_search_path == NULL) {
-        status = calculate_module_search_path(calculate, pathconfig,
-                                              argv0_path, prefix, zip_path);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-
-done:
-    if (pathconfig->stdlib_dir == NULL) {
-        pathconfig->stdlib_dir = _Py_join_relfile(prefix, STDLIB_SUBDIR);
-        if (pathconfig->stdlib_dir == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-    }
-    if (pathconfig->prefix == NULL) {
-        pathconfig->prefix = _PyMem_RawWcsdup(prefix);
-        if (pathconfig->prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-    }
-    if (pathconfig->exec_prefix == NULL) {
-        pathconfig->exec_prefix = _PyMem_RawWcsdup(prefix);
-        if (pathconfig->exec_prefix == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-    }
-
-    return _PyStatus_OK();
-}
-
-
-static PyStatus
-calculate_init(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
-               const PyConfig *config)
-{
-    calculate->home = pathconfig->home;
-    calculate->path_env = _wgetenv(L"PATH");
-
-    calculate->pythonpath_env = config->pythonpath_env;
-
-    return _PyStatus_OK();
-}
-
-
-static void
-calculate_free(PyCalculatePath *calculate)
-{
-    PyMem_RawFree(calculate->machine_path);
-    PyMem_RawFree(calculate->user_path);
-}
-
-
-/* Calculate the Python path configuration.
-
-   Inputs:
-
-   - PyConfig.pythonpath_env: PYTHONPATH environment variable
-   - _PyPathConfig.home: Py_SetPythonHome() or PYTHONHOME environment variable
-   - PATH environment variable
-   - __PYVENV_LAUNCHER__ environment variable
-   - GetModuleFileNameW(NULL): fully qualified path of the executable file of
-     the current process
-   - ._pth configuration file
-   - pyvenv.cfg configuration file
-   - Registry key "Software\Python\PythonCore\X.Y\PythonPath"
-     of HKEY_CURRENT_USER and HKEY_LOCAL_MACHINE where X.Y is the Python
-     version.
-
-   Outputs, 'pathconfig' fields:
-
-   - base_executable
-   - program_full_path
-   - module_search_path
-   - prefix
-   - exec_prefix
-   - isolated
-   - site_import
-
-   If a field is already set (non NULL), it is left unchanged. */
-PyStatus
-_PyPathConfig_Calculate(_PyPathConfig *pathconfig, const PyConfig *config)
-{
-    PyStatus status;
-    PyCalculatePath calculate;
-    memset(&calculate, 0, sizeof(calculate));
-
-    status = calculate_init(&calculate, pathconfig, config);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto done;
-    }
-
-    status = calculate_path(&calculate, pathconfig);
-
-done:
-    calculate_free(&calculate);
-    return status;
-}
-
-
-/* Load python3.dll before loading any extension module that might refer
-   to it. That way, we can be sure that always the python3.dll corresponding
-   to this python DLL is loaded, not a python3.dll that might be on the path
-   by chance.
-   Return whether the DLL was found.
-*/
-static int python3_checked = 0;
-static HANDLE hPython3;
-int
-_Py_CheckPython3(void)
-{
-    wchar_t py3path[MAXPATHLEN+1];
-    if (python3_checked) {
-        return hPython3 != NULL;
-    }
-    python3_checked = 1;
-
-    /* If there is a python3.dll next to the python3y.dll,
-       use that DLL */
-    if (!get_dllpath(py3path)) {
-        reduce(py3path);
-        join(py3path, PY3_DLLNAME);
-        hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
-        if (hPython3 != NULL) {
-            return 1;
-        }
-    }
-
-    /* If we can locate python3.dll in our application dir,
-       use that DLL */
-    hPython3 = LoadLibraryExW(PY3_DLLNAME, NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR);
-    if (hPython3 != NULL) {
-        return 1;
-    }
-
-    /* For back-compat, also search {sys.prefix}\DLLs, though
-       that has not been a normal install layout for a while */
-    wcscpy(py3path, Py_GetPrefix());
-    if (py3path[0]) {
-        join(py3path, L"DLLs\\" PY3_DLLNAME);
-        hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
-    }
-    return hPython3 != NULL;
-}
diff --git a/PC/pyconfig.h b/PC/pyconfig.h
index 5d8d9f3618454..e0d875adf2e4a 100644
--- a/PC/pyconfig.h
+++ b/PC/pyconfig.h
@@ -66,9 +66,6 @@ WIN32 is still required for the locale module.
 
 #define MS_WIN32 /* only support win32 and greater. */
 #define MS_WINDOWS
-#ifndef PYTHONPATH
-#       define PYTHONPATH L".\\DLLs;.\\lib"
-#endif
 #define NT_THREADS
 #define WITH_THREAD
 #ifndef NETSCAPE_PI
@@ -671,6 +668,4 @@ Py_NO_ENABLE_SHARED to find out.  Also support MS_NO_COREDLL for b/w compat */
 /* Define if libssl has X509_VERIFY_PARAM_set1_host and related function */
 #define HAVE_X509_VERIFY_PARAM_SET1_HOST 1
 
-#define PLATLIBDIR "lib"
-
 #endif /* !Py_CONFIG_H */
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index 54fef9ca629b1..7b2df4b8afcc2 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -104,6 +104,7 @@
     <ClCompile Include="..\Modules\faulthandler.c" />
     <ClCompile Include="..\Modules\gcmodule.c" />
     <ClCompile Include="..\Modules\getbuildinfo.c" />
+    <ClCompile Include="..\Modules\getpath_noop.c" />
     <ClCompile Include="..\Modules\posixmodule.c" />
     <ClCompile Include="..\Modules\signalmodule.c" />
     <ClCompile Include="..\Modules\_tracemalloc.c" />
@@ -168,7 +169,6 @@
     <ClCompile Include="..\Parser\string_parser.c" />
     <ClCompile Include="..\Parser\token.c" />
     <ClCompile Include="..\Parser\tokenizer.c" />
-    <ClCompile Include="..\PC\getpathp.c" />
     <ClCompile Include="..\PC\invalid_parameter_handler.c" />
     <ClCompile Include="..\PC\msvcrtmodule.c" />
     <ClCompile Include="..\PC\winreg.c" />
@@ -374,9 +374,29 @@
     </None>
     <!-- END frozen modules -->
   </ItemGroup>
+  <ItemGroup>
+    <!-- We manually freeze getpath.py rather than through freeze_modules -->
+    <GetPath Include="..\Modules\getpath.py">
+      <ModName>getpath</ModName>
+      <IntFile>$(IntDir)getpath.g.h</IntFile>
+      <OutFile>$(PySourcePath)Modules\getpath.h</OutFile>
+    </GetPath>
+  </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
+  <Target Name="_RebuildGetPath" AfterTargets="_RebuildFrozen" Condition="$(Configuration) != 'PGUpdate'">
+    <Exec Command='"$(TargetPath)" "%(GetPath.ModName)" "%(GetPath.FullPath)" "%(GetPath.IntFile)"' />
+
+    <Copy SourceFiles="%(GetPath.IntFile)"
+          DestinationFiles="%(GetPath.OutFile)"
+          Condition="!Exists(%(GetPath.OutFile)) or (Exists(%(GetPath.IntFile)) and '$([System.IO.File]::ReadAllText(%(GetPath.OutFile)).Replace(`&#x0D;&#x0A;`, `&#x0A;`))' != '$([System.IO.File]::ReadAllText(%(GetPath.IntFile)).Replace(`&#x0D;&#x0A;`, `&#x0A;`))')">
+      <Output TaskParameter="CopiedFiles" ItemName="_UpdatedGetPath" />
+    </Copy>
+
+    <Message Text="Updated files: @(_UpdatedGetPath->'%(Filename)%(Extension)',', ')"
+             Condition="'@(_UpdatedGetPath)' != ''" Importance="high" />
+  </Target>
   <Target Name="_RebuildFrozen" AfterTargets="AfterBuild" Condition="$(Configuration) != 'PGUpdate'">
     <Exec Command='"$(TargetPath)" "%(None.ModName)" "%(None.FullPath)" "%(None.DeepIntFile)"' />
 
diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters
index 5894909e0fbe1..1c8f1b0dcf449 100644
--- a/PCbuild/_freeze_module.vcxproj.filters
+++ b/PCbuild/_freeze_module.vcxproj.filters
@@ -13,6 +13,395 @@
     <ClCompile Include="..\Programs\_freeze_module.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Modules\_io\_iomodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_tracemalloc.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\_warnings.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\abstract.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\accu.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\asdl.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\ast.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\ast_opt.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\ast_unparse.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\atexitmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\bltinmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\boolobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\bootstrap_hash.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\bufferedio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\bytearrayobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\bytes_methods.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\bytesio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\bytesobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\call.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\capsule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\cellobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\ceval.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\classobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\codecs.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\codeobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\compile.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\complexobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\PC\config_minimal.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\context.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\descrobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\dictobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\dtoa.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\dynamic_annotations.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\dynload_win.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\enumobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\errors.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\exceptions.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\faulthandler.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\fileio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\fileobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\fileutils.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\floatobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\formatter_unicode.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\frame.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\frameobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\funcobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\future.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\gcmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\genericaliasobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\genobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getargs.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\getbuildinfo.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getcompiler.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getcopyright.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getopt.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\getpath_noop.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getplatform.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\getversion.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\hamt.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\hashtable.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\import.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\importdl.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\initconfig.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\interpreteridobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\PC\invalid_parameter_handler.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\iobase.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\iterobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\listobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\longobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\marshal.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\memoryobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\methodobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\modsupport.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\moduleobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\PC\msvcrtmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\myreadline.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\mysnprintf.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\mystrtoul.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\namespaceobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\object.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\obmalloc.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\odictobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\parser.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pathconfig.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\peg_api.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\pegen.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\picklebufobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\posixmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\preconfig.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pyarena.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pyctype.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pyfpe.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pyhash.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pylifecycle.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pymath.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pystate.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pystrcmp.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pystrhex.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pystrtod.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\Python-ast.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pythonrun.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\Python-tokenize.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\pytime.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\rangeobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\setobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\signalmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\sliceobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\specialize.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\string_parser.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\stringio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\structmember.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\structseq.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\suggestions.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\symtable.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\sysmodule.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\textio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\thread.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\token.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Parser\tokenizer.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\traceback.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\tupleobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\typeobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\unicodectype.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\unicodeobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\unionobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Objects\weakrefobject.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\_io\winconsoleio.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\PC\winreg.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="..\Modules\getpath.py">
+      <Filter>Python Files</Filter>
+    </None>
   </ItemGroup>
   <ItemGroup>
     <!-- BEGIN frozen modules -->
@@ -78,4 +467,4 @@
     </None>
     <!-- END frozen modules -->
   </ItemGroup>
-</Project>
+</Project>
\ No newline at end of file
diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj
index b58945a4d19b6..0b4329dd07636 100644
--- a/PCbuild/python.vcxproj
+++ b/PCbuild/python.vcxproj
@@ -147,4 +147,11 @@ $(_PGOPath)
     </PropertyGroup>
     <WriteLinesToFile File="$(PySourcePath)python.bat" Lines="$(_Content)" Overwrite="true" Condition="'$(_Content)' != '$(_ExistingContent)'" />
   </Target>
+  <Target Name="GeneratePyBuildDirTxt" BeforeTargets="AfterBuild">
+    <PropertyGroup>
+      <_Content>$(OutDir)</_Content>
+      <_ExistingContent Condition="Exists('$(OutDir)pybuilddir.txt')">$([System.IO.File]::ReadAllText('$(OutDir)pybuilddir.txt'))</_ExistingContent>
+    </PropertyGroup>
+    <WriteLinesToFile File="$(OutDir)pybuilddir.txt" Lines="$(_Content)" Overwrite="true" Condition="'$(_Content)' != '$(_ExistingContent)'" />
+  </Target>
 </Project>
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index 9d96f4bd5361b..b446e0995565e 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -109,6 +109,19 @@
       <AdditionalDependencies>version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Link>
   </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="..\Modules\getpath.c">
+      <PreprocessorDefinitions>
+        PREFIX=NULL;
+        EXEC_PREFIX=NULL;
+        VERSION=NULL;
+        VPATH="..\\..";
+        PYDEBUGEXT="$(PyDebugExt)";
+        PLATLIBDIR="DLLs";
+        %(PreprocessorDefinitions)
+      </PreprocessorDefinitions>
+    </ClCompile>
+  </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\Include\Python.h" />
     <ClInclude Include="..\Include\abstract.h" />
@@ -349,6 +362,7 @@
     <ClCompile Include="..\Modules\errnomodule.c" />
     <ClCompile Include="..\Modules\faulthandler.c" />
     <ClCompile Include="..\Modules\gcmodule.c" />
+    <ClCompile Include="..\Modules\getbuildinfo.c" />
     <ClCompile Include="..\Modules\itertoolsmodule.c" />
     <ClCompile Include="..\Modules\main.c" />
     <ClCompile Include="..\Modules\mathmodule.c" />
@@ -442,7 +456,6 @@
     <ClCompile Include="..\PC\invalid_parameter_handler.c" />
     <ClCompile Include="..\PC\winreg.c" />
     <ClCompile Include="..\PC\config.c" />
-    <ClCompile Include="..\PC\getpathp.c" />
     <ClCompile Include="..\PC\msvcrtmodule.c" />
     <ClCompile Include="..\Python\pyhash.c" />
     <ClCompile Include="..\Python\_warnings.c" />
@@ -570,7 +583,7 @@
     </PropertyGroup>
     <Message Text="Building $(GitTag):$(GitVersion) $(GitBranch)" Importance="high" />
     <ItemGroup>
-      <ClCompile Include="..\Modules\getbuildinfo.c">
+      <ClCompile Condition="$(Filename) == 'getbuildinfo'">
         <PreprocessorDefinitions>GITVERSION="$(GitVersion)";GITTAG="$(GitTag)";GITBRANCH="$(GitBranch)";%(PreprocessorDefinitions)</PreprocessorDefinitions>
       </ClCompile>
     </ItemGroup>
@@ -590,4 +603,8 @@
   <Target Name="_CleanVCRuntime" AfterTargets="Clean">
     <Delete Files="@(VCRuntimeDLL->'$(OutDir)%(Filename)%(Extension)')" />
   </Target>
+
+  <Target Name="_DeletePyBuildDirTxt" BeforeTargets="PrepareForBuild">
+    <Delete Files="$(OutDir)pybuilddir.txt" />
+  </Target>
 </Project>
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index b19f0279ec311..c1667e3fb74fd 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -648,6 +648,9 @@
     <ClInclude Include="$(zlibDir)\zutil.h">
       <Filter>Modules\zlib</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_structseq.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\Modules\_abc.c">
@@ -971,9 +974,6 @@
     <ClCompile Include="..\PC\dl_nt.c">
       <Filter>PC</Filter>
     </ClCompile>
-    <ClCompile Include="..\PC\getpathp.c">
-      <Filter>PC</Filter>
-    </ClCompile>
     <ClCompile Include="..\PC\msvcrtmodule.c">
       <Filter>PC</Filter>
     </ClCompile>
@@ -1229,10 +1229,25 @@
     <ClCompile Include="..\Objects\unionobject.c">
       <Filter>Objects</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\frame.c">
+      <Filter>Python</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\getpath.c">
+      <Filter>Modules</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\suggestions.c">
+      <Filter>Python</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Python\Python-tokenize.c">
+      <Filter>Python</Filter>
+    </ClCompile>
+    <ClCompile Include="..\Modules\getbuildinfo.c">
+      <Filter>Modules</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="..\PC\python_nt.rc">
       <Filter>Resource Files</Filter>
     </ResourceCompile>
   </ItemGroup>
-</Project>
+</Project>
\ No newline at end of file
diff --git a/Python/dynload_win.c b/Python/dynload_win.c
index 5702ab2cd71ba..854b1e64c15fa 100644
--- a/Python/dynload_win.c
+++ b/Python/dynload_win.c
@@ -2,6 +2,9 @@
 /* Support for dynamic loading of extension modules */
 
 #include "Python.h"
+#include "pycore_fileutils.h"     // _Py_add_relfile()
+#include "pycore_pathconfig.h"    // _PyPathConfig_ComputeSysPath0()
+#include "pycore_pystate.h"       // _PyInterpreterState_GET()
 
 #ifdef HAVE_DIRECT_H
 #include <direct.h>
@@ -160,6 +163,60 @@ static char *GetPythonImport (HINSTANCE hModule)
     return NULL;
 }
 
+/* Load python3.dll before loading any extension module that might refer
+   to it. That way, we can be sure that always the python3.dll corresponding
+   to this python DLL is loaded, not a python3.dll that might be on the path
+   by chance.
+   Return whether the DLL was found.
+*/
+extern HMODULE PyWin_DLLhModule;
+static int
+_Py_CheckPython3(void)
+{
+    static int python3_checked = 0;
+    static HANDLE hPython3;
+    #define MAXPATHLEN 512
+    wchar_t py3path[MAXPATHLEN+1];
+    if (python3_checked) {
+        return hPython3 != NULL;
+    }
+    python3_checked = 1;
+
+    /* If there is a python3.dll next to the python3y.dll,
+       use that DLL */
+    if (PyWin_DLLhModule && GetModuleFileNameW(PyWin_DLLhModule, py3path, MAXPATHLEN)) {
+        wchar_t *p = wcsrchr(py3path, L'\\');
+        if (p) {
+            wcscpy(p + 1, PY3_DLLNAME);
+            hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+            if (hPython3 != NULL) {
+                return 1;
+            }
+        }
+    }
+
+    /* If we can locate python3.dll in our application dir,
+       use that DLL */
+    hPython3 = LoadLibraryExW(PY3_DLLNAME, NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR);
+    if (hPython3 != NULL) {
+        return 1;
+    }
+
+    /* For back-compat, also search {sys.prefix}\DLLs, though
+       that has not been a normal install layout for a while */
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp);
+    assert(config->prefix);
+    if (config->prefix) {
+        wcscpy_s(py3path, MAXPATHLEN, config->prefix);
+        if (py3path[0] && _Py_add_relfile(py3path, L"DLLs\\" PY3_DLLNAME, MAXPATHLEN) >= 0) {
+            hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+        }
+    }
+    return hPython3 != NULL;
+    #undef MAXPATHLEN
+}
+
 dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix,
                                               const char *shortname,
                                               PyObject *pathname, FILE *fp)
diff --git a/Python/fileutils.c b/Python/fileutils.c
index ac0046cdac37c..cae6b75b6ae9b 100644
--- a/Python/fileutils.c
+++ b/Python/fileutils.c
@@ -2000,13 +2000,28 @@ _Py_wrealpath(const wchar_t *path,
 #endif
 
 
-#ifndef MS_WINDOWS
 int
 _Py_isabs(const wchar_t *path)
 {
+#ifdef MS_WINDOWS
+    const wchar_t *tail;
+    HRESULT hr = PathCchSkipRoot(path, &tail);
+    if (FAILED(hr) || path == tail) {
+        return 0;
+    }
+    if (tail == &path[1] && (path[0] == SEP || path[0] == ALTSEP)) {
+        // Exclude paths with leading SEP
+        return 0;
+    }
+    if (tail == &path[2] && path[1] == L':') {
+        // Exclude drive-relative paths (e.g. C:filename.ext)
+        return 0;
+    }
+    return 1;
+#else
     return (path[0] == SEP);
-}
 #endif
+}
 
 
 /* Get an absolute path.
@@ -2017,6 +2032,22 @@ _Py_isabs(const wchar_t *path)
 int
 _Py_abspath(const wchar_t *path, wchar_t **abspath_p)
 {
+    if (path[0] == '\0' || !wcscmp(path, L".")) {
+        wchar_t cwd[MAXPATHLEN + 1];
+        cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0;
+        if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) {
+            /* unable to get the current directory */
+            return -1;
+        }
+        *abspath_p = _PyMem_RawWcsdup(cwd);
+        return 0;
+    }
+
+    if (_Py_isabs(path)) {
+        *abspath_p = _PyMem_RawWcsdup(path);
+        return 0;
+    }
+
 #ifdef MS_WINDOWS
     wchar_t woutbuf[MAX_PATH], *woutbufp = woutbuf;
     DWORD result;
@@ -2028,7 +2059,7 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p)
         return -1;
     }
 
-    if (result > Py_ARRAY_LENGTH(woutbuf)) {
+    if (result >= Py_ARRAY_LENGTH(woutbuf)) {
         if ((size_t)result <= (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) {
             woutbufp = PyMem_RawMalloc((size_t)result * sizeof(wchar_t));
         }
@@ -2055,11 +2086,6 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p)
     *abspath_p = _PyMem_RawWcsdup(woutbufp);
     return 0;
 #else
-    if (_Py_isabs(path)) {
-        *abspath_p = _PyMem_RawWcsdup(path);
-        return 0;
-    }
-
     wchar_t cwd[MAXPATHLEN + 1];
     cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0;
     if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) {
@@ -2102,7 +2128,8 @@ join_relfile(wchar_t *buffer, size_t bufsize,
              const wchar_t *dirname, const wchar_t *relfile)
 {
 #ifdef MS_WINDOWS
-    if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, 0))) {
+    if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, 
+        PATHCCH_ALLOW_LONG_PATHS | PATHCCH_FORCE_ENABLE_LONG_NAME_PROCESS))) {
         return -1;
     }
 #else
@@ -2180,99 +2207,125 @@ _Py_find_basename(const wchar_t *filename)
     return 0;
 }
 
-
-/* Remove navigation elements such as "." and "..".
-
-   This is mostly a C implementation of posixpath.normpath().
-   Return 0 on success.  Return -1 if "orig" is too big for the buffer. */
-int
-_Py_normalize_path(const wchar_t *path, wchar_t *buf, const size_t buf_len)
+/* In-place path normalisation. Returns the start of the normalized
+   path, which will be within the original buffer. Guaranteed to not
+   make the path longer, and will not fail. 'size' is the length of
+   the path, if known. If -1, the first null character will be assumed
+   to be the end of the path. */
+wchar_t *
+_Py_normpath(wchar_t *path, Py_ssize_t size)
 {
-    assert(path && *path != L'\0');
-    assert(*path == SEP);  // an absolute path
-    if (wcslen(path) + 1 >= buf_len) {
-        return -1;
+    if (!path[0] || size == 0) {
+        return path;
     }
+    wchar_t lastC = L'\0';
+    wchar_t *p1 = path;
+    wchar_t *pEnd = size >= 0 ? &path[size] : NULL;
+    wchar_t *p2 = path;
+    wchar_t *minP2 = path;
 
-    int dots = -1;
-    int check_leading = 1;
-    const wchar_t *buf_start = buf;
-    wchar_t *buf_next = buf;
-    // The resulting filename will never be longer than path.
-    for (const wchar_t *remainder = path; *remainder != L'\0'; remainder++) {
-        wchar_t c = *remainder;
-        buf_next[0] = c;
-        buf_next++;
-        if (c == SEP) {
-            assert(dots <= 2);
-            if (dots == 2) {
-                // Turn "/x/y/../z" into "/x/z".
-                buf_next -= 4;  // "/../"
-                assert(*buf_next == SEP);
-                // We cap it off at the root, so "/../spam" becomes "/spam".
-                if (buf_next == buf_start) {
-                    buf_next++;
-                }
-                else {
-                    // Move to the previous SEP in the buffer.
-                    while (*(buf_next - 1) != SEP) {
-                        assert(buf_next != buf_start);
-                        buf_next--;
-                    }
-                }
-            }
-            else if (dots == 1) {
-                // Turn "/./" into "/".
-                buf_next -= 2;  // "./"
-                assert(*(buf_next - 1) == SEP);
-            }
-            else if (dots == 0) {
-                // Turn "//" into "/".
-                buf_next--;
-                assert(*(buf_next - 1) == SEP);
-                if (check_leading) {
-                    if (buf_next - 1 == buf && *(remainder + 1) != SEP) {
-                        // Leave a leading "//" alone, unless "///...".
-                        buf_next++;
-                        buf_start++;
-                    }
-                    check_leading = 0;
-                }
-            }
-            dots = 0;
+#define IS_END(x) (pEnd ? (x) == pEnd : !*(x))
+#ifdef ALTSEP
+#define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP)
+#else
+#define IS_SEP(x) (*(x) == SEP)
+#endif
+#define SEP_OR_END(x) (IS_SEP(x) || IS_END(x))
+
+    // Skip leading '.\'
+    if (p1[0] == L'.' && IS_SEP(&p1[1])) {
+        path = &path[2];
+        while (IS_SEP(path) && !IS_END(path)) {
+            path++;
         }
-        else {
-            check_leading = 0;
-            if (dots >= 0) {
-                if (c == L'.' && dots < 2) {
-                    dots++;
-                }
-                else {
-                    dots = -1;
-                }
+        p1 = p2 = minP2 = path;
+        lastC = SEP;
+    }
+#ifdef MS_WINDOWS
+    // Skip past drive segment and update minP2
+    else if (p1[0] && p1[1] == L':') {
+        *p2++ = *p1++;
+        *p2++ = *p1++;
+        minP2 = p2;
+        lastC = L':';
+    }
+    // Skip past all \\-prefixed paths, including \\?\, \\.\,
+    // and network paths, including the first segment.
+    else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1])) {
+        int sepCount = 2;
+        *p2++ = SEP;
+        *p2++ = SEP;
+        p1 += 2;
+        for (; !IS_END(p1) && sepCount; ++p1) {
+            if (IS_SEP(p1)) {
+                --sepCount;
+                *p2++ = lastC = SEP;
+            } else {
+                *p2++ = lastC = *p1;
             }
         }
+        minP2 = p2;
     }
-    if (dots >= 0) {
-        // Strip any trailing dots and trailing slash.
-        buf_next -= dots + 1;  // "/" or "/." or "/.."
-        assert(*buf_next == SEP);
-        if (buf_next == buf_start) {
-            // Leave the leading slash for root.
-            buf_next++;
+#else
+    // Skip past two leading SEPs
+    else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1]) && !IS_SEP(&p1[2])) {
+        *p2++ = *p1++;
+        *p2++ = *p1++;
+        minP2 = p2;
+        lastC = SEP;
+    }
+#endif /* MS_WINDOWS */
+
+    /* if pEnd is specified, check that. Else, check for null terminator */
+    for (; !IS_END(p1); ++p1) {
+        wchar_t c = *p1;
+#ifdef ALTSEP
+        if (c == ALTSEP) {
+            c = SEP;
         }
-        else {
-            if (dots == 2) {
-                // Move to the previous SEP in the buffer.
-                do {
-                    assert(buf_next != buf_start);
-                    buf_next--;
-                } while (*(buf_next) != SEP);
+#endif
+        if (lastC == SEP) {
+            if (c == L'.') {
+                int sep_at_1 = SEP_OR_END(&p1[1]);
+                int sep_at_2 = !sep_at_1 && SEP_OR_END(&p1[2]);
+                if (sep_at_2 && p1[1] == L'.') {
+                    wchar_t *p3 = p2;
+                    while (p3 != minP2 && *--p3 == SEP) { }
+                    while (p3 != minP2 && *(p3 - 1) != SEP) { --p3; }
+                    if (p3[0] == L'.' && p3[1] == L'.' && IS_SEP(&p3[2])) {
+                        // Previous segment is also ../, so append instead
+                        *p2++ = L'.';
+                        *p2++ = L'.';
+                        lastC = L'.';
+                    } else if (p3[0] == SEP) {
+                        // Absolute path, so absorb segment
+                        p2 = p3 + 1;
+                    } else {
+                        p2 = p3;
+                    }
+                    p1 += 1;
+                } else if (sep_at_1) {
+                } else {
+                    *p2++ = lastC = c;
+                }
+            } else if (c == SEP) {
+            } else {
+                *p2++ = lastC = c;
             }
+        } else {
+            *p2++ = lastC = c;
+        } 
+    }
+    *p2 = L'\0';
+    if (p2 != minP2) {
+        while (--p2 != minP2 && *p2 == SEP) {
+            *p2 = L'\0';
         }
     }
-    *buf_next = L'\0';
-    return 0;
+#undef SEP_OR_END
+#undef IS_SEP
+#undef IS_END
+    return path;
 }
 
 
diff --git a/Python/initconfig.c b/Python/initconfig.c
index 53b624fdd72ec..47ebc64c8470a 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -22,11 +22,6 @@
 #  endif
 #endif
 
-#ifndef PLATLIBDIR
-#  error "PLATLIBDIR macro must be defined"
-#endif
-
-
 /* --- Command line options --------------------------------------- */
 
 /* Short usage message (with %s for argv0) */
@@ -632,7 +627,6 @@ config_check_consistency(const PyConfig *config)
     assert(config->parse_argv >= 0);
     assert(config->configure_c_stdio >= 0);
     assert(config->buffered_stdio >= 0);
-    assert(config->program_name != NULL);
     assert(_PyWideStringList_CheckConsistency(&config->orig_argv));
     assert(_PyWideStringList_CheckConsistency(&config->argv));
     /* sys.argv must be non-empty: empty argv is replaced with [''] */
@@ -641,7 +635,6 @@ config_check_consistency(const PyConfig *config)
     assert(_PyWideStringList_CheckConsistency(&config->warnoptions));
     assert(_PyWideStringList_CheckConsistency(&config->module_search_paths));
     assert(config->module_search_paths_set >= 0);
-    assert(config->platlibdir != NULL);
     assert(config->filesystem_encoding != NULL);
     assert(config->filesystem_errors != NULL);
     assert(config->stdio_encoding != NULL);
@@ -740,6 +733,7 @@ _PyConfig_InitCompatConfig(PyConfig *config)
     config->legacy_windows_stdio = -1;
 #endif
     config->use_frozen_modules = -1;
+    config->_is_python_build = 0;
     config->code_debug_ranges = 1;
 }
 
@@ -962,6 +956,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_ATTR(_isolated_interpreter);
     COPY_ATTR(use_frozen_modules);
     COPY_WSTRLIST(orig_argv);
+    COPY_ATTR(_is_python_build);
 
 #undef COPY_ATTR
 #undef COPY_WSTR_ATTR
@@ -1066,6 +1061,7 @@ _PyConfig_AsDict(const PyConfig *config)
     SET_ITEM_INT(_isolated_interpreter);
     SET_ITEM_WSTRLIST(orig_argv);
     SET_ITEM_INT(use_frozen_modules);
+    SET_ITEM_INT(_is_python_build);
 
     return dict;
 
@@ -1350,6 +1346,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
     GET_UINT(_init_main);
     GET_UINT(_isolated_interpreter);
     GET_UINT(use_frozen_modules);
+    GET_UINT(_is_python_build);
 
 #undef CHECK_VALUE
 #undef GET_UINT
@@ -1489,117 +1486,6 @@ config_set_global_vars(const PyConfig *config)
 }
 
 
-/* Get the program name: use PYTHONEXECUTABLE and __PYVENV_LAUNCHER__
-   environment variables on macOS if available. */
-static PyStatus
-config_init_program_name(PyConfig *config)
-{
-    PyStatus status;
-
-    /* If Py_SetProgramName() was called, use its value */
-    const wchar_t *program_name = _Py_path_config.program_name;
-    if (program_name != NULL) {
-        config->program_name = _PyMem_RawWcsdup(program_name);
-        if (config->program_name == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-        return _PyStatus_OK();
-    }
-
-#ifdef __APPLE__
-    /* On MacOS X, when the Python interpreter is embedded in an
-       application bundle, it gets executed by a bootstrapping script
-       that does os.execve() with an argv[0] that's different from the
-       actual Python executable. This is needed to keep the Finder happy,
-       or rather, to work around Apple's overly strict requirements of
-       the process name. However, we still need a usable sys.executable,
-       so the actual executable path is passed in an environment variable.
-       See Lib/plat-mac/bundlebuilder.py for details about the bootstrap
-       script. */
-    const char *p = config_get_env(config, "PYTHONEXECUTABLE");
-    if (p != NULL) {
-        status = CONFIG_SET_BYTES_STR(config, &config->program_name, p,
-                                      "PYTHONEXECUTABLE environment variable");
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-        return _PyStatus_OK();
-    }
-#ifdef WITH_NEXT_FRAMEWORK
-    else {
-        const char* pyvenv_launcher = getenv("__PYVENV_LAUNCHER__");
-        if (pyvenv_launcher && *pyvenv_launcher) {
-            /* Used by Mac/Tools/pythonw.c to forward
-             * the argv0 of the stub executable
-             */
-            status = CONFIG_SET_BYTES_STR(config,
-                                          &config->program_name,
-                                          pyvenv_launcher,
-                                          "__PYVENV_LAUNCHER__ environment variable");
-            if (_PyStatus_EXCEPTION(status)) {
-                return status;
-            }
-
-            /*
-             * This environment variable is used to communicate between
-             * the stub launcher and the real interpreter and isn't needed
-             * beyond this point.
-             *
-             * Clean up to avoid problems when launching other programs
-             * later on.
-             */
-            (void)unsetenv("__PYVENV_LAUNCHER__");
-
-            return _PyStatus_OK();
-        }
-    }
-#endif   /* WITH_NEXT_FRAMEWORK */
-#endif   /* __APPLE__ */
-
-    /* Use argv[0] if available and non-empty */
-    const PyWideStringList *argv = &config->argv;
-    if (argv->length >= 1 && argv->items[0][0] != L'\0') {
-        config->program_name = _PyMem_RawWcsdup(argv->items[0]);
-        if (config->program_name == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-        return _PyStatus_OK();
-    }
-
-    /* Last fall back: hardcoded name */
-#ifdef MS_WINDOWS
-    const wchar_t *default_program_name = L"python";
-#else
-    const wchar_t *default_program_name = L"python3";
-#endif
-    status = PyConfig_SetString(config, &config->program_name,
-                                default_program_name);
-    if (_PyStatus_EXCEPTION(status)) {
-        return status;
-    }
-    return _PyStatus_OK();
-}
-
-static PyStatus
-config_init_executable(PyConfig *config)
-{
-    assert(config->executable == NULL);
-
-    /* If Py_SetProgramFullPath() was called, use its value */
-    const wchar_t *program_full_path = _Py_path_config.program_full_path;
-    if (program_full_path != NULL) {
-        PyStatus status = PyConfig_SetString(config,
-                                             &config->executable,
-                                             program_full_path);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-        return _PyStatus_OK();
-    }
-    return _PyStatus_OK();
-}
-
-
 static const wchar_t*
 config_get_xoption(const PyConfig *config, wchar_t *name)
 {
@@ -1618,25 +1504,6 @@ config_get_xoption_value(const PyConfig *config, wchar_t *name)
 }
 
 
-static PyStatus
-config_init_home(PyConfig *config)
-{
-    assert(config->home == NULL);
-
-    /* If Py_SetPythonHome() was called, use its value */
-    wchar_t *home = _Py_path_config.home;
-    if (home) {
-        PyStatus status = PyConfig_SetString(config, &config->home, home);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-        return _PyStatus_OK();
-    }
-
-    return CONFIG_GET_ENV_DUP(config, &config->home,
-                              L"PYTHONHOME", "PYTHONHOME");
-}
-
 static PyStatus
 config_init_hash_seed(PyConfig *config)
 {
@@ -2092,44 +1959,6 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig)
 }
 
 
-/* Determine if the current build is a "development" build (e.g. running
-   out of the source tree) or not.
-
-   A return value of -1 indicates that we do not know.
-  */
-static int
-is_dev_env(PyConfig *config)
-{
-    // This should only ever get called early in runtime initialization,
-    // before the global path config is written.  Otherwise we would
-    // use Py_GetProgramFullPath() and _Py_GetStdlibDir().
-    assert(config != NULL);
-
-    const wchar_t *executable = config->executable;
-    const wchar_t *stdlib = config->stdlib_dir;
-    if (executable == NULL || *executable == L'\0' ||
-            stdlib == NULL || *stdlib == L'\0') {
-        // _PyPathConfig_Calculate() hasn't run yet.
-        return -1;
-    }
-    size_t len = _Py_find_basename(executable);
-    if (wcscmp(executable + len, L"python") != 0 &&
-            wcscmp(executable + len, L"python.exe") != 0) {
-        return 0;
-    }
-    /* If dirname() is the same for both then it is a dev build. */
-    if (len != _Py_find_basename(stdlib)) {
-        return 0;
-    }
-    // We do not bother normalizing the two filenames first since
-    // for config_init_import() is does the right thing as-is.
-    if (wcsncmp(stdlib, executable, len) != 0) {
-        return 0;
-    }
-    return 1;
-}
-
-
 static PyStatus
 config_init_import(PyConfig *config, int compute_path_config)
 {
@@ -2144,10 +1973,7 @@ config_init_import(PyConfig *config, int compute_path_config)
     if (config->use_frozen_modules < 0) {
         const wchar_t *value = config_get_xoption_value(config, L"frozen_modules");
         if (value == NULL) {
-            int isdev = is_dev_env(config);
-            if (isdev >= 0) {
-                config->use_frozen_modules = !isdev;
-            }
+            config->use_frozen_modules = !config->_is_python_build;
         }
         else if (wcscmp(value, L"on") == 0) {
             config->use_frozen_modules = 1;
@@ -2246,28 +2072,6 @@ config_read(PyConfig *config, int compute_path_config)
         return status;
     }
 
-    if (config->home == NULL) {
-        status = config_init_home(config);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-
-    if (config->executable == NULL) {
-        status = config_init_executable(config);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-
-    if(config->platlibdir == NULL) {
-        status = CONFIG_SET_BYTES_STR(config, &config->platlibdir, PLATLIBDIR,
-                                      "PLATLIBDIR macro");
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-
     if (config->_install_importlib) {
         status = config_init_import(config, compute_path_config);
         if (_PyStatus_EXCEPTION(status)) {
@@ -2890,13 +2694,6 @@ config_read_cmdline(PyConfig *config)
         config->parse_argv = 1;
     }
 
-    if (config->program_name == NULL) {
-        status = config_init_program_name(config);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-    }
-
     if (config->parse_argv == 1) {
         Py_ssize_t opt_index;
         status = config_parse_cmdline(config, &cmdline_warnoptions, &opt_index);
@@ -3076,7 +2873,7 @@ _PyConfig_Read(PyConfig *config, int compute_path_config)
 PyStatus
 PyConfig_Read(PyConfig *config)
 {
-    return _PyConfig_Read(config, 1);
+    return _PyConfig_Read(config, 0);
 }
 
 
@@ -3124,16 +2921,6 @@ _Py_GetConfigsAsDict(void)
     }
     Py_CLEAR(dict);
 
-    /* path config */
-    dict = _PyPathConfig_AsDict();
-    if (dict == NULL) {
-        goto error;
-    }
-    if (PyDict_SetItemString(result, "path_config", dict) < 0) {
-        goto error;
-    }
-    Py_CLEAR(dict);
-
     return result;
 
 error:
@@ -3199,6 +2986,7 @@ _Py_DumpPathConfig(PyThreadState *tstate)
     PySys_WriteStderr("  environment = %i\n", config->use_environment);
     PySys_WriteStderr("  user site = %i\n", config->user_site_directory);
     PySys_WriteStderr("  import site = %i\n", config->site_import);
+    PySys_WriteStderr("  is in build tree = %i\n", config->_is_python_build);
     DUMP_CONFIG("stdlib dir", stdlib_dir);
 #undef DUMP_CONFIG
 
diff --git a/Python/pathconfig.c b/Python/pathconfig.c
index ad22222e000fc..4271928571fa1 100644
--- a/Python/pathconfig.c
+++ b/Python/pathconfig.c
@@ -1,6 +1,7 @@
 /* Path configuration like module_search_path (sys.path) */
 
 #include "Python.h"
+#include "marshal.h"              // PyMarshal_ReadObjectFromString
 #include "osdefs.h"               // DELIM
 #include "pycore_initconfig.h"
 #include "pycore_fileutils.h"
@@ -9,6 +10,8 @@
 #include <wchar.h>
 #ifdef MS_WINDOWS
 #  include <windows.h>            // GetFullPathNameW(), MAX_PATH
+#  include <pathcch.h>
+#  include <shlwapi.h>
 #endif
 
 #ifdef __cplusplus
@@ -16,462 +19,170 @@ extern "C" {
 #endif
 
 
+/* External interface */
+
+/* Stored values set by C API functions */
+typedef struct _PyPathConfig {
+    /* Full path to the Python program */
+    wchar_t *program_full_path;
+    wchar_t *prefix;
+    wchar_t *exec_prefix;
+    wchar_t *stdlib_dir;
+    /* Set by Py_SetPath */
+    wchar_t *module_search_path;
+    /* Set by _PyPathConfig_UpdateGlobal */
+    wchar_t *calculated_module_search_path;
+    /* Python program name */
+    wchar_t *program_name;
+    /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */
+    wchar_t *home;
+} _PyPathConfig;
+
+#  define _PyPathConfig_INIT \
+      {.module_search_path = NULL}
+
+
 _PyPathConfig _Py_path_config = _PyPathConfig_INIT;
 
 
-static int
-copy_wstr(wchar_t **dst, const wchar_t *src)
+const wchar_t *
+_PyPathConfig_GetGlobalModuleSearchPath(void)
 {
-    assert(*dst == NULL);
-    if (src != NULL) {
-        *dst = _PyMem_RawWcsdup(src);
-        if (*dst == NULL) {
-            return -1;
-        }
-    }
-    else {
-        *dst = NULL;
-    }
-    return 0;
+    return _Py_path_config.module_search_path;
 }
 
 
-static void
-pathconfig_clear(_PyPathConfig *config)
+void
+_PyPathConfig_ClearGlobal(void)
 {
-    /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator,
-       since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be
-       called before Py_Initialize() which can changes the memory allocator. */
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
 #define CLEAR(ATTR) \
     do { \
-        PyMem_RawFree(ATTR); \
-        ATTR = NULL; \
+        PyMem_RawFree(_Py_path_config.ATTR); \
+        _Py_path_config.ATTR = NULL; \
     } while (0)
 
-    CLEAR(config->program_full_path);
-    CLEAR(config->prefix);
-    CLEAR(config->exec_prefix);
-    CLEAR(config->stdlib_dir);
-    CLEAR(config->module_search_path);
-    CLEAR(config->program_name);
-    CLEAR(config->home);
-#ifdef MS_WINDOWS
-    CLEAR(config->base_executable);
-#endif
+    CLEAR(program_full_path);
+    CLEAR(prefix);
+    CLEAR(exec_prefix);
+    CLEAR(stdlib_dir);
+    CLEAR(module_search_path);
+    CLEAR(calculated_module_search_path);
+    CLEAR(program_name);
+    CLEAR(home);
 
 #undef CLEAR
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 }
 
-
-static PyStatus
-pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2)
+PyStatus
+_PyPathConfig_ReadGlobal(PyConfig *config)
 {
-    pathconfig_clear(config);
+    PyStatus status = _PyStatus_OK();
 
-#define COPY_ATTR(ATTR) \
+#define COPY(ATTR) \
     do { \
-        if (copy_wstr(&config->ATTR, config2->ATTR) < 0) { \
-            return _PyStatus_NO_MEMORY(); \
+        if (_Py_path_config.ATTR && !config->ATTR) { \
+            status = PyConfig_SetString(config, &config->ATTR, _Py_path_config.ATTR); \
+            if (_PyStatus_EXCEPTION(status)) goto done; \
         } \
     } while (0)
 
-    COPY_ATTR(program_full_path);
-    COPY_ATTR(prefix);
-    COPY_ATTR(exec_prefix);
-    COPY_ATTR(module_search_path);
-    COPY_ATTR(stdlib_dir);
-    COPY_ATTR(program_name);
-    COPY_ATTR(home);
-#ifdef MS_WINDOWS
-    config->isolated = config2->isolated;
-    config->site_import = config2->site_import;
-    COPY_ATTR(base_executable);
-#endif
-
-#undef COPY_ATTR
-
-    return _PyStatus_OK();
-}
-
-
-void
-_PyPathConfig_ClearGlobal(void)
-{
-    PyMemAllocatorEx old_alloc;
-    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-
-    pathconfig_clear(&_Py_path_config);
-
-    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-}
-
-
-static wchar_t*
-_PyWideStringList_Join(const PyWideStringList *list, wchar_t sep)
-{
-    size_t len = 1;   /* NUL terminator */
-    for (Py_ssize_t i=0; i < list->length; i++) {
-        if (i != 0) {
-            len++;
-        }
-        len += wcslen(list->items[i]);
-    }
-
-    wchar_t *text = PyMem_RawMalloc(len * sizeof(wchar_t));
-    if (text == NULL) {
-        return NULL;
-    }
-    wchar_t *str = text;
-    for (Py_ssize_t i=0; i < list->length; i++) {
-        wchar_t *path = list->items[i];
-        if (i != 0) {
-            *str++ = sep;
-        }
-        len = wcslen(path);
-        memcpy(str, path, len * sizeof(wchar_t));
-        str += len;
-    }
-    *str = L'\0';
-
-    return text;
-}
-
-
-static PyStatus
-pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
-{
-    PyStatus status;
-    PyMemAllocatorEx old_alloc;
-    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-
-    if (config->module_search_paths_set) {
-        PyMem_RawFree(pathconfig->module_search_path);
-        pathconfig->module_search_path = _PyWideStringList_Join(&config->module_search_paths, DELIM);
-        if (pathconfig->module_search_path == NULL) {
-            goto no_memory;
-        }
-    }
-
-#define COPY_CONFIG(PATH_ATTR, CONFIG_ATTR) \
-        if (config->CONFIG_ATTR) { \
-            PyMem_RawFree(pathconfig->PATH_ATTR); \
-            pathconfig->PATH_ATTR = NULL; \
-            if (copy_wstr(&pathconfig->PATH_ATTR, config->CONFIG_ATTR) < 0) { \
-                goto no_memory; \
-            } \
-        }
-
-    COPY_CONFIG(program_full_path, executable);
-    COPY_CONFIG(prefix, prefix);
-    COPY_CONFIG(exec_prefix, exec_prefix);
-    COPY_CONFIG(stdlib_dir, stdlib_dir);
-    COPY_CONFIG(program_name, program_name);
-    COPY_CONFIG(home, home);
-#ifdef MS_WINDOWS
-    COPY_CONFIG(base_executable, base_executable);
-#endif
-
-#undef COPY_CONFIG
-
-    status = _PyStatus_OK();
-    goto done;
+#define COPY2(ATTR, SRCATTR) \
+    do { \
+        if (_Py_path_config.SRCATTR && !config->ATTR) { \
+            status = PyConfig_SetString(config, &config->ATTR, _Py_path_config.SRCATTR); \
+            if (_PyStatus_EXCEPTION(status)) goto done; \
+        } \
+    } while (0)
 
-no_memory:
-    status = _PyStatus_NO_MEMORY();
+    COPY(prefix);
+    COPY(exec_prefix);
+    COPY(stdlib_dir);
+    COPY(program_name);
+    COPY(home);
+    COPY2(executable, program_full_path);
+    // module_search_path must be initialised - not read
+#undef COPY
+#undef COPY2
 
 done:
-    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
     return status;
 }
 
-PyObject *
-_PyPathConfig_AsDict(void)
-{
-    PyObject *dict = PyDict_New();
-    if (dict == NULL) {
-        return NULL;
-    }
-
-#define SET_ITEM(KEY, EXPR) \
-        do { \
-            PyObject *obj = (EXPR); \
-            if (obj == NULL) { \
-                goto fail; \
-            } \
-            int res = PyDict_SetItemString(dict, KEY, obj); \
-            Py_DECREF(obj); \
-            if (res < 0) { \
-                goto fail; \
-            } \
-        } while (0)
-#define SET_ITEM_STR(KEY) \
-        SET_ITEM(#KEY, \
-            (_Py_path_config.KEY \
-             ? PyUnicode_FromWideChar(_Py_path_config.KEY, -1) \
-             : (Py_INCREF(Py_None), Py_None)))
-#define SET_ITEM_INT(KEY) \
-        SET_ITEM(#KEY, PyLong_FromLong(_Py_path_config.KEY))
-
-    SET_ITEM_STR(program_full_path);
-    SET_ITEM_STR(prefix);
-    SET_ITEM_STR(exec_prefix);
-    SET_ITEM_STR(module_search_path);
-    SET_ITEM_STR(stdlib_dir);
-    SET_ITEM_STR(program_name);
-    SET_ITEM_STR(home);
-#ifdef MS_WINDOWS
-    SET_ITEM_INT(isolated);
-    SET_ITEM_INT(site_import);
-    SET_ITEM_STR(base_executable);
-
-    {
-        wchar_t py3path[MAX_PATH];
-        HMODULE hPython3 = GetModuleHandleW(PY3_DLLNAME);
-        PyObject *obj;
-        if (hPython3
-            && GetModuleFileNameW(hPython3, py3path, Py_ARRAY_LENGTH(py3path)))
-        {
-            obj = PyUnicode_FromWideChar(py3path, -1);
-            if (obj == NULL) {
-                goto fail;
-            }
-        }
-        else {
-            obj = Py_None;
-            Py_INCREF(obj);
-        }
-        if (PyDict_SetItemString(dict, "python3_dll", obj) < 0) {
-            Py_DECREF(obj);
-            goto fail;
-        }
-        Py_DECREF(obj);
-    }
-#endif
-
-#undef SET_ITEM
-#undef SET_ITEM_STR
-#undef SET_ITEM_INT
-
-    return dict;
-
-fail:
-    Py_DECREF(dict);
-    return NULL;
-}
-
-
 PyStatus
-_PyConfig_WritePathConfig(const PyConfig *config)
+_PyPathConfig_UpdateGlobal(const PyConfig *config)
 {
-    return pathconfig_set_from_config(&_Py_path_config, config);
-}
-
-
-static PyStatus
-config_init_module_search_paths(PyConfig *config, _PyPathConfig *pathconfig)
-{
-    assert(!config->module_search_paths_set);
-
-    _PyWideStringList_Clear(&config->module_search_paths);
-
-    const wchar_t *sys_path = pathconfig->module_search_path;
-    const wchar_t delim = DELIM;
-    while (1) {
-        const wchar_t *p = wcschr(sys_path, delim);
-        if (p == NULL) {
-            p = sys_path + wcslen(sys_path); /* End of string */
-        }
-
-        size_t path_len = (p - sys_path);
-        wchar_t *path = PyMem_RawMalloc((path_len + 1) * sizeof(wchar_t));
-        if (path == NULL) {
-            return _PyStatus_NO_MEMORY();
-        }
-        memcpy(path, sys_path, path_len * sizeof(wchar_t));
-        path[path_len] = L'\0';
-
-        PyStatus status = PyWideStringList_Append(&config->module_search_paths, path);
-        PyMem_RawFree(path);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
-
-        if (*p == '\0') {
-            break;
-        }
-        sys_path = p + 1;
-    }
-    config->module_search_paths_set = 1;
-    return _PyStatus_OK();
-}
-
-
-/* Calculate the path configuration:
-
-   - exec_prefix
-   - module_search_path
-   - stdlib_dir
-   - prefix
-   - program_full_path
-
-   On Windows, more fields are calculated:
-
-   - base_executable
-   - isolated
-   - site_import
-
-   On other platforms, isolated and site_import are left unchanged, and
-   _PyConfig_InitPathConfig() copies executable to base_executable (if it's not
-   set).
-
-   Priority, highest to lowest:
-
-   - PyConfig
-   - _Py_path_config: set by Py_SetPath(), Py_SetPythonHome()
-     and Py_SetProgramName()
-   - _PyPathConfig_Calculate()
-*/
-static PyStatus
-pathconfig_init(_PyPathConfig *pathconfig, const PyConfig *config,
-                int compute_path_config)
-{
-    PyStatus status;
-
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    status = pathconfig_copy(pathconfig, &_Py_path_config);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto done;
-    }
-
-    status = pathconfig_set_from_config(pathconfig, config);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto done;
-    }
-
-    if (compute_path_config) {
-        status = _PyPathConfig_Calculate(pathconfig, config);
-    }
-
-done:
-    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-    return status;
-}
-
-
-static PyStatus
-config_init_pathconfig(PyConfig *config, int compute_path_config)
-{
-    _PyPathConfig pathconfig = _PyPathConfig_INIT;
-    PyStatus status;
-
-    status = pathconfig_init(&pathconfig, config, compute_path_config);
-    if (_PyStatus_EXCEPTION(status)) {
-        goto done;
-    }
-
-    if (!config->module_search_paths_set
-        && pathconfig.module_search_path != NULL)
-    {
-        status = config_init_module_search_paths(config, &pathconfig);
-        if (_PyStatus_EXCEPTION(status)) {
-            goto done;
-        }
-    }
-
-#define COPY_ATTR(PATH_ATTR, CONFIG_ATTR) \
-        if (config->CONFIG_ATTR == NULL && pathconfig.PATH_ATTR != NULL) { \
-            if (copy_wstr(&config->CONFIG_ATTR, pathconfig.PATH_ATTR) < 0) { \
-                goto no_memory; \
-            } \
-        }
-
-#ifdef MS_WINDOWS
-    if (config->executable != NULL && config->base_executable == NULL) {
-        /* If executable is set explicitly in the configuration,
-           ignore calculated base_executable: _PyConfig_InitPathConfig()
-           will copy executable to base_executable */
-    }
-    else {
-        COPY_ATTR(base_executable, base_executable);
-    }
-#endif
-
-    COPY_ATTR(program_full_path, executable);
-    COPY_ATTR(prefix, prefix);
-    COPY_ATTR(exec_prefix, exec_prefix);
-    COPY_ATTR(stdlib_dir, stdlib_dir);
-
-#undef COPY_ATTR
-
-#ifdef MS_WINDOWS
-    /* If a ._pth file is found: isolated and site_import are overridden */
-    if (pathconfig.isolated != -1) {
-        config->isolated = pathconfig.isolated;
-    }
-    if (pathconfig.site_import != -1) {
-        config->site_import = pathconfig.site_import;
-    }
-#endif
-
-    status = _PyStatus_OK();
-    goto done;
-
-no_memory:
-    status = _PyStatus_NO_MEMORY();
+#define COPY(ATTR) \
+    do { \
+        if (config->ATTR) { \
+            PyMem_RawFree(_Py_path_config.ATTR); \
+            _Py_path_config.ATTR = _PyMem_RawWcsdup(config->ATTR); \
+            if (!_Py_path_config.ATTR) goto error; \
+        } \
+    } while (0)
 
-done:
-    pathconfig_clear(&pathconfig);
-    return status;
-}
+#define COPY2(ATTR, SRCATTR) \
+    do { \
+        if (config->SRCATTR) { \
+            PyMem_RawFree(_Py_path_config.ATTR); \
+            _Py_path_config.ATTR = _PyMem_RawWcsdup(config->SRCATTR); \
+            if (!_Py_path_config.ATTR) goto error; \
+        } \
+    } while (0)
 
+    COPY(prefix);
+    COPY(exec_prefix);
+    COPY(stdlib_dir);
+    COPY(program_name);
+    COPY(home);
+    COPY2(program_full_path, executable);
+#undef COPY
+#undef COPY2
 
-PyStatus
-_PyConfig_InitPathConfig(PyConfig *config, int compute_path_config)
-{
-    /* Do we need to calculate the path? */
-    if (!config->module_search_paths_set
-        || config->executable == NULL
-        || config->prefix == NULL
-        || config->exec_prefix == NULL)
-    {
-        PyStatus status = config_init_pathconfig(config, compute_path_config);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
+    PyMem_RawFree(_Py_path_config.module_search_path);
+    _Py_path_config.module_search_path = NULL;
+    PyMem_RawFree(_Py_path_config.calculated_module_search_path);
+    _Py_path_config.calculated_module_search_path = NULL;
+
+    do {
+        size_t cch = 1;
+        for (Py_ssize_t i = 0; i < config->module_search_paths.length; ++i) {
+            cch += 1 + wcslen(config->module_search_paths.items[i]);
         }
-    }
 
-    if (config->base_prefix == NULL && config->prefix != NULL) {
-        if (copy_wstr(&config->base_prefix, config->prefix) < 0) {
-            return _PyStatus_NO_MEMORY();
+        wchar_t *path = (wchar_t*)PyMem_RawMalloc(sizeof(wchar_t) * cch);
+        if (!path) {
+            goto error;
         }
-    }
-
-    if (config->base_exec_prefix == NULL && config->exec_prefix != NULL) {
-        if (copy_wstr(&config->base_exec_prefix,
-                      config->exec_prefix) < 0) {
-            return _PyStatus_NO_MEMORY();
+        wchar_t *p = path;
+        for (Py_ssize_t i = 0; i < config->module_search_paths.length; ++i) {
+            wcscpy(p, config->module_search_paths.items[i]);
+            p = wcschr(p, L'\0');
+            *p++ = DELIM;
+            *p = L'\0';
         }
-    }
 
-    if (config->base_executable == NULL && config->executable != NULL) {
-        if (copy_wstr(&config->base_executable,
-                      config->executable) < 0) {
-            return _PyStatus_NO_MEMORY();
-        }
-    }
+        do {
+            *p = L'\0';
+        } while (p != path && *--p == DELIM);
+        _Py_path_config.calculated_module_search_path = path;
+    } while (0);
 
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
     return _PyStatus_OK();
-}
 
+error:
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+    return _PyStatus_NO_MEMORY();
+}
 
-/* External interface */
 
 static void _Py_NO_RETURN
 path_out_of_memory(const char *func)
@@ -483,7 +194,7 @@ void
 Py_SetPath(const wchar_t *path)
 {
     if (path == NULL) {
-        pathconfig_clear(&_Py_path_config);
+        _PyPathConfig_ClearGlobal();
         return;
     }
 
@@ -494,6 +205,7 @@ Py_SetPath(const wchar_t *path)
     PyMem_RawFree(_Py_path_config.exec_prefix);
     PyMem_RawFree(_Py_path_config.stdlib_dir);
     PyMem_RawFree(_Py_path_config.module_search_path);
+    PyMem_RawFree(_Py_path_config.calculated_module_search_path);
 
     _Py_path_config.prefix = _PyMem_RawWcsdup(L"");
     _Py_path_config.exec_prefix = _PyMem_RawWcsdup(L"");
@@ -505,6 +217,7 @@ Py_SetPath(const wchar_t *path)
         _Py_path_config.stdlib_dir = _PyMem_RawWcsdup(L"");
     }
     _Py_path_config.module_search_path = _PyMem_RawWcsdup(path);
+    _Py_path_config.calculated_module_search_path = NULL;
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
@@ -521,19 +234,19 @@ Py_SetPath(const wchar_t *path)
 void
 Py_SetPythonHome(const wchar_t *home)
 {
-    if (home == NULL) {
-        return;
-    }
+    int has_value = home && home[0];
 
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
     PyMem_RawFree(_Py_path_config.home);
-    _Py_path_config.home = _PyMem_RawWcsdup(home);
+    if (has_value) {
+        _Py_path_config.home = _PyMem_RawWcsdup(home);
+    }
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    if (_Py_path_config.home == NULL) {
+    if (has_value && _Py_path_config.home == NULL) {
         path_out_of_memory(__func__);
     }
 }
@@ -542,19 +255,19 @@ Py_SetPythonHome(const wchar_t *home)
 void
 Py_SetProgramName(const wchar_t *program_name)
 {
-    if (program_name == NULL || program_name[0] == L'\0') {
-        return;
-    }
+    int has_value = program_name && program_name[0];
 
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
     PyMem_RawFree(_Py_path_config.program_name);
-    _Py_path_config.program_name = _PyMem_RawWcsdup(program_name);
+    if (has_value) {
+        _Py_path_config.program_name = _PyMem_RawWcsdup(program_name);
+    }
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    if (_Py_path_config.program_name == NULL) {
+    if (has_value && _Py_path_config.program_name == NULL) {
         path_out_of_memory(__func__);
     }
 }
@@ -562,19 +275,19 @@ Py_SetProgramName(const wchar_t *program_name)
 void
 _Py_SetProgramFullPath(const wchar_t *program_full_path)
 {
-    if (program_full_path == NULL || program_full_path[0] == L'\0') {
-        return;
-    }
+    int has_value = program_full_path && program_full_path[0];
 
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
     PyMem_RawFree(_Py_path_config.program_full_path);
-    _Py_path_config.program_full_path = _PyMem_RawWcsdup(program_full_path);
+    if (has_value) {
+        _Py_path_config.program_full_path = _PyMem_RawWcsdup(program_full_path);
+    }
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    if (_Py_path_config.program_full_path == NULL) {
+    if (has_value && _Py_path_config.program_full_path == NULL) {
         path_out_of_memory(__func__);
     }
 }
@@ -583,7 +296,12 @@ _Py_SetProgramFullPath(const wchar_t *program_full_path)
 wchar_t *
 Py_GetPath(void)
 {
-    return _Py_path_config.module_search_path;
+    /* If the user has provided a path, return that */
+    if (_Py_path_config.module_search_path) {
+        return _Py_path_config.module_search_path;
+    }
+    /* If we have already done calculations, return the calculated path */
+    return _Py_path_config.calculated_module_search_path;
 }
 
 
@@ -632,6 +350,8 @@ Py_GetProgramName(void)
     return _Py_path_config.program_name;
 }
 
+
+
 /* Compute module search path from argv[0] or the current working
    directory ("-m module" case) which will be prepended to sys.argv:
    sys.path[0].
@@ -772,73 +492,6 @@ _PyPathConfig_ComputeSysPath0(const PyWideStringList *argv, PyObject **path0_p)
 }
 
 
-#ifdef MS_WINDOWS
-#define WCSTOK wcstok_s
-#else
-#define WCSTOK wcstok
-#endif
-
-/* Search for a prefix value in an environment file (pyvenv.cfg).
-
-   - If found, copy it into *value_p: string which must be freed by
-     PyMem_RawFree().
-   - If not found, *value_p is set to NULL.
-*/
-PyStatus
-_Py_FindEnvConfigValue(FILE *env_file, const wchar_t *key,
-                       wchar_t **value_p)
-{
-    *value_p = NULL;
-
-    char buffer[MAXPATHLEN * 2 + 1];  /* allow extra for key, '=', etc. */
-    buffer[Py_ARRAY_LENGTH(buffer)-1] = '\0';
-
-    while (!feof(env_file)) {
-        char * p = fgets(buffer, Py_ARRAY_LENGTH(buffer) - 1, env_file);
-
-        if (p == NULL) {
-            break;
-        }
-
-        size_t n = strlen(p);
-        if (p[n - 1] != '\n') {
-            /* line has overflowed - bail */
-            break;
-        }
-        if (p[0] == '#') {
-            /* Comment - skip */
-            continue;
-        }
-
-        wchar_t *tmpbuffer = _Py_DecodeUTF8_surrogateescape(buffer, n, NULL);
-        if (tmpbuffer) {
-            wchar_t * state;
-            wchar_t * tok = WCSTOK(tmpbuffer, L" \t\r\n", &state);
-            if ((tok != NULL) && !wcscmp(tok, key)) {
-                tok = WCSTOK(NULL, L" \t", &state);
-                if ((tok != NULL) && !wcscmp(tok, L"=")) {
-                    tok = WCSTOK(NULL, L"\r\n", &state);
-                    if (tok != NULL) {
-                        *value_p = _PyMem_RawWcsdup(tok);
-                        PyMem_RawFree(tmpbuffer);
-
-                        if (*value_p == NULL) {
-                            return _PyStatus_NO_MEMORY();
-                        }
-
-                        /* found */
-                        return _PyStatus_OK();
-                    }
-                }
-            }
-            PyMem_RawFree(tmpbuffer);
-        }
-    }
-
-    /* not found */
-    return _PyStatus_OK();
-}
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index c6b2a1eb96148..84b76ea3275a7 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -459,7 +459,7 @@ interpreter_update_config(PyThreadState *tstate, int only_update_path_config)
     }
 
     if (_Py_IsMainInterpreter(tstate->interp)) {
-        PyStatus status = _PyConfig_WritePathConfig(config);
+        PyStatus status = _PyPathConfig_UpdateGlobal(config);
         if (_PyStatus_EXCEPTION(status)) {
             _PyErr_SetFromPyStatus(status);
             return -1;
@@ -488,7 +488,7 @@ _PyInterpreterState_SetConfig(const PyConfig *src_config)
         goto done;
     }
 
-    status = PyConfig_Read(&config);
+    status = _PyConfig_Read(&config, 1);
     if (_PyStatus_EXCEPTION(status)) {
         _PyErr_SetFromPyStatus(status);
         goto done;
@@ -549,7 +549,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime,
     config = _PyInterpreterState_GetConfig(interp);
 
     if (config->_install_importlib) {
-        status = _PyConfig_WritePathConfig(config);
+        status = _PyPathConfig_UpdateGlobal(config);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }


From webhook-mailer at python.org  Fri Dec  3 06:29:17 2021
From: webhook-mailer at python.org (markshannon)
Date: Fri, 03 Dec 2021 11:29:17 -0000
Subject: [Python-checkins] bpo-45885: Specialize COMPARE_OP (GH-29734)
Message-ID: <mailman.23.1638530958.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/03768c4d139df46212a091ed931aad03bec18b57
commit: 03768c4d139df46212a091ed931aad03bec18b57
branch: main
author: Dennis Sweeney <36520290+sweeneyde at users.noreply.github.com>
committer: markshannon <mark at hotpy.org>
date: 2021-12-03T11:29:12Z
summary:

bpo-45885: Specialize COMPARE_OP (GH-29734)

* Add COMPARE_OP_ADAPTIVE adaptive instruction.

* Add COMPARE_OP_FLOAT_JUMP, COMPARE_OP_INT_JUMP and COMPARE_OP_STR_JUMP specialized instructions.

* Introduce and use _PyUnicode_Equal

files:
A Misc/NEWS.d/next/Core and Builtins/2021-11-23-21-01-56.bpo-45885.3IxeCX.rst
M Include/cpython/unicodeobject.h
M Include/internal/pycore_code.h
M Include/opcode.h
M Lib/opcode.py
M Objects/unicodeobject.c
M Python/ceval.c
M Python/opcode_targets.h
M Python/specialize.c

diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h
index ab4aebf5e70b9..e02137c7cad7d 100644
--- a/Include/cpython/unicodeobject.h
+++ b/Include/cpython/unicodeobject.h
@@ -1016,6 +1016,9 @@ PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*);
    and where the hash values are equal (i.e. a very probable match) */
 PyAPI_FUNC(int) _PyUnicode_EQ(PyObject *, PyObject *);
 
+/* Equality check. Returns -1 on failure. */
+PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *);
+
 PyAPI_FUNC(int) _PyUnicode_WideCharString_Converter(PyObject *, void *);
 PyAPI_FUNC(int) _PyUnicode_WideCharString_Opt_Converter(PyObject *, void *);
 
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index d4d1392d05bde..496d52f580f1f 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -42,6 +42,7 @@ typedef struct {
     uint16_t defaults_len;
 } _PyCallCache;
 
+
 /* Add specialized versions of entries to this union.
  *
  * Do not break the invariant: sizeof(SpecializedCacheEntry) == 8
@@ -273,6 +274,7 @@ int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT
 int _Py_Specialize_CallFunction(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SpecializedCacheEntry *cache, PyObject *builtins);
 void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
                              SpecializedCacheEntry *cache);
+void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache);
 
 #define PRINT_SPECIALIZATION_STATS 0
 #define PRINT_SPECIALIZATION_STATS_DETAILED 0
diff --git a/Include/opcode.h b/Include/opcode.h
index 2c1a212cbd634..f22f7e94f6190 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -121,43 +121,47 @@ extern "C" {
 #define BINARY_OP_MULTIPLY_FLOAT         18
 #define BINARY_OP_SUBTRACT_INT           19
 #define BINARY_OP_SUBTRACT_FLOAT         20
-#define BINARY_SUBSCR_ADAPTIVE           21
-#define BINARY_SUBSCR_GETITEM            22
-#define BINARY_SUBSCR_LIST_INT           23
-#define BINARY_SUBSCR_TUPLE_INT          24
-#define BINARY_SUBSCR_DICT               26
-#define STORE_SUBSCR_ADAPTIVE            27
-#define STORE_SUBSCR_LIST_INT            28
-#define STORE_SUBSCR_DICT                29
-#define CALL_FUNCTION_ADAPTIVE           34
-#define CALL_FUNCTION_BUILTIN_O          36
-#define CALL_FUNCTION_BUILTIN_FAST       38
-#define CALL_FUNCTION_LEN                39
-#define CALL_FUNCTION_ISINSTANCE         40
-#define CALL_FUNCTION_PY_SIMPLE          41
-#define JUMP_ABSOLUTE_QUICK              42
-#define LOAD_ATTR_ADAPTIVE               43
-#define LOAD_ATTR_INSTANCE_VALUE         44
-#define LOAD_ATTR_WITH_HINT              45
-#define LOAD_ATTR_SLOT                   46
-#define LOAD_ATTR_MODULE                 47
-#define LOAD_GLOBAL_ADAPTIVE             48
-#define LOAD_GLOBAL_MODULE               55
-#define LOAD_GLOBAL_BUILTIN              56
-#define LOAD_METHOD_ADAPTIVE             57
-#define LOAD_METHOD_CACHED               58
-#define LOAD_METHOD_CLASS                59
-#define LOAD_METHOD_MODULE               62
-#define LOAD_METHOD_NO_DICT              63
-#define STORE_ATTR_ADAPTIVE              64
-#define STORE_ATTR_INSTANCE_VALUE        65
-#define STORE_ATTR_SLOT                  66
-#define STORE_ATTR_WITH_HINT             67
-#define LOAD_FAST__LOAD_FAST             75
-#define STORE_FAST__LOAD_FAST            76
-#define LOAD_FAST__LOAD_CONST            77
-#define LOAD_CONST__LOAD_FAST            78
-#define STORE_FAST__STORE_FAST           79
+#define COMPARE_OP_ADAPTIVE              21
+#define COMPARE_OP_FLOAT_JUMP            22
+#define COMPARE_OP_INT_JUMP              23
+#define COMPARE_OP_STR_JUMP              24
+#define BINARY_SUBSCR_ADAPTIVE           26
+#define BINARY_SUBSCR_GETITEM            27
+#define BINARY_SUBSCR_LIST_INT           28
+#define BINARY_SUBSCR_TUPLE_INT          29
+#define BINARY_SUBSCR_DICT               34
+#define STORE_SUBSCR_ADAPTIVE            36
+#define STORE_SUBSCR_LIST_INT            38
+#define STORE_SUBSCR_DICT                39
+#define CALL_FUNCTION_ADAPTIVE           40
+#define CALL_FUNCTION_BUILTIN_O          41
+#define CALL_FUNCTION_BUILTIN_FAST       42
+#define CALL_FUNCTION_LEN                43
+#define CALL_FUNCTION_ISINSTANCE         44
+#define CALL_FUNCTION_PY_SIMPLE          45
+#define JUMP_ABSOLUTE_QUICK              46
+#define LOAD_ATTR_ADAPTIVE               47
+#define LOAD_ATTR_INSTANCE_VALUE         48
+#define LOAD_ATTR_WITH_HINT              55
+#define LOAD_ATTR_SLOT                   56
+#define LOAD_ATTR_MODULE                 57
+#define LOAD_GLOBAL_ADAPTIVE             58
+#define LOAD_GLOBAL_MODULE               59
+#define LOAD_GLOBAL_BUILTIN              62
+#define LOAD_METHOD_ADAPTIVE             63
+#define LOAD_METHOD_CACHED               64
+#define LOAD_METHOD_CLASS                65
+#define LOAD_METHOD_MODULE               66
+#define LOAD_METHOD_NO_DICT              67
+#define STORE_ATTR_ADAPTIVE              75
+#define STORE_ATTR_INSTANCE_VALUE        76
+#define STORE_ATTR_SLOT                  77
+#define STORE_ATTR_WITH_HINT             78
+#define LOAD_FAST__LOAD_FAST             79
+#define STORE_FAST__LOAD_FAST            80
+#define LOAD_FAST__LOAD_CONST            81
+#define LOAD_CONST__LOAD_FAST            87
+#define STORE_FAST__STORE_FAST           88
 #define DO_TRACING                      255
 #ifdef NEED_OPCODE_JUMP_TABLES
 static uint32_t _PyOpcode_RelativeJump[8] = {
diff --git a/Lib/opcode.py b/Lib/opcode.py
index 60805e92ff3c4..e5889bca4c161 100644
--- a/Lib/opcode.py
+++ b/Lib/opcode.py
@@ -234,6 +234,10 @@ def jabs_op(name, op):
     "BINARY_OP_MULTIPLY_FLOAT",
     "BINARY_OP_SUBTRACT_INT",
     "BINARY_OP_SUBTRACT_FLOAT",
+    "COMPARE_OP_ADAPTIVE",
+    "COMPARE_OP_FLOAT_JUMP",
+    "COMPARE_OP_INT_JUMP",
+    "COMPARE_OP_STR_JUMP",
     "BINARY_SUBSCR_ADAPTIVE",
     "BINARY_SUBSCR_GETITEM",
     "BINARY_SUBSCR_LIST_INT",
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-23-21-01-56.bpo-45885.3IxeCX.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-23-21-01-56.bpo-45885.3IxeCX.rst
new file mode 100644
index 0000000000000..316daf966f149
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-11-23-21-01-56.bpo-45885.3IxeCX.rst	
@@ -0,0 +1 @@
+Specialized the ``COMPARE_OP`` opcode using the PEP 659 machinery.
\ No newline at end of file
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 61fc34d71da3c..532c48ad4d4aa 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -11168,6 +11168,20 @@ unicode_compare_eq(PyObject *str1, PyObject *str2)
     return (cmp == 0);
 }
 
+int
+_PyUnicode_Equal(PyObject *str1, PyObject *str2)
+{
+    assert(PyUnicode_CheckExact(str1));
+    assert(PyUnicode_CheckExact(str2));
+    if (str1 == str2) {
+        return 1;
+    }
+    if (PyUnicode_READY(str1) || PyUnicode_READY(str2)) {
+        return -1;
+    }
+    return unicode_compare_eq(str1, str2);
+}
+
 
 int
 PyUnicode_Compare(PyObject *left, PyObject *right)
diff --git a/Python/ceval.c b/Python/ceval.c
index 97c684479abdc..05897c561a16e 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -3778,6 +3778,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
         }
 
         TARGET(COMPARE_OP) {
+            PREDICTED(COMPARE_OP);
+            STAT_INC(COMPARE_OP, unquickened);
             assert(oparg <= Py_GE);
             PyObject *right = POP();
             PyObject *left = TOP();
@@ -3792,6 +3794,125 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
             DISPATCH();
         }
 
+        TARGET(COMPARE_OP_ADAPTIVE) {
+            assert(cframe.use_tracing == 0);
+            SpecializedCacheEntry *cache = GET_CACHE();
+            if (cache->adaptive.counter == 0) {
+                PyObject *right = TOP();
+                PyObject *left = SECOND();
+                next_instr--;
+                _Py_Specialize_CompareOp(left, right, next_instr, cache);
+                DISPATCH();
+            }
+            else {
+                STAT_INC(COMPARE_OP, deferred);
+                cache->adaptive.counter--;
+                oparg = cache->adaptive.original_oparg;
+                STAT_DEC(COMPARE_OP, unquickened);
+                JUMP_TO_INSTRUCTION(COMPARE_OP);
+            }
+        }
+
+        TARGET(COMPARE_OP_FLOAT_JUMP) {
+            assert(cframe.use_tracing == 0);
+            // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false)
+            SpecializedCacheEntry *caches = GET_CACHE();
+            int when_to_jump_mask = caches[0].adaptive.index;
+            PyObject *right = TOP();
+            PyObject *left = SECOND();
+            DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
+            DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
+            double dleft = PyFloat_AS_DOUBLE(left);
+            double dright = PyFloat_AS_DOUBLE(right);
+            int sign = (dleft > dright) - (dleft < dright);
+            DEOPT_IF(isnan(dleft), COMPARE_OP);
+            DEOPT_IF(isnan(dright), COMPARE_OP);
+            STAT_INC(COMPARE_OP, hit);
+            NEXTOPARG();
+            STACK_SHRINK(2);
+            Py_DECREF(left);
+            Py_DECREF(right);
+            assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
+            int jump = (1 << (sign + 1)) & when_to_jump_mask;
+            if (!jump) {
+                next_instr++;
+                NOTRACE_DISPATCH();
+            }
+            else {
+                JUMPTO(oparg);
+                CHECK_EVAL_BREAKER();
+                NOTRACE_DISPATCH();
+            }
+        }
+
+        TARGET(COMPARE_OP_INT_JUMP) {
+            assert(cframe.use_tracing == 0);
+            // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false)
+            SpecializedCacheEntry *caches = GET_CACHE();
+            int when_to_jump_mask = caches[0].adaptive.index;
+            PyObject *right = TOP();
+            PyObject *left = SECOND();
+            DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
+            DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
+            DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP);
+            DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP);
+            STAT_INC(COMPARE_OP, hit);
+            assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1);
+            Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0];
+            Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0];
+            int sign = (ileft > iright) - (ileft < iright);
+            NEXTOPARG();
+            STACK_SHRINK(2);
+            Py_DECREF(left);
+            Py_DECREF(right);
+            assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
+            int jump = (1 << (sign + 1)) & when_to_jump_mask;
+            if (!jump) {
+                next_instr++;
+                NOTRACE_DISPATCH();
+            }
+            else {
+                JUMPTO(oparg);
+                CHECK_EVAL_BREAKER();
+                NOTRACE_DISPATCH();
+            }
+        }
+
+        TARGET(COMPARE_OP_STR_JUMP) {
+            assert(cframe.use_tracing == 0);
+            // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false)
+            SpecializedCacheEntry *caches = GET_CACHE();
+            int invert = caches[0].adaptive.index;
+            PyObject *right = TOP();
+            PyObject *left = SECOND();
+            DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
+            DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
+            STAT_INC(COMPARE_OP, hit);
+            int res = _PyUnicode_Equal(left, right);
+            if (res < 0) {
+                goto error;
+            }
+            assert(caches[0].adaptive.original_oparg == Py_EQ ||
+                   caches[0].adaptive.original_oparg == Py_NE);
+            NEXTOPARG();
+            assert(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE);
+            STACK_SHRINK(2);
+            Py_DECREF(left);
+            Py_DECREF(right);
+            assert(res == 0 || res == 1);
+            assert(invert == 0 || invert == 1);
+            int jump = res ^ invert;
+            if (!jump) {
+                next_instr++;
+                NOTRACE_DISPATCH();
+            }
+            else {
+                JUMPTO(oparg);
+                CHECK_EVAL_BREAKER();
+                NOTRACE_DISPATCH();
+            }
+        }
+
         TARGET(IS_OP) {
             PyObject *right = POP();
             PyObject *left = TOP();
@@ -5083,6 +5204,7 @@ MISS_WITH_CACHE(LOAD_GLOBAL)
 MISS_WITH_CACHE(LOAD_METHOD)
 MISS_WITH_CACHE(CALL_FUNCTION)
 MISS_WITH_CACHE(BINARY_OP)
+MISS_WITH_CACHE(COMPARE_OP)
 MISS_WITH_CACHE(BINARY_SUBSCR)
 MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
 
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index c9d430d26814c..872a688311992 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -20,23 +20,27 @@ static void *opcode_targets[256] = {
     &&TARGET_BINARY_OP_MULTIPLY_FLOAT,
     &&TARGET_BINARY_OP_SUBTRACT_INT,
     &&TARGET_BINARY_OP_SUBTRACT_FLOAT,
+    &&TARGET_COMPARE_OP_ADAPTIVE,
+    &&TARGET_COMPARE_OP_FLOAT_JUMP,
+    &&TARGET_COMPARE_OP_INT_JUMP,
+    &&TARGET_COMPARE_OP_STR_JUMP,
+    &&TARGET_BINARY_SUBSCR,
     &&TARGET_BINARY_SUBSCR_ADAPTIVE,
     &&TARGET_BINARY_SUBSCR_GETITEM,
     &&TARGET_BINARY_SUBSCR_LIST_INT,
     &&TARGET_BINARY_SUBSCR_TUPLE_INT,
-    &&TARGET_BINARY_SUBSCR,
-    &&TARGET_BINARY_SUBSCR_DICT,
-    &&TARGET_STORE_SUBSCR_ADAPTIVE,
-    &&TARGET_STORE_SUBSCR_LIST_INT,
-    &&TARGET_STORE_SUBSCR_DICT,
     &&TARGET_GET_LEN,
     &&TARGET_MATCH_MAPPING,
     &&TARGET_MATCH_SEQUENCE,
     &&TARGET_MATCH_KEYS,
-    &&TARGET_CALL_FUNCTION_ADAPTIVE,
+    &&TARGET_BINARY_SUBSCR_DICT,
     &&TARGET_PUSH_EXC_INFO,
-    &&TARGET_CALL_FUNCTION_BUILTIN_O,
+    &&TARGET_STORE_SUBSCR_ADAPTIVE,
     &&TARGET_POP_EXCEPT_AND_RERAISE,
+    &&TARGET_STORE_SUBSCR_LIST_INT,
+    &&TARGET_STORE_SUBSCR_DICT,
+    &&TARGET_CALL_FUNCTION_ADAPTIVE,
+    &&TARGET_CALL_FUNCTION_BUILTIN_O,
     &&TARGET_CALL_FUNCTION_BUILTIN_FAST,
     &&TARGET_CALL_FUNCTION_LEN,
     &&TARGET_CALL_FUNCTION_ISINSTANCE,
@@ -44,29 +48,25 @@ static void *opcode_targets[256] = {
     &&TARGET_JUMP_ABSOLUTE_QUICK,
     &&TARGET_LOAD_ATTR_ADAPTIVE,
     &&TARGET_LOAD_ATTR_INSTANCE_VALUE,
-    &&TARGET_LOAD_ATTR_WITH_HINT,
-    &&TARGET_LOAD_ATTR_SLOT,
-    &&TARGET_LOAD_ATTR_MODULE,
-    &&TARGET_LOAD_GLOBAL_ADAPTIVE,
     &&TARGET_WITH_EXCEPT_START,
     &&TARGET_GET_AITER,
     &&TARGET_GET_ANEXT,
     &&TARGET_BEFORE_ASYNC_WITH,
     &&TARGET_BEFORE_WITH,
     &&TARGET_END_ASYNC_FOR,
+    &&TARGET_LOAD_ATTR_WITH_HINT,
+    &&TARGET_LOAD_ATTR_SLOT,
+    &&TARGET_LOAD_ATTR_MODULE,
+    &&TARGET_LOAD_GLOBAL_ADAPTIVE,
     &&TARGET_LOAD_GLOBAL_MODULE,
+    &&TARGET_STORE_SUBSCR,
+    &&TARGET_DELETE_SUBSCR,
     &&TARGET_LOAD_GLOBAL_BUILTIN,
     &&TARGET_LOAD_METHOD_ADAPTIVE,
     &&TARGET_LOAD_METHOD_CACHED,
     &&TARGET_LOAD_METHOD_CLASS,
-    &&TARGET_STORE_SUBSCR,
-    &&TARGET_DELETE_SUBSCR,
     &&TARGET_LOAD_METHOD_MODULE,
     &&TARGET_LOAD_METHOD_NO_DICT,
-    &&TARGET_STORE_ATTR_ADAPTIVE,
-    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
-    &&TARGET_STORE_ATTR_SLOT,
-    &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_GET_ITER,
     &&TARGET_GET_YIELD_FROM_ITER,
     &&TARGET_PRINT_EXPR,
@@ -74,20 +74,20 @@ static void *opcode_targets[256] = {
     &&TARGET_YIELD_FROM,
     &&TARGET_GET_AWAITABLE,
     &&TARGET_LOAD_ASSERTION_ERROR,
+    &&TARGET_STORE_ATTR_ADAPTIVE,
+    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
+    &&TARGET_STORE_ATTR_SLOT,
+    &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_LOAD_FAST__LOAD_FAST,
     &&TARGET_STORE_FAST__LOAD_FAST,
     &&TARGET_LOAD_FAST__LOAD_CONST,
-    &&TARGET_LOAD_CONST__LOAD_FAST,
-    &&TARGET_STORE_FAST__STORE_FAST,
-    &&_unknown_opcode,
-    &&_unknown_opcode,
     &&TARGET_LIST_TO_TUPLE,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
     &&TARGET_SETUP_ANNOTATIONS,
     &&TARGET_YIELD_VALUE,
-    &&_unknown_opcode,
-    &&_unknown_opcode,
+    &&TARGET_LOAD_CONST__LOAD_FAST,
+    &&TARGET_STORE_FAST__STORE_FAST,
     &&TARGET_POP_EXCEPT,
     &&TARGET_STORE_NAME,
     &&TARGET_DELETE_NAME,
diff --git a/Python/specialize.c b/Python/specialize.c
index f5f12139df79b..b384675560be7 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -129,6 +129,7 @@ _Py_GetSpecializationStats(void) {
     err += add_stat_dict(stats, STORE_ATTR, "store_attr");
     err += add_stat_dict(stats, CALL_FUNCTION, "call_function");
     err += add_stat_dict(stats, BINARY_OP, "binary_op");
+    err += add_stat_dict(stats, COMPARE_OP, "compare_op");
     if (err < 0) {
         Py_DECREF(stats);
         return NULL;
@@ -187,6 +188,7 @@ _Py_PrintSpecializationStats(void)
     print_stats(out, &_specialization_stats[STORE_ATTR], "store_attr");
     print_stats(out, &_specialization_stats[CALL_FUNCTION], "call_function");
     print_stats(out, &_specialization_stats[BINARY_OP], "binary_op");
+    print_stats(out, &_specialization_stats[COMPARE_OP], "compare_op");
     if (out != stderr) {
         fclose(out);
     }
@@ -239,6 +241,7 @@ static uint8_t adaptive_opcodes[256] = {
     [CALL_FUNCTION] = CALL_FUNCTION_ADAPTIVE,
     [STORE_ATTR] = STORE_ATTR_ADAPTIVE,
     [BINARY_OP] = BINARY_OP_ADAPTIVE,
+    [COMPARE_OP] = COMPARE_OP_ADAPTIVE,
 };
 
 /* The number of cache entries required for a "family" of instructions. */
@@ -251,6 +254,7 @@ static uint8_t cache_requirements[256] = {
     [CALL_FUNCTION] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
     [STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */
     [BINARY_OP] = 1,  // _PyAdaptiveEntry
+    [COMPARE_OP] = 1, /* _PyAdaptiveEntry */
 };
 
 /* Return the oparg for the cache_offset and instruction index.
@@ -487,6 +491,10 @@ initial_counter_value(void) {
 #define SPEC_FAIL_BAD_CALL_FLAGS 17
 #define SPEC_FAIL_CLASS 18
 
+/* COMPARE_OP */
+#define SPEC_FAIL_STRING_COMPARE 13
+#define SPEC_FAIL_NOT_FOLLOWED_BY_COND_JUMP 14
+#define SPEC_FAIL_BIG_INT 15
 
 static int
 specialize_module_load_attr(
@@ -1536,3 +1544,74 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
     STAT_INC(BINARY_OP, specialization_success);
     adaptive->counter = initial_counter_value();
 }
+
+static int compare_masks[] = {
+    // 1-bit: jump if less than
+    // 2-bit: jump if equal
+    // 4-bit: jump if greater
+    [Py_LT] = 1 | 0 | 0,
+    [Py_LE] = 1 | 2 | 0,
+    [Py_EQ] = 0 | 2 | 0,
+    [Py_NE] = 1 | 0 | 4,
+    [Py_GT] = 0 | 0 | 4,
+    [Py_GE] = 0 | 2 | 4,
+};
+
+void
+_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs,
+                         _Py_CODEUNIT *instr, SpecializedCacheEntry *cache)
+{
+    _PyAdaptiveEntry *adaptive = &cache->adaptive;
+    int op = adaptive->original_oparg;
+    int next_opcode = _Py_OPCODE(instr[1]);
+    if (next_opcode != POP_JUMP_IF_FALSE && next_opcode != POP_JUMP_IF_TRUE) {
+        // Can't ever combine, so don't don't bother being adaptive.
+        SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_NOT_FOLLOWED_BY_COND_JUMP);
+        *instr = _Py_MAKECODEUNIT(COMPARE_OP, adaptive->original_oparg);
+        goto failure;
+    }
+    assert(op <= Py_GE);
+    int when_to_jump_mask = compare_masks[op];
+    if (next_opcode == POP_JUMP_IF_FALSE) {
+        when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask;
+    }
+    if (Py_TYPE(lhs) != Py_TYPE(rhs)) {
+        SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_DIFFERENT_TYPES);
+        goto failure;
+    }
+    if (PyFloat_CheckExact(lhs)) {
+        *instr = _Py_MAKECODEUNIT(COMPARE_OP_FLOAT_JUMP, _Py_OPARG(*instr));
+        adaptive->index = when_to_jump_mask;
+        goto success;
+    }
+    if (PyLong_CheckExact(lhs)) {
+        if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) {
+            *instr = _Py_MAKECODEUNIT(COMPARE_OP_INT_JUMP, _Py_OPARG(*instr));
+            adaptive->index = when_to_jump_mask;
+            goto success;
+        }
+        else {
+            SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_BIG_INT);
+            goto failure;
+        }
+    }
+    if (PyUnicode_CheckExact(lhs)) {
+        if (op != Py_EQ && op != Py_NE) {
+            SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_STRING_COMPARE);
+            goto failure;
+        }
+        else {
+            *instr = _Py_MAKECODEUNIT(COMPARE_OP_STR_JUMP, _Py_OPARG(*instr));
+            adaptive->index = (when_to_jump_mask & 2) == 0;
+            goto success;
+        }
+    }
+    SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_OTHER);
+failure:
+    STAT_INC(COMPARE_OP, specialization_failure);
+    cache_backoff(adaptive);
+    return;
+success:
+    STAT_INC(COMPARE_OP, specialization_success);
+    adaptive->counter = initial_counter_value();
+}


From webhook-mailer at python.org  Fri Dec  3 08:47:17 2021
From: webhook-mailer at python.org (tiran)
Date: Fri, 03 Dec 2021 13:47:17 -0000
Subject: [Python-checkins] bpo-45582: Fix out-of-tree build issues with new
 getpath (GH-29902)
Message-ID: <mailman.24.1638539237.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/ccb73a0d50dd03bc8455fe210cb83e41a6dc91d8
commit: ccb73a0d50dd03bc8455fe210cb83e41a6dc91d8
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-03T14:47:06+01:00
summary:

bpo-45582: Fix out-of-tree build issues with new getpath (GH-29902)

files:
M .gitignore
M Makefile.pre.in
M Modules/getpath.c
M PCbuild/_freeze_module.vcxproj

diff --git a/.gitignore b/.gitignore
index 7e80446d4d4a6..0363244bdaf63 100644
--- a/.gitignore
+++ b/.gitignore
@@ -74,7 +74,6 @@ Mac/pythonw
 Misc/python.pc
 Misc/python-embed.pc
 Misc/python-config.sh
-Modules/getpath.h
 Modules/Setup.config
 Modules/Setup.local
 Modules/Setup.stdlib
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 3dc131b6c9651..264011f226bbc 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1062,10 +1062,9 @@ FROZEN_FILES_OUT = \
 Programs/_freeze_module.o: Programs/_freeze_module.c Makefile
 
 Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile
-	$(CC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Modules/getpath_noop.c
 
-Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Modules/getpath_noop.o
-	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) Modules/getpath_noop.o $(LIBS) $(MODLIBS) $(SYSLIBS)
+Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN)
+	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
 
 # BEGIN: freezing modules
 
@@ -1131,11 +1130,11 @@ Python/frozen_modules/frozen_only.h: $(FREEZE_MODULE) Tools/freeze/flag.py
 
 # END: freezing modules
 
-Tools/scripts/freeze_modules.py: $(FREEZE_MODULE)
-
 # We manually freeze getpath.py rather than through freeze_modules
-Modules/getpath.h: Programs/_freeze_module Modules/getpath.py
-	Programs/_freeze_module getpath $(srcdir)/Modules/getpath.py $(srcdir)/Modules/getpath.h
+Python/frozen_modules/getpath.h: $(FREEZE_MODULE) Modules/getpath.py
+	$(FREEZE_MODULE) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h
+
+Tools/scripts/freeze_modules.py: $(FREEZE_MODULE)
 
 .PHONY: regen-frozen
 regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN)
@@ -1177,7 +1176,7 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \
 	      -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \
 	      -o $@ $(srcdir)/Modules/getbuildinfo.c
 
-Modules/getpath.o: $(srcdir)/Modules/getpath.c Modules/getpath.h Makefile
+Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h Makefile $(PYTHON_HEADERS)
 	$(CC) -c $(PY_CORE_CFLAGS) -DPYTHONPATH='"$(PYTHONPATH)"' \
 		-DPREFIX='"$(prefix)"' \
 		-DEXEC_PREFIX='"$(exec_prefix)"' \
diff --git a/Modules/getpath.c b/Modules/getpath.c
index 32d5db9d2c4dc..f66fc8abbd4ff 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -19,7 +19,7 @@
 #endif
 
 /* Reference the precompiled getpath.py */
-#include "getpath.h"
+#include "../Python/frozen_modules/getpath.h"
 
 #if (!defined(PREFIX) || !defined(EXEC_PREFIX) \
         || !defined(VERSION) || !defined(VPATH) \
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index 7b2df4b8afcc2..07a6bfdde7402 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -379,7 +379,7 @@
     <GetPath Include="..\Modules\getpath.py">
       <ModName>getpath</ModName>
       <IntFile>$(IntDir)getpath.g.h</IntFile>
-      <OutFile>$(PySourcePath)Modules\getpath.h</OutFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\getpath.h</OutFile>
     </GetPath>
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />


From webhook-mailer at python.org  Fri Dec  3 10:01:20 2021
From: webhook-mailer at python.org (tiran)
Date: Fri, 03 Dec 2021 15:01:20 -0000
Subject: [Python-checkins] bpo-45950: Introduce Bootstrap Python again
 (#29859)
Message-ID: <mailman.25.1638543681.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/84ca1232b0f1e4be368e89550a9ceb46f64a0eff
commit: 84ca1232b0f1e4be368e89550a9ceb46f64a0eff
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-03T16:01:11+01:00
summary:

bpo-45950: Introduce Bootstrap Python again (#29859)

The build system now uses a :program:`_bootstrap_python` interpreter for
freezing and deepfreezing again. To speed up build process the build tools
:program:`_bootstrap_python` and :program:`_freeze_module` are no longer
build with LTO.

Cross building depends on a build Python interpreter, which must have same
version and bytecode as target host Python.

files:
A Misc/NEWS.d/next/Build/2021-12-01-17-28-39.bpo-45950.eEVLoz.rst
A Programs/_bootstrap_python.c
D Python/bootstrap_frozen.c
M .gitignore
M Makefile.pre.in
M Tools/scripts/deepfreeze.py
M Tools/scripts/freeze_modules.py
M configure
M configure.ac

diff --git a/.gitignore b/.gitignore
index 0363244bdaf63..cfd3163cd1818 100644
--- a/.gitignore
+++ b/.gitignore
@@ -60,6 +60,7 @@ Lib/distutils/command/*.pdb
 Lib/lib2to3/*.pickle
 Lib/test/data/*
 !Lib/test/data/README
+/_bootstrap_python
 /Makefile
 /Makefile.pre
 Mac/Makefile
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 264011f226bbc..8e6e553554de1 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -90,6 +90,9 @@ CONFIGURE_CFLAGS_NODIST=@CFLAGS_NODIST@
 # Use it when a linker flag should _not_ be part of the distutils LDFLAGS
 # once Python is installed (bpo-35257)
 CONFIGURE_LDFLAGS_NODIST=@LDFLAGS_NODIST@
+# LDFLAGS_NOLTO is an extra flag to disable lto. It is used to speed up building
+# of _bootstrap_python and _freeze_module tools, which don't need LTO.
+CONFIGURE_LDFLAGS_NOLTO=@LDFLAGS_NOLTO@
 CONFIGURE_CPPFLAGS=	@CPPFLAGS@
 CONFIGURE_LDFLAGS=	@LDFLAGS@
 # Avoid assigning CFLAGS, LDFLAGS, etc. so users can use them on the
@@ -103,6 +106,7 @@ PY_CFLAGS_NODIST=$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST) -I$(srcdir)/Include
 PY_CPPFLAGS=	$(BASECPPFLAGS) -I. -I$(srcdir)/Include $(CONFIGURE_CPPFLAGS) $(CPPFLAGS)
 PY_LDFLAGS=	$(CONFIGURE_LDFLAGS) $(LDFLAGS)
 PY_LDFLAGS_NODIST=$(CONFIGURE_LDFLAGS_NODIST) $(LDFLAGS_NODIST)
+PY_LDFLAGS_NOLTO=$(PY_LDFLAGS) $(CONFIGURE_LDFLAGS_NOLTO) $(LDFLAGS_NODIST)
 NO_AS_NEEDED=	@NO_AS_NEEDED@
 CCSHARED=	@CCSHARED@
 # LINKFORSHARED are the flags passed to the $(CC) command that links
@@ -279,6 +283,9 @@ BUILDPYTHON=	python$(BUILDEXE)
 PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
 UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
 PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
+# Standard builds use _bootstrap_python for freezing, cross compiling
+# uses build Python, which must have the same version and bytecode,
+PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@
 _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
 BUILD_GNU_TYPE=	@build@
 HOST_GNU_TYPE=	@host@
@@ -938,75 +945,88 @@ regen-test-frozenmain: $(BUILDPYTHON)
 Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
 	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
 
+############################################################################
+# "Bootstrap Python" used to run deepfreeze.py
+
+BOOTSTRAP_HEADERS = \
+	Python/frozen_modules/importlib._bootstrap.h \
+	Python/frozen_modules/importlib._bootstrap_external.h
+
+Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS)
+
+_bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local
+	$(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \
+		Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS)
+
 ############################################################################
 # Deepfreeze targets
 
 .PHONY: regen-deepfreeze
 regen-deepfreeze: $(DEEPFREEZE_OBJS)
 
-DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py
+DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py _bootstrap_python
 
 # BEGIN: deepfreeze modules
 
 Python/deepfreeze/importlib._bootstrap.c: Python/frozen_modules/importlib._bootstrap.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
 
 Python/deepfreeze/importlib._bootstrap_external.c: Python/frozen_modules/importlib._bootstrap_external.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
 
 Python/deepfreeze/zipimport.c: Python/frozen_modules/zipimport.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c
 
 Python/deepfreeze/abc.c: Python/frozen_modules/abc.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c
 
 Python/deepfreeze/codecs.c: Python/frozen_modules/codecs.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c
 
 Python/deepfreeze/io.c: Python/frozen_modules/io.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c
 
 Python/deepfreeze/_collections_abc.c: Python/frozen_modules/_collections_abc.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c
 
 Python/deepfreeze/_sitebuiltins.c: Python/frozen_modules/_sitebuiltins.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
 
 Python/deepfreeze/genericpath.c: Python/frozen_modules/genericpath.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c
 
 Python/deepfreeze/ntpath.c: Python/frozen_modules/ntpath.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c
 
 Python/deepfreeze/posixpath.c: Python/frozen_modules/posixpath.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c
 
 Python/deepfreeze/os.c: Python/frozen_modules/os.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c
 
 Python/deepfreeze/site.c: Python/frozen_modules/site.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c
 
 Python/deepfreeze/stat.c: Python/frozen_modules/stat.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c
 
 Python/deepfreeze/__hello__.c: Python/frozen_modules/__hello__.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c
 
 Python/deepfreeze/__phello__.c: Python/frozen_modules/__phello__.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c
 
 Python/deepfreeze/__phello__.ham.c: Python/frozen_modules/__phello__.ham.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
 
 Python/deepfreeze/__phello__.ham.eggs.c: Python/frozen_modules/__phello__.ham.eggs.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
 
 Python/deepfreeze/__phello__.spam.c: Python/frozen_modules/__phello__.spam.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
 
 Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c
 
 # END: deepfreeze modules
 
@@ -2295,6 +2315,7 @@ clean-retain-profile: pycremoval
 	find build -name '*.py[co]' -exec rm -f {} ';' || true
 	-rm -f pybuilddir.txt
 	-rm -f Lib/lib2to3/*Grammar*.pickle
+	-rm -f _bootstrap_python
 	-rm -f Programs/_testembed Programs/_freeze_module
 	-rm -f Python/deepfreeze/*.[co]
 	-rm -f Python/frozen_modules/*.h
diff --git a/Misc/NEWS.d/next/Build/2021-12-01-17-28-39.bpo-45950.eEVLoz.rst b/Misc/NEWS.d/next/Build/2021-12-01-17-28-39.bpo-45950.eEVLoz.rst
new file mode 100644
index 0000000000000..91f7560fd91d6
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2021-12-01-17-28-39.bpo-45950.eEVLoz.rst
@@ -0,0 +1,4 @@
+The build system now uses a :program:`_bootstrap_python` interpreter for
+freezing and deepfreezing again. To speed up build process the build tools
+:program:`_bootstrap_python` and :program:`_freeze_module` are no longer
+build with LTO.
diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c
new file mode 100644
index 0000000000000..f2103fd5aaae5
--- /dev/null
+++ b/Programs/_bootstrap_python.c
@@ -0,0 +1,105 @@
+
+/* Frozen modules bootstrap
+ *
+ * Limited and restricted Python interpreter to run
+ * "Tools/scripts/deepfreeze.py" on systems with no or older Python
+ * interpreter.
+ */
+
+#include "Python.h"
+#include "pycore_import.h"
+
+/* Includes for frozen modules: */
+#include "Python/frozen_modules/importlib._bootstrap.h"
+#include "Python/frozen_modules/importlib._bootstrap_external.h"
+/* End includes */
+
+/* Note that a negative size indicates a package. */
+
+static const struct _frozen bootstrap_modules[] = {
+    {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)},
+    {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)},
+    {0, 0, 0} /* bootstrap sentinel */
+};
+static const struct _frozen stdlib_modules[] = {
+    {0, 0, 0} /* stdlib sentinel */
+};
+static const struct _frozen test_modules[] = {
+    {0, 0, 0} /* test sentinel */
+};
+const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules;
+const struct _frozen *_PyImport_FrozenStdlib = stdlib_modules;
+const struct _frozen *_PyImport_FrozenTest = test_modules;
+
+static const struct _module_alias aliases[] = {
+    {"_frozen_importlib", "importlib._bootstrap"},
+    {"_frozen_importlib_external", "importlib._bootstrap_external"},
+    {0, 0} /* aliases sentinel */
+};
+const struct _module_alias *_PyImport_FrozenAliases = aliases;
+
+/* Embedding apps may change this pointer to point to their favorite
+   collection of frozen modules: */
+
+const struct _frozen *PyImport_FrozenModules = NULL;
+
+int
+#ifdef MS_WINDOWS
+wmain(int argc, wchar_t **argv)
+#else
+main(int argc, char **argv)
+#endif
+{
+    PyStatus status;
+
+    PyConfig config;
+    PyConfig_InitIsolatedConfig(&config);
+    // don't warn, pybuilddir.txt does not exist yet
+    config.pathconfig_warnings = 0;
+    // parse arguments
+    config.parse_argv = 1;
+    // add current script dir to sys.path
+    config.isolated = 0;
+
+#ifdef MS_WINDOWS
+    status = PyConfig_SetArgv(&config, argc, argv);
+#else
+    status = PyConfig_SetBytesArgv(&config, argc, argv);
+#endif
+    if (PyStatus_Exception(status)) {
+        goto error;
+    }
+
+    status = PyConfig_Read(&config);
+    if (config.run_filename == NULL) {
+        status = PyStatus_Error("Run filename expected");
+        goto error;
+    }
+
+#define CLEAR(ATTR) \
+    do { \
+        PyMem_RawFree(ATTR); \
+        ATTR = NULL; \
+    } while (0)
+
+    // isolate from system Python
+    CLEAR(config.base_prefix);
+    CLEAR(config.prefix);
+    CLEAR(config.base_exec_prefix);
+    CLEAR(config.exec_prefix);
+
+    status = Py_InitializeFromConfig(&config);
+    if (PyStatus_Exception(status)) {
+        goto error;
+    }
+    PyConfig_Clear(&config);
+
+    return Py_RunMain();
+
+error:
+    PyConfig_Clear(&config);
+    if (PyStatus_IsExit(status)) {
+        return status.exitcode;
+    }
+    Py_ExitStatusException(status);
+}
diff --git a/Python/bootstrap_frozen.c b/Python/bootstrap_frozen.c
deleted file mode 100644
index 68ba147a72710..0000000000000
--- a/Python/bootstrap_frozen.c
+++ /dev/null
@@ -1,45 +0,0 @@
-
-/* Frozen modules bootstrap */
-
-/* This file is linked with "bootstrap Python"
-   which is used (only) to run Tools/scripts/deepfreeze.py. */
-
-#include "Python.h"
-#include "pycore_import.h"
-
-/* Includes for frozen modules: */
-#include "frozen_modules/importlib._bootstrap.h"
-#include "frozen_modules/importlib._bootstrap_external.h"
-#include "frozen_modules/zipimport.h"
-/* End includes */
-
-/* Note that a negative size indicates a package. */
-
-static const struct _frozen bootstrap_modules[] = {
-    {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)},
-    {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)},
-    {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
-    {0, 0, 0} /* bootstrap sentinel */
-};
-static const struct _frozen stdlib_modules[] = {
-    {0, 0, 0} /* stdlib sentinel */
-};
-static const struct _frozen test_modules[] = {
-    {0, 0, 0} /* test sentinel */
-};
-const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules;
-const struct _frozen *_PyImport_FrozenStdlib = stdlib_modules;
-const struct _frozen *_PyImport_FrozenTest = test_modules;
-
-static const struct _module_alias aliases[] = {
-    {"_frozen_importlib", "importlib._bootstrap"},
-    {"_frozen_importlib_external", "importlib._bootstrap_external"},
-    {0, 0} /* aliases sentinel */
-};
-const struct _module_alias *_PyImport_FrozenAliases = aliases;
-
-
-/* Embedding apps may change this pointer to point to their favorite
-   collection of frozen modules: */
-
-const struct _frozen *PyImport_FrozenModules = NULL;
diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py
index 46776c768e648..30ca7bb0b6090 100644
--- a/Tools/scripts/deepfreeze.py
+++ b/Tools/scripts/deepfreeze.py
@@ -1,3 +1,8 @@
+"""Deep freeze
+
+The script is executed by _bootstrap_python interpreter. Shared library
+extension modules are not available.
+"""
 import argparse
 import ast
 import builtins
@@ -8,7 +13,6 @@
 import sys
 import time
 import types
-import unicodedata
 from typing import Dict, FrozenSet, Tuple, TextIO
 
 import umarshal
diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py
index 7496b6c868407..363a2e66bef52 100644
--- a/Tools/scripts/freeze_modules.py
+++ b/Tools/scripts/freeze_modules.py
@@ -589,7 +589,7 @@ def regen_makefile(modules):
         ])
         deepfreezerules.append(f'{cfile}: {frozen_header} $(DEEPFREEZE_DEPS)')
         deepfreezerules.append(
-            f"\t$(PYTHON_FOR_REGEN) "
+            f"\t$(PYTHON_FOR_FREEZE) "
             f"$(srcdir)/Tools/scripts/deepfreeze.py "
             f"{frozen_header} -m {src.frozenid} -o {cfile}")
         deepfreezerules.append('')
diff --git a/configure b/configure
index 0aceffb35065a..5d47d03542e40 100755
--- a/configure
+++ b/configure
@@ -845,6 +845,7 @@ SHLIB_SUFFIX
 LIBTOOL_CRUFT
 OTHER_LIBTOOL_OPT
 UNIVERSAL_ARCH_FLAGS
+LDFLAGS_NOLTO
 LDFLAGS_NODIST
 CFLAGS_NODIST
 BASECFLAGS
@@ -932,6 +933,7 @@ CONFIG_ARGS
 SOVERSION
 VERSION
 PYTHON_FOR_REGEN
+PYTHON_FOR_FREEZE
 PYTHON_FOR_BUILD
 FREEZE_MODULE
 host_os
@@ -3250,6 +3252,7 @@ fi
       as_fn_error $? "\"$with_build_python\" has incompatible version $build_python_ver (expected: $PACKAGE_VERSION)" "$LINENO" 5
     fi
         ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python
+    PYTHON_FOR_FREEZE="$with_build_python"
     PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python
     { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_build_python" >&5
 $as_echo "$with_build_python" >&6; }
@@ -3261,12 +3264,19 @@ else
 
 fi
     PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E'
+    PYTHON_FOR_FREEZE="./_bootstrap_python"
 
 
 fi
 
 
 
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for Python interpreter freezing" >&5
+$as_echo_n "checking for Python interpreter freezing... " >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTHON_FOR_FREEZE" >&5
+$as_echo "$PYTHON_FOR_FREEZE" >&6; }
+
+
 for ac_prog in python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python
 do
   # Extract the first word of "$ac_prog", so it can be a program name with args.
@@ -3314,7 +3324,7 @@ test -n "$PYTHON_FOR_REGEN" || PYTHON_FOR_REGEN="python3"
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking Python for regen version" >&5
 $as_echo_n "checking Python for regen version... " >&6; }
-if command -v $PYTHON_FOR_REGEN >/dev/null 2>&1; then
+if command -v "$PYTHON_FOR_REGEN" >/dev/null 2>&1; then
   { $as_echo "$as_me:${as_lineno-$LINENO}: result: $($PYTHON_FOR_REGEN -V 2>/dev/null)" >&5
 $as_echo "$($PYTHON_FOR_REGEN -V 2>/dev/null)" >&6; }
 else
@@ -7101,6 +7111,7 @@ fi
 if test "$Py_LTO" = 'true' ; then
   case $CC in
     *clang*)
+            LDFLAGS_NOLTO="-fno-lto"
 
       if test -n "$ac_tool_prefix"; then
   # Extract the first word of "${ac_tool_prefix}llvm-ar", so it can be a program name with args.
@@ -7254,6 +7265,7 @@ $as_echo "$as_me: llvm-ar found via xcrun: ${LLVM_AR}" >&6;}
       then
         as_fn_error $? "thin lto is not supported under gcc compiler." "$LINENO" 5
       fi
+            LDFLAGS_NOLTO="-fno-lto"
       case $ac_sys_system in
         Darwin*)
           LTOFLAGS="-flto -Wl,-export_dynamic"
@@ -7534,6 +7546,7 @@ fi
 
 
 
+
 # The -arch flags for universal builds on macOS
 UNIVERSAL_ARCH_FLAGS=
 
diff --git a/configure.ac b/configure.ac
index 99fa94d102909..b35c92acb85db 100644
--- a/configure.ac
+++ b/configure.ac
@@ -143,8 +143,9 @@ AC_ARG_WITH(
     if test "$build_python_ver" != "$PACKAGE_VERSION"; then
       AC_MSG_ERROR(["$with_build_python" has incompatible version $build_python_ver (expected: $PACKAGE_VERSION)])
     fi
-    dnl use build Python for regeneration, too
+    dnl Build Python interpreter is used for regeneration and freezing.
     ac_cv_prog_PYTHON_FOR_REGEN=$with_build_python
+    PYTHON_FOR_FREEZE="$with_build_python"
     PYTHON_FOR_BUILD='_PYTHON_PROJECT_BASE=$(abs_builddir) _PYTHON_HOST_PLATFORM=$(_PYTHON_HOST_PLATFORM) PYTHONPATH=$(shell test -f pybuilddir.txt && echo $(abs_builddir)/`cat pybuilddir.txt`:)$(srcdir)/Lib _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH) '$with_build_python
     AC_MSG_RESULT([$with_build_python])
   ], [
@@ -152,17 +153,22 @@ AC_ARG_WITH(
       [AC_MSG_ERROR([Cross compiling requires --with-build-python])]
     )
     PYTHON_FOR_BUILD='./$(BUILDPYTHON) -E'
+    PYTHON_FOR_FREEZE="./_bootstrap_python"
   ]
 )
 AC_SUBST([PYTHON_FOR_BUILD])
 
+AC_MSG_CHECKING([for Python interpreter freezing])
+AC_MSG_RESULT([$PYTHON_FOR_FREEZE])
+AC_SUBST([PYTHON_FOR_FREEZE])
+
 AC_CHECK_PROGS([PYTHON_FOR_REGEN],
   [python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python],
   [python3])
 AC_SUBST(PYTHON_FOR_REGEN)
 
 AC_MSG_CHECKING([Python for regen version])
-if command -v $PYTHON_FOR_REGEN >/dev/null 2>&1; then
+if command -v "$PYTHON_FOR_REGEN" >/dev/null 2>&1; then
   AC_MSG_RESULT([$($PYTHON_FOR_REGEN -V 2>/dev/null)])
 else
   AC_MSG_RESULT([missing])
@@ -1510,6 +1516,8 @@ esac
 if test "$Py_LTO" = 'true' ; then
   case $CC in
     *clang*)
+      dnl flag to disable lto during linking
+      LDFLAGS_NOLTO="-fno-lto"
       AC_SUBST(LLVM_AR)
       AC_PATH_TOOL(LLVM_AR, llvm-ar, '', ${llvm_path})
       AC_SUBST(LLVM_AR_FOUND)
@@ -1565,6 +1573,8 @@ if test "$Py_LTO" = 'true' ; then
       then
         AC_MSG_ERROR([thin lto is not supported under gcc compiler.])
       fi
+      dnl flag to disable lto during linking
+      LDFLAGS_NOLTO="-fno-lto"
       case $ac_sys_system in
         Darwin*)
           LTOFLAGS="-flto -Wl,-export_dynamic"
@@ -1746,6 +1756,7 @@ fi
 AC_SUBST(BASECFLAGS)
 AC_SUBST(CFLAGS_NODIST)
 AC_SUBST(LDFLAGS_NODIST)
+AC_SUBST(LDFLAGS_NOLTO)
 
 # The -arch flags for universal builds on macOS
 UNIVERSAL_ARCH_FLAGS=


From webhook-mailer at python.org  Fri Dec  3 14:05:24 2021
From: webhook-mailer at python.org (iritkatriel)
Date: Fri, 03 Dec 2021 19:05:24 -0000
Subject: [Python-checkins] bpo-45711: [asyncio] Normalize exceptions
 immediately after Fetch, before they are stored as StackItem,
 which should be normalized (GH-29890)
Message-ID: <mailman.26.1638558325.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/2ff758bd1a144ee712e96ae1db91f476c3b252bb
commit: 2ff758bd1a144ee712e96ae1db91f476c3b252bb
branch: main
author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com>
committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com>
date: 2021-12-03T19:05:14Z
summary:

bpo-45711: [asyncio] Normalize exceptions immediately after Fetch, before they are stored as StackItem, which should be normalized (GH-29890)

files:
A Misc/NEWS.d/next/Library/2021-12-02-17-22-06.bpo-45711.D6jsdv.rst
M Modules/_asynciomodule.c

diff --git a/Misc/NEWS.d/next/Library/2021-12-02-17-22-06.bpo-45711.D6jsdv.rst b/Misc/NEWS.d/next/Library/2021-12-02-17-22-06.bpo-45711.D6jsdv.rst
new file mode 100644
index 0000000000000..b2b57730c3ba8
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-12-02-17-22-06.bpo-45711.D6jsdv.rst
@@ -0,0 +1 @@
+Make :mod:`asyncio` normalize exceptions as soon as they are captured with :c:func:`PyErr_Fetch`, and before they are stored as an exc_info triplet. This brings :mod:`asyncio` in line with the rest of the codebase, where an exc_info triplet is always normalized.
\ No newline at end of file
diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c
index df6644ba248ed..267faacde8a17 100644
--- a/Modules/_asynciomodule.c
+++ b/Modules/_asynciomodule.c
@@ -2702,6 +2702,11 @@ task_step_impl(TaskObj *task, PyObject *exc)
         if (PyErr_ExceptionMatches(asyncio_CancelledError)) {
             /* CancelledError */
             PyErr_Fetch(&et, &ev, &tb);
+            assert(et);
+            PyErr_NormalizeException(&et, &ev, &tb);
+            if (tb != NULL) {
+                PyException_SetTraceback(ev, tb);
+            }
 
             FutureObj *fut = (FutureObj*)task;
             _PyErr_StackItem *exc_state = &fut->fut_cancelled_exc_state;
@@ -2714,14 +2719,12 @@ task_step_impl(TaskObj *task, PyObject *exc)
 
         /* Some other exception; pop it and call Task.set_exception() */
         PyErr_Fetch(&et, &ev, &tb);
-
         assert(et);
-        if (!ev || !PyObject_TypeCheck(ev, (PyTypeObject *) et)) {
-            PyErr_NormalizeException(&et, &ev, &tb);
-        }
+        PyErr_NormalizeException(&et, &ev, &tb);
         if (tb != NULL) {
             PyException_SetTraceback(ev, tb);
         }
+
         o = future_set_exception((FutureObj*)task, ev);
         if (!o) {
             /* An exception in Task.set_exception() */
@@ -2965,7 +2968,7 @@ task_step(TaskObj *task, PyObject *exc)
         PyObject *et, *ev, *tb;
         PyErr_Fetch(&et, &ev, &tb);
         leave_task(task->task_loop, (PyObject*)task);
-        _PyErr_ChainExceptions(et, ev, tb);
+        _PyErr_ChainExceptions(et, ev, tb); /* Normalizes (et, ev, tb) */
         return NULL;
     }
     else {
@@ -3014,8 +3017,10 @@ task_wakeup(TaskObj *task, PyObject *o)
     }
 
     PyErr_Fetch(&et, &ev, &tb);
-    if (!ev || !PyObject_TypeCheck(ev, (PyTypeObject *) et)) {
-        PyErr_NormalizeException(&et, &ev, &tb);
+    assert(et);
+    PyErr_NormalizeException(&et, &ev, &tb);
+    if (tb != NULL) {
+        PyException_SetTraceback(ev, tb);
     }
 
     result = task_step(task, ev);


From webhook-mailer at python.org  Fri Dec  3 14:47:49 2021
From: webhook-mailer at python.org (zooba)
Date: Fri, 03 Dec 2021 19:47:49 -0000
Subject: [Python-checkins] bpo-45816: Support building with VS 2022 (v143
 toolset) on Windows (GH-29577)
Message-ID: <mailman.27.1638560870.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/d9301703fb1086cafbd730c17e3d450a192485d6
commit: d9301703fb1086cafbd730c17e3d450a192485d6
branch: main
author: Crowthebird <78076854+thatbirdguythatuknownot at users.noreply.github.com>
committer: zooba <steve.dower at microsoft.com>
date: 2021-12-03T19:47:38Z
summary:

bpo-45816: Support building with VS 2022 (v143 toolset) on Windows (GH-29577)

files:
A Misc/NEWS.d/next/Build/2021-11-16-14-44-06.bpo-45816.nbdmVK.rst
M PCbuild/pyproject.props
M PCbuild/python.props
M PCbuild/pythoncore.vcxproj
M Tools/msi/bundle/bootstrap/pythonba.vcxproj

diff --git a/Misc/NEWS.d/next/Build/2021-11-16-14-44-06.bpo-45816.nbdmVK.rst b/Misc/NEWS.d/next/Build/2021-11-16-14-44-06.bpo-45816.nbdmVK.rst
new file mode 100644
index 0000000000000..4a14c90bf0138
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2021-11-16-14-44-06.bpo-45816.nbdmVK.rst
@@ -0,0 +1 @@
+Python now supports building with Visual Studio 2022 (MSVC v143, VS Version 17.0). Patch by Jeremiah Vivian.
\ No newline at end of file
diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props
index bbcabb5cdb405..0eaeb6b1229a2 100644
--- a/PCbuild/pyproject.props
+++ b/PCbuild/pyproject.props
@@ -73,7 +73,7 @@
       <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument'">PGInstrument</LinkTimeCodeGeneration>
       <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate'">PGUpdate</LinkTimeCodeGeneration>
       <AdditionalDependencies>advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;%(AdditionalDependencies)</AdditionalDependencies>
-      <AdditionalOptions Condition="$(Configuration) != 'Debug'">/OPT:REF,NOICF %(AdditionalOptions)</AdditionalOptions>
+      <AdditionalOptions Condition="$(Configuration) != 'Debug'">/OPT:REF,NOICF /CGTHREADS:1 /PDBTHREADS:1 %(AdditionalOptions)</AdditionalOptions>
     </Link>
     <Lib>
       <LinkTimeCodeGeneration Condition="$(Configuration) == 'Release'">true</LinkTimeCodeGeneration>
diff --git a/PCbuild/python.props b/PCbuild/python.props
index c608fb9ee7187..7b85ad1c94f16 100644
--- a/PCbuild/python.props
+++ b/PCbuild/python.props
@@ -11,7 +11,7 @@
 
     We set BasePlatformToolset for ICC's benefit, it's otherwise ignored.
     -->
-    <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VisualStudioVersion)' == '17.0'">v142</BasePlatformToolset>
+    <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VisualStudioVersion)' == '17.0'">v143</BasePlatformToolset>
     <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VisualStudioVersion)' == '16.0'">v142</BasePlatformToolset>
     <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and ('$(MSBuildToolsVersion)' == '15.0' or '$(VisualStudioVersion)' == '15.0')">v141</BasePlatformToolset>
     <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VCTargetsPath14)' != ''">v140</BasePlatformToolset>
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index b446e0995565e..119650a1eec51 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -588,7 +588,7 @@
       </ClCompile>
     </ItemGroup>
   </Target>
-  <Target Name="_WarnAboutToolset" BeforeTargets="PrepareForBuild" Condition="$(PlatformToolset) != 'v140' and $(PlatformToolset) != 'v141' and $(PlatformToolset) != 'v142'">
+  <Target Name="_WarnAboutToolset" BeforeTargets="PrepareForBuild" Condition="$(PlatformToolset) != 'v140' and $(PlatformToolset) != 'v141' and $(PlatformToolset) != 'v142' and $(PlatformToolset) != 'v143'">
     <Warning Text="Toolset $(PlatformToolset) is not used for official builds. Your build may have errors or incompatibilities." />
   </Target>
   <Target Name="_WarnAboutZlib" BeforeTargets="PrepareForBuild" Condition="!$(IncludeExternals)">
diff --git a/Tools/msi/bundle/bootstrap/pythonba.vcxproj b/Tools/msi/bundle/bootstrap/pythonba.vcxproj
index ef71fe7da08d3..d90b5e3ff0009 100644
--- a/Tools/msi/bundle/bootstrap/pythonba.vcxproj
+++ b/Tools/msi/bundle/bootstrap/pythonba.vcxproj
@@ -73,4 +73,4 @@
     <None Include="pythonba.def" />
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
-</Project>
\ No newline at end of file
+</Project>


From webhook-mailer at python.org  Fri Dec  3 17:01:24 2021
From: webhook-mailer at python.org (iritkatriel)
Date: Fri, 03 Dec 2021 22:01:24 -0000
Subject: [Python-checkins] bpo-45607: Make it possible to enrich exception
 displays via setting their __note__ field (GH-29880)
Message-ID: <mailman.28.1638568885.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/5bb7ef2768be5979b306e4c7552862b1746c251d
commit: 5bb7ef2768be5979b306e4c7552862b1746c251d
branch: main
author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com>
committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com>
date: 2021-12-03T22:01:15Z
summary:

bpo-45607: Make it possible to enrich exception displays via setting their __note__ field (GH-29880)

files:
A Misc/NEWS.d/next/Core and Builtins/2021-12-01-15-38-04.bpo-45607.JhuF8b.rst
M Doc/library/exceptions.rst
M Doc/whatsnew/3.11.rst
M Include/cpython/pyerrors.h
M Lib/test/test_exceptions.py
M Lib/test/test_sys.py
M Lib/test/test_traceback.py
M Lib/traceback.py
M Objects/exceptions.c
M Python/pythonrun.c

diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst
index 8fa82a98a199d..12d7d8abb2650 100644
--- a/Doc/library/exceptions.rst
+++ b/Doc/library/exceptions.rst
@@ -127,6 +127,14 @@ The following exceptions are used mostly as base classes for other exceptions.
              tb = sys.exc_info()[2]
              raise OtherException(...).with_traceback(tb)
 
+   .. attribute:: __note__
+
+      A mutable field which is :const:`None` by default and can be set to a string.
+      If it is not :const:`None`, it is included in the traceback. This field can
+      be used to enrich exceptions after they have been caught.
+
+   .. versionadded:: 3.11
+
 
 .. exception:: Exception
 
diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index 1ec629d8229cb..c498225591a74 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -146,6 +146,12 @@ The :option:`-X` ``no_debug_ranges`` option and the environment variable
 See :pep:`657` for more details. (Contributed by Pablo Galindo, Batuhan Taskaya
 and Ammar Askar in :issue:`43950`.)
 
+Exceptions can be enriched with a string ``__note__``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ``__note__`` field was added to :exc:`BaseException`. It is ``None``
+by default but can be set to a string which is added to the exception's
+traceback. (Contributed by Irit Katriel in :issue:`45607`.)
 
 Other Language Changes
 ======================
diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h
index a07018abae0cf..5281fde1f1a54 100644
--- a/Include/cpython/pyerrors.h
+++ b/Include/cpython/pyerrors.h
@@ -6,7 +6,7 @@
 
 /* PyException_HEAD defines the initial segment of every exception class. */
 #define PyException_HEAD PyObject_HEAD PyObject *dict;\
-             PyObject *args; PyObject *traceback;\
+             PyObject *args; PyObject *note; PyObject *traceback;\
              PyObject *context; PyObject *cause;\
              char suppress_context;
 
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index c6660043c805f..e4b7b8f0a6406 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -516,6 +516,27 @@ def testAttributes(self):
                                              'pickled "%r", attribute "%s' %
                                              (e, checkArgName))
 
+    def test_note(self):
+        for e in [BaseException(1), Exception(2), ValueError(3)]:
+            with self.subTest(e=e):
+                self.assertIsNone(e.__note__)
+                e.__note__ = "My Note"
+                self.assertEqual(e.__note__, "My Note")
+
+                with self.assertRaises(TypeError):
+                    e.__note__ = 42
+                self.assertEqual(e.__note__, "My Note")
+
+                e.__note__ = "Your Note"
+                self.assertEqual(e.__note__, "Your Note")
+
+                with self.assertRaises(TypeError):
+                    del e.__note__
+                self.assertEqual(e.__note__, "Your Note")
+
+                e.__note__ = None
+                self.assertIsNone(e.__note__)
+
     def testWithTraceback(self):
         try:
             raise IndexError(4)
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index db8d0082085cb..2b1ba2457f50d 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -1298,13 +1298,13 @@ def inner():
         class C(object): pass
         check(C.__dict__, size('P'))
         # BaseException
-        check(BaseException(), size('5Pb'))
+        check(BaseException(), size('6Pb'))
         # UnicodeEncodeError
-        check(UnicodeEncodeError("", "", 0, 0, ""), size('5Pb 2P2nP'))
+        check(UnicodeEncodeError("", "", 0, 0, ""), size('6Pb 2P2nP'))
         # UnicodeDecodeError
-        check(UnicodeDecodeError("", b"", 0, 0, ""), size('5Pb 2P2nP'))
+        check(UnicodeDecodeError("", b"", 0, 0, ""), size('6Pb 2P2nP'))
         # UnicodeTranslateError
-        check(UnicodeTranslateError("", 0, 1, ""), size('5Pb 2P2nP'))
+        check(UnicodeTranslateError("", 0, 1, ""), size('6Pb 2P2nP'))
         # ellipses
         check(Ellipsis, size(''))
         # EncodingMap
diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py
index cde35f5dacb2d..a458b21b09466 100644
--- a/Lib/test/test_traceback.py
+++ b/Lib/test/test_traceback.py
@@ -1224,6 +1224,22 @@ def test_syntax_error_various_offsets(self):
                 exp = "\n".join(expected)
                 self.assertEqual(exp, err)
 
+    def test_exception_with_note(self):
+        e = ValueError(42)
+        vanilla = self.get_report(e)
+
+        e.__note__ = 'My Note'
+        self.assertEqual(self.get_report(e), vanilla + 'My Note\n')
+
+        e.__note__ = ''
+        self.assertEqual(self.get_report(e), vanilla + '\n')
+
+        e.__note__ = 'Your Note'
+        self.assertEqual(self.get_report(e), vanilla + 'Your Note\n')
+
+        e.__note__ = None
+        self.assertEqual(self.get_report(e), vanilla)
+
     def test_exception_qualname(self):
         class A:
             class B:
@@ -1566,6 +1582,59 @@ def test_exception_group_depth_limit(self):
         report = self.get_report(exc)
         self.assertEqual(report, expected)
 
+    def test_exception_group_with_notes(self):
+        def exc():
+            try:
+                excs = []
+                for msg in ['bad value', 'terrible value']:
+                    try:
+                        raise ValueError(msg)
+                    except ValueError as e:
+                        e.__note__ = f'the {msg}'
+                        excs.append(e)
+                raise ExceptionGroup("nested", excs)
+            except ExceptionGroup as e:
+                e.__note__ = ('>> Multi line note\n'
+                              '>> Because I am such\n'
+                              '>> an important exception.\n'
+                              '>> empty lines work too\n'
+                              '\n'
+                              '(that was an empty line)')
+                raise
+
+        expected = (f'  + Exception Group Traceback (most recent call last):\n'
+                    f'  |   File "{__file__}", line {self.callable_line}, in get_exception\n'
+                    f'  |     exception_or_callable()\n'
+                    f'  |     ^^^^^^^^^^^^^^^^^^^^^^^\n'
+                    f'  |   File "{__file__}", line {exc.__code__.co_firstlineno + 9}, in exc\n'
+                    f'  |     raise ExceptionGroup("nested", excs)\n'
+                    f'  |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n'
+                    f'  | ExceptionGroup: nested\n'
+                    f'  | >> Multi line note\n'
+                    f'  | >> Because I am such\n'
+                    f'  | >> an important exception.\n'
+                    f'  | >> empty lines work too\n'
+                    f'  | \n'
+                    f'  | (that was an empty line)\n'
+                    f'  +-+---------------- 1 ----------------\n'
+                    f'    | Traceback (most recent call last):\n'
+                    f'    |   File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n'
+                    f'    |     raise ValueError(msg)\n'
+                    f'    |     ^^^^^^^^^^^^^^^^^^^^^\n'
+                    f'    | ValueError: bad value\n'
+                    f'    | the bad value\n'
+                    f'    +---------------- 2 ----------------\n'
+                    f'    | Traceback (most recent call last):\n'
+                    f'    |   File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n'
+                    f'    |     raise ValueError(msg)\n'
+                    f'    |     ^^^^^^^^^^^^^^^^^^^^^\n'
+                    f'    | ValueError: terrible value\n'
+                    f'    | the terrible value\n'
+                    f'    +------------------------------------\n')
+
+        report = self.get_report(exc)
+        self.assertEqual(report, expected)
+
 
 class PyExcReportingTests(BaseExceptionReportingTests, unittest.TestCase):
     #
diff --git a/Lib/traceback.py b/Lib/traceback.py
index 77f8590719eb8..b244750fd016e 100644
--- a/Lib/traceback.py
+++ b/Lib/traceback.py
@@ -685,6 +685,8 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None,
         # Capture now to permit freeing resources: only complication is in the
         # unofficial API _format_final_exc_line
         self._str = _some_str(exc_value)
+        self.__note__ = exc_value.__note__ if exc_value else None
+
         if exc_type and issubclass(exc_type, SyntaxError):
             # Handle SyntaxError's specially
             self.filename = exc_value.filename
@@ -816,6 +818,8 @@ def format_exception_only(self):
             yield _format_final_exc_line(stype, self._str)
         else:
             yield from self._format_syntax_error(stype)
+        if self.__note__ is not None:
+            yield from [l + '\n' for l in self.__note__.split('\n')]
 
     def _format_syntax_error(self, stype):
         """Format SyntaxError exceptions (internal helper)."""
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-01-15-38-04.bpo-45607.JhuF8b.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-01-15-38-04.bpo-45607.JhuF8b.rst
new file mode 100644
index 0000000000000..3e38c3e6f95e8
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-12-01-15-38-04.bpo-45607.JhuF8b.rst	
@@ -0,0 +1,4 @@
+The ``__note__`` field was added to :exc:`BaseException`. It is ``None``
+by default but can be set to a string which is added to the exception's
+traceback.
+
diff --git a/Objects/exceptions.c b/Objects/exceptions.c
index a5459da89a073..c99f17a30f169 100644
--- a/Objects/exceptions.c
+++ b/Objects/exceptions.c
@@ -46,6 +46,7 @@ BaseException_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         return NULL;
     /* the dict is created on the fly in PyObject_GenericSetAttr */
     self->dict = NULL;
+    self->note = NULL;
     self->traceback = self->cause = self->context = NULL;
     self->suppress_context = 0;
 
@@ -81,6 +82,7 @@ BaseException_clear(PyBaseExceptionObject *self)
 {
     Py_CLEAR(self->dict);
     Py_CLEAR(self->args);
+    Py_CLEAR(self->note);
     Py_CLEAR(self->traceback);
     Py_CLEAR(self->cause);
     Py_CLEAR(self->context);
@@ -105,6 +107,7 @@ BaseException_traverse(PyBaseExceptionObject *self, visitproc visit, void *arg)
 {
     Py_VISIT(self->dict);
     Py_VISIT(self->args);
+    Py_VISIT(self->note);
     Py_VISIT(self->traceback);
     Py_VISIT(self->cause);
     Py_VISIT(self->context);
@@ -216,6 +219,33 @@ BaseException_set_args(PyBaseExceptionObject *self, PyObject *val, void *Py_UNUS
     return 0;
 }
 
+static PyObject *
+BaseException_get_note(PyBaseExceptionObject *self, void *Py_UNUSED(ignored))
+{
+    if (self->note == NULL) {
+        Py_RETURN_NONE;
+    }
+    return Py_NewRef(self->note);
+}
+
+static int
+BaseException_set_note(PyBaseExceptionObject *self, PyObject *note,
+                       void *Py_UNUSED(ignored))
+{
+    if (note == NULL) {
+        PyErr_SetString(PyExc_TypeError, "__note__ may not be deleted");
+        return -1;
+    }
+    else if (note != Py_None && !PyUnicode_CheckExact(note)) {
+        PyErr_SetString(PyExc_TypeError, "__note__ must be a string or None");
+        return -1;
+    }
+
+    Py_INCREF(note);
+    Py_XSETREF(self->note, note);
+    return 0;
+}
+
 static PyObject *
 BaseException_get_tb(PyBaseExceptionObject *self, void *Py_UNUSED(ignored))
 {
@@ -306,6 +336,7 @@ BaseException_set_cause(PyObject *self, PyObject *arg, void *Py_UNUSED(ignored))
 static PyGetSetDef BaseException_getset[] = {
     {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict},
     {"args", (getter)BaseException_get_args, (setter)BaseException_set_args},
+    {"__note__", (getter)BaseException_get_note, (setter)BaseException_set_note},
     {"__traceback__", (getter)BaseException_get_tb, (setter)BaseException_set_tb},
     {"__context__", BaseException_get_context,
      BaseException_set_context, PyDoc_STR("exception context")},
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 2f68b214603e1..5a118b4821ec0 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -1083,6 +1083,41 @@ print_exception(struct exception_print_context *ctx, PyObject *value)
         PyErr_Clear();
     }
     err += PyFile_WriteString("\n", f);
+
+    if (err == 0 && PyExceptionInstance_Check(value)) {
+        _Py_IDENTIFIER(__note__);
+
+        PyObject *note = _PyObject_GetAttrId(value, &PyId___note__);
+        if (note == NULL) {
+            err = -1;
+        }
+        if (err == 0 && PyUnicode_Check(note)) {
+            _Py_static_string(PyId_newline, "\n");
+            PyObject *lines = PyUnicode_Split(
+                note, _PyUnicode_FromId(&PyId_newline), -1);
+            if (lines == NULL) {
+                err = -1;
+            }
+            else {
+                Py_ssize_t n = PyList_GET_SIZE(lines);
+                for (Py_ssize_t i = 0; i < n; i++) {
+                    if (err == 0) {
+                        PyObject *line = PyList_GET_ITEM(lines, i);
+                        assert(PyUnicode_Check(line));
+                        err = write_indented_margin(ctx, f);
+                        if (err == 0) {
+                            err = PyFile_WriteObject(line, f, Py_PRINT_RAW);
+                        }
+                        if (err == 0) {
+                            err = PyFile_WriteString("\n", f);
+                        }
+                    }
+                }
+            }
+            Py_DECREF(lines);
+        }
+        Py_XDECREF(note);
+    }
     Py_XDECREF(tb);
     Py_DECREF(value);
     /* If an error happened here, don't show it.


From webhook-mailer at python.org  Fri Dec  3 17:04:21 2021
From: webhook-mailer at python.org (zooba)
Date: Fri, 03 Dec 2021 22:04:21 -0000
Subject: [Python-checkins] bpo-45582: Add a NOT operator to the condition in
 getpath_isxfile (GH-29906)
Message-ID: <mailman.29.1638569062.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/7d7c91a8e8c0bb04105a21a17d1061ffc1c04d80
commit: 7d7c91a8e8c0bb04105a21a17d1061ffc1c04d80
branch: main
author: neonene <53406459+neonene at users.noreply.github.com>
committer: zooba <steve.dower at microsoft.com>
date: 2021-12-03T22:04:11Z
summary:

bpo-45582: Add a NOT operator to the condition in getpath_isxfile (GH-29906)

files:
M Modules/getpath.c

diff --git a/Modules/getpath.c b/Modules/getpath.c
index f66fc8abbd4ff..c8c85a8540d39 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -173,7 +173,9 @@ getpath_isdir(PyObject *Py_UNUSED(self), PyObject *args)
     path = PyUnicode_AsWideCharString(pathobj, NULL);
     if (path) {
 #ifdef MS_WINDOWS
-        r = (GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) ? Py_True : Py_False;
+        DWORD attr = GetFileAttributesW(path);
+        r = (attr != INVALID_FILE_ATTRIBUTES) &&
+            (attr & FILE_ATTRIBUTE_DIRECTORY) ? Py_True : Py_False;
 #else
         struct stat st;
         r = (_Py_wstat(path, &st) == 0) && S_ISDIR(st.st_mode) ? Py_True : Py_False;
@@ -197,7 +199,9 @@ getpath_isfile(PyObject *Py_UNUSED(self), PyObject *args)
     path = PyUnicode_AsWideCharString(pathobj, NULL);
     if (path) {
 #ifdef MS_WINDOWS
-        r = !(GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) ? Py_True : Py_False;
+        DWORD attr = GetFileAttributesW(path);
+        r = (attr != INVALID_FILE_ATTRIBUTES) &&
+            !(attr & FILE_ATTRIBUTE_DIRECTORY) ? Py_True : Py_False;
 #else
         struct stat st;
         r = (_Py_wstat(path, &st) == 0) && S_ISREG(st.st_mode) ? Py_True : Py_False;
@@ -223,7 +227,9 @@ getpath_isxfile(PyObject *Py_UNUSED(self), PyObject *args)
     if (path) {
 #ifdef MS_WINDOWS
         const wchar_t *ext;
-        r = (GetFileAttributesW(path) & FILE_ATTRIBUTE_DIRECTORY) &&
+        DWORD attr = GetFileAttributesW(path);
+        r = (attr != INVALID_FILE_ATTRIBUTES) &&
+            !(attr & FILE_ATTRIBUTE_DIRECTORY) &&
             SUCCEEDED(PathCchFindExtension(path, cchPath, &ext)) &&
             (CompareStringOrdinal(ext, -1, L".exe", -1, 1 /* ignore case */) == CSTR_EQUAL)
             ? Py_True : Py_False;


From webhook-mailer at python.org  Fri Dec  3 18:22:05 2021
From: webhook-mailer at python.org (miss-islington)
Date: Fri, 03 Dec 2021 23:22:05 -0000
Subject: [Python-checkins] bpo-45582 Fix prototype of
 _Py_Get_Getpath_CodeObject. (GH-29907)
Message-ID: <mailman.30.1638573726.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/0ae4e0c959bbc90ec18180ef3cc957759d346ced
commit: 0ae4e0c959bbc90ec18180ef3cc957759d346ced
branch: main
author: Benjamin Peterson <benjamin at python.org>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-03T15:21:58-08:00
summary:

bpo-45582 Fix prototype of _Py_Get_Getpath_CodeObject. (GH-29907)



Automerge-Triggered-By: GH:tiran

files:
M Include/internal/pycore_initconfig.h

diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h
index be545f495dc0f..a2f4cd182e8e9 100644
--- a/Include/internal/pycore_initconfig.h
+++ b/Include/internal/pycore_initconfig.h
@@ -168,7 +168,7 @@ PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict);
 
 extern void _Py_DumpPathConfig(PyThreadState *tstate);
 
-PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject();
+PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject(void);
 
 
 /* --- Function used for testing ---------------------------------- */


From webhook-mailer at python.org  Fri Dec  3 21:37:16 2021
From: webhook-mailer at python.org (rhettinger)
Date: Sat, 04 Dec 2021 02:37:16 -0000
Subject: [Python-checkins] bpo-20751: Replace method example with attribute
 example, matching the descriptor howto (GH-29909)
Message-ID: <mailman.31.1638585437.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/135ecc3492cee259090fd4aaed9056c130cd2eba
commit: 135ecc3492cee259090fd4aaed9056c130cd2eba
branch: main
author: Raymond Hettinger <rhettinger at users.noreply.github.com>
committer: rhettinger <rhettinger at users.noreply.github.com>
date: 2021-12-03T20:37:08-06:00
summary:

bpo-20751:  Replace method example with attribute example, matching the descriptor howto (GH-29909)

files:
M Doc/reference/datamodel.rst

diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index b14b2cb1ef5e4..41092458102de 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -1818,10 +1818,38 @@ Class Binding
    ``A.__dict__['x'].__get__(None, A)``.
 
 Super Binding
-   If ``a`` is an instance of :class:`super`, then the binding ``super(B, obj).m()``
-   searches ``obj.__class__.__mro__`` for the base class ``A``
-   immediately following ``B`` and then invokes the descriptor with the call:
-   ``A.__dict__['m'].__get__(obj, obj.__class__)``.
+   A dotted lookup such as ``super(A, a).x`` searches
+   ``obj.__class__.__mro__`` for a base class ``B`` following ``A`` and then
+   returns ``B.__dict__['x'].__get__(a, A)``.  If not a descriptor, ``x`` is
+   returned unchanged.
+
+.. testcode::
+    :hide:
+
+    class Desc:
+        def __get__(*args):
+            return args
+
+    class B:
+
+        x = Desc()
+
+    class A(B):
+
+        x = 999
+
+        def m(self):
+            'Demonstrate these two calls are equivalent'
+            result1 = super(A, a).x
+            result2 = B.__dict__['x'].__get__(a, A)
+            return result1 == result2
+
+.. doctest::
+    :hide:
+
+    >>> a = A()
+    >>> a.m()
+    True
 
 For instance bindings, the precedence of descriptor invocation depends on
 which descriptor methods are defined.  A descriptor can define any combination


From webhook-mailer at python.org  Sat Dec  4 00:28:39 2021
From: webhook-mailer at python.org (terryjreedy)
Date: Sat, 04 Dec 2021 05:28:39 -0000
Subject: [Python-checkins] bpo-45916: Use HTTPS link for The Perils of
 Floating Point (GH-29896)
Message-ID: <mailman.32.1638595720.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/257eea55860c5f35acce49c062c5823c7a3a1317
commit: 257eea55860c5f35acce49c062c5823c7a3a1317
branch: 3.9
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: terryjreedy <tjreedy at udel.edu>
date: 2021-12-04T00:28:30-05:00
summary:

bpo-45916: Use HTTPS link for The Perils of Floating Point (GH-29896)

(cherry picked from commit 9f2f7e42269db74a89fc8cd74d82a875787f01d7)

Co-authored-by: Zachary Ware <zach at python.org>

files:
M Doc/tutorial/floatingpoint.rst

diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst
index b98de6e56a003..7212b40be8377 100644
--- a/Doc/tutorial/floatingpoint.rst
+++ b/Doc/tutorial/floatingpoint.rst
@@ -133,7 +133,7 @@ with inexact values become comparable to one another::
 
 Binary floating-point arithmetic holds many surprises like this.  The problem
 with "0.1" is explained in precise detail below, in the "Representation Error"
-section.  See `The Perils of Floating Point <http://www.lahey.com/float.htm>`_
+section.  See `The Perils of Floating Point <https://www.lahey.com/float.htm>`_
 for a more complete account of other common surprises.
 
 As that says near the end, "there are no easy answers."  Still, don't be unduly


From webhook-mailer at python.org  Sat Dec  4 05:08:04 2021
From: webhook-mailer at python.org (tiran)
Date: Sat, 04 Dec 2021 10:08:04 -0000
Subject: [Python-checkins] bpo-45695: Test out-of-tree builds on GHA
 (GH-29904)
Message-ID: <mailman.33.1638612485.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/cee07b162843694e8166ad8715162d4d5886b50f
commit: cee07b162843694e8166ad8715162d4d5886b50f
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-04T11:07:59+01:00
summary:

bpo-45695: Test out-of-tree builds on GHA (GH-29904)

files:
A Misc/NEWS.d/next/Tests/2021-12-03-14-19-16.bpo-45695.QKBn2E.rst
M .github/workflows/build.yml

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fe4b14828699c..fbf7bb6fe2128 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -179,13 +179,28 @@ jobs:
         echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
     - name: Configure ccache action
       uses: hendrikmuhs/ccache-action at v1
-    - name: Configure CPython
-      run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR
-    - name: Build CPython
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: Create directories for read-only out-of-tree builds
+      run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
+    - name: Bind mount sources read-only
+      run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Configure CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+    - name: Build CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make -j4
     - name: Display build info
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make pythoninfo
+    - name: Remount sources writable for tests
+      # some tests write to srcdir, lack of pyc files slows down testing
+      run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
     - name: Tests
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
 
   build_ubuntu_ssltests:
diff --git a/Misc/NEWS.d/next/Tests/2021-12-03-14-19-16.bpo-45695.QKBn2E.rst b/Misc/NEWS.d/next/Tests/2021-12-03-14-19-16.bpo-45695.QKBn2E.rst
new file mode 100644
index 0000000000000..14ecd923dbd55
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2021-12-03-14-19-16.bpo-45695.QKBn2E.rst
@@ -0,0 +1 @@
+Out-of-tree builds with a read-only source directory are now tested by CI.


From webhook-mailer at python.org  Sat Dec  4 05:21:50 2021
From: webhook-mailer at python.org (tiran)
Date: Sat, 04 Dec 2021 10:21:50 -0000
Subject: [Python-checkins] bpo-45847: Port _ctypes partly to PY_STDLIB_MOD
 (GH-29747)
Message-ID: <mailman.34.1638613311.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/4045392e0e3446362841b3336497cb6eeccfcd23
commit: 4045392e0e3446362841b3336497cb6eeccfcd23
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-04T11:21:43+01:00
summary:

bpo-45847: Port _ctypes partly to PY_STDLIB_MOD (GH-29747)

Co-authored-by: Erlend Egeberg Aasland <erlend.aasland at innova.no>

files:
M Modules/Setup.stdlib.in
M configure
M configure.ac
M pyconfig.h.in
M setup.py

diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in
index 9dfe918d05a09..ff92db3cc3784 100644
--- a/Modules/Setup.stdlib.in
+++ b/Modules/Setup.stdlib.in
@@ -130,6 +130,10 @@
 ############################################################################
 # Modules with third party dependencies
 #
+
+# needs -lffi and -ldl
+ at MODULE__CTYPES_TRUE@_ctypes _ctypes/_ctypes.c _ctypes/callbacks.c _ctypes/callproc.c _ctypes/stgdict.c _ctypes/cfield.c
+
 @MODULE__SQLITE3_TRUE at _sqlite3 _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c
 
 # needs -lssl and -lcrypt
diff --git a/configure b/configure
index 5d47d03542e40..8582224dfd28f 100755
--- a/configure
+++ b/configure
@@ -664,6 +664,8 @@ MODULE__GDBM_FALSE
 MODULE__GDBM_TRUE
 MODULE__DECIMAL_FALSE
 MODULE__DECIMAL_TRUE
+MODULE__CTYPES_FALSE
+MODULE__CTYPES_TRUE
 MODULE__CRYPT_FALSE
 MODULE__CRYPT_TRUE
 MODULE__BLAKE2_FALSE
@@ -828,6 +830,8 @@ LIBMPDEC_INTERNAL
 LIBMPDEC_LDFLAGS
 LIBMPDEC_CFLAGS
 LIBFFI_INCLUDEDIR
+LIBFFI_LIBS
+LIBFFI_CFLAGS
 LIBEXPAT_INTERNAL
 LIBEXPAT_LDFLAGS
 LIBEXPAT_CFLAGS
@@ -1060,6 +1064,8 @@ CPP
 PROFILE_TASK
 LIBUUID_CFLAGS
 LIBUUID_LIBS
+LIBFFI_CFLAGS
+LIBFFI_LIBS
 LIBNSL_CFLAGS
 LIBNSL_LIBS
 LIBSQLITE3_CFLAGS
@@ -1857,6 +1863,9 @@ Some influential environment variables:
               C compiler flags for LIBUUID, overriding pkg-config
   LIBUUID_LIBS
               linker flags for LIBUUID, overriding pkg-config
+  LIBFFI_CFLAGS
+              C compiler flags for LIBFFI, overriding pkg-config
+  LIBFFI_LIBS linker flags for LIBFFI, overriding pkg-config
   LIBNSL_CFLAGS
               C compiler flags for LIBNSL, overriding pkg-config
   LIBNSL_LIBS linker flags for LIBNSL, overriding pkg-config
@@ -11375,6 +11384,335 @@ $as_echo "$as_me: WARNING: --with(out)-system-ffi is ignored on this platform" >
     with_system_ffi="yes"
 fi
 
+have_libffi=missing
+if test "x$with_system_ffi" = xyes; then :
+
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBFFI" >&5
+$as_echo_n "checking for LIBFFI... " >&6; }
+
+if test -n "$LIBFFI_CFLAGS"; then
+    pkg_cv_LIBFFI_CFLAGS="$LIBFFI_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libffi\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "libffi") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LIBFFI_CFLAGS=`$PKG_CONFIG --cflags "libffi" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+if test -n "$LIBFFI_LIBS"; then
+    pkg_cv_LIBFFI_LIBS="$LIBFFI_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+    if test -n "$PKG_CONFIG" && \
+    { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libffi\""; } >&5
+  ($PKG_CONFIG --exists --print-errors "libffi") 2>&5
+  ac_status=$?
+  $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+  test $ac_status = 0; }; then
+  pkg_cv_LIBFFI_LIBS=`$PKG_CONFIG --libs "libffi" 2>/dev/null`
+		      test "x$?" != "x0" && pkg_failed=yes
+else
+  pkg_failed=yes
+fi
+ else
+    pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+   	{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+        _pkg_short_errors_supported=yes
+else
+        _pkg_short_errors_supported=no
+fi
+        if test $_pkg_short_errors_supported = yes; then
+	        LIBFFI_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libffi" 2>&1`
+        else
+	        LIBFFI_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libffi" 2>&1`
+        fi
+	# Put the nasty error message in config.log where it belongs
+	echo "$LIBFFI_PKG_ERRORS" >&5
+
+
+    ac_fn_c_check_header_mongrel "$LINENO" "ffi.h" "ac_cv_header_ffi_h" "$ac_includes_default"
+if test "x$ac_cv_header_ffi_h" = xyes; then :
+
+      save_CFLAGS=$CFLAGS
+save_CPPFLAGS=$CPPFLAGS
+save_LDFLAGS=$LDFLAGS
+save_LIBS=$LIBS
+
+
+        { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ffi_call in -lffi" >&5
+$as_echo_n "checking for ffi_call in -lffi... " >&6; }
+if ${ac_cv_lib_ffi_ffi_call+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lffi  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ffi_call ();
+int
+main ()
+{
+return ffi_call ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_ffi_ffi_call=yes
+else
+  ac_cv_lib_ffi_ffi_call=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5
+$as_echo "$ac_cv_lib_ffi_ffi_call" >&6; }
+if test "x$ac_cv_lib_ffi_ffi_call" = xyes; then :
+  have_libffi=yes
+else
+  have_libffi=no
+fi
+
+
+CFLAGS=$save_CFLAGS
+CPPFLAGS=$save_CPPFLAGS
+LDFLAGS=$save_LDFLAGS
+LIBS=$save_LIBS
+
+
+
+fi
+
+
+
+elif test $pkg_failed = untried; then
+     	{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+    ac_fn_c_check_header_mongrel "$LINENO" "ffi.h" "ac_cv_header_ffi_h" "$ac_includes_default"
+if test "x$ac_cv_header_ffi_h" = xyes; then :
+
+      save_CFLAGS=$CFLAGS
+save_CPPFLAGS=$CPPFLAGS
+save_LDFLAGS=$LDFLAGS
+save_LIBS=$LIBS
+
+
+        { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ffi_call in -lffi" >&5
+$as_echo_n "checking for ffi_call in -lffi... " >&6; }
+if ${ac_cv_lib_ffi_ffi_call+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lffi  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char ffi_call ();
+int
+main ()
+{
+return ffi_call ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_ffi_ffi_call=yes
+else
+  ac_cv_lib_ffi_ffi_call=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ffi_ffi_call" >&5
+$as_echo "$ac_cv_lib_ffi_ffi_call" >&6; }
+if test "x$ac_cv_lib_ffi_ffi_call" = xyes; then :
+  have_libffi=yes
+else
+  have_libffi=no
+fi
+
+
+CFLAGS=$save_CFLAGS
+CPPFLAGS=$save_CPPFLAGS
+LDFLAGS=$save_LDFLAGS
+LIBS=$save_LIBS
+
+
+
+fi
+
+
+
+else
+	LIBFFI_CFLAGS=$pkg_cv_LIBFFI_CFLAGS
+	LIBFFI_LIBS=$pkg_cv_LIBFFI_LIBS
+        { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+	have_libffi=yes
+fi
+
+else
+
+    have_libffi=yes
+  LIBFFI_CFLAGS="-I\$(srcdir)/Modules/_ctypes/darwin -DUSING_MALLOC_CLOSURE_DOT_C=1 -DMACOSX"
+  LIBFFI_LIBS=
+
+fi
+
+case $LIBS in #(
+  *-ldl*) :
+    LIBFFI_LIBS="$LIBFFI_LIBS -ldl"
+ ;; #(
+  *) :
+     ;;
+esac
+
+if test "$with_system_ffi" = yes -a "$have_libffi" = yes; then
+  save_CFLAGS=$CFLAGS
+save_CPPFLAGS=$CPPFLAGS
+save_LDFLAGS=$LDFLAGS
+save_LIBS=$LIBS
+
+
+    CFLAGS="$LIBFFI_CFLAGS $CFLAGS"
+    LIBS="$LIBFFI_LIBS $LIBS"
+
+    { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ffi_prep_cif_var in ffi.h" >&5
+$as_echo_n "checking for ffi_prep_cif_var in ffi.h... " >&6; }
+if ${ac_cv_ffi_prep_cif_var+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+
+      cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <ffi.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "ffi_prep_cif_var" >/dev/null 2>&1; then :
+  ac_cv_ffi_prep_cif_var=yes
+else
+  ac_cv_ffi_prep_cif_var=no
+fi
+rm -f conftest*
+
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_ffi_prep_cif_var" >&5
+$as_echo "$ac_cv_ffi_prep_cif_var" >&6; }
+    if test "x$ac_cv_ffi_prep_cif_var" = xyes; then :
+
+
+$as_echo "#define HAVE_FFI_PREP_CIF_VAR 1" >>confdefs.h
+
+
+fi
+
+    { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ffi_prep_closure_loc in ffi.h" >&5
+$as_echo_n "checking for ffi_prep_closure_loc in ffi.h... " >&6; }
+if ${ac_cv_ffi_prep_closure_loc+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+
+      cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <ffi.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "ffi_prep_closure_loc" >/dev/null 2>&1; then :
+  ac_cv_ffi_prep_closure_loc=yes
+else
+  ac_cv_ffi_prep_closure_loc=no
+fi
+rm -f conftest*
+
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_ffi_prep_closure_loc" >&5
+$as_echo "$ac_cv_ffi_prep_closure_loc" >&6; }
+    if test "x$ac_cv_ffi_prep_closure_loc" = xyes; then :
+
+
+$as_echo "#define HAVE_FFI_PREP_CLOSURE_LOC 1" >>confdefs.h
+
+
+fi
+
+    { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ffi_closure_alloc in ffi.h" >&5
+$as_echo_n "checking for ffi_closure_alloc in ffi.h... " >&6; }
+if ${ac_cv_ffi_closure_alloc+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+
+      cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#include <ffi.h>
+
+_ACEOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  $EGREP "ffi_closure_alloc" >/dev/null 2>&1; then :
+  ac_cv_ffi_closure_alloc=yes
+else
+  ac_cv_ffi_closure_alloc=no
+fi
+rm -f conftest*
+
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_ffi_closure_alloc" >&5
+$as_echo "$ac_cv_ffi_closure_alloc" >&6; }
+    if test "x$ac_cv_ffi_closure_alloc" = xyes; then :
+
+
+$as_echo "#define HAVE_FFI_CLOSURE_ALLOC 1" >>confdefs.h
+
+
+fi
+
+CFLAGS=$save_CFLAGS
+CPPFLAGS=$save_CPPFLAGS
+LDFLAGS=$save_LDFLAGS
+LIBS=$save_LIBS
+
+
+fi
+
+
 if test "$with_system_ffi" = "yes" && test -n "$PKG_CONFIG"; then
     LIBFFI_INCLUDEDIR="`"$PKG_CONFIG" libffi --cflags-only-I 2>/dev/null | sed -e 's/^-I//;s/ *$//'`"
 else
@@ -22726,6 +23064,42 @@ fi
 $as_echo "$py_cv_module__crypt" >&6; }
 
 
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _ctypes" >&5
+$as_echo_n "checking for stdlib extension module _ctypes... " >&6; }
+      case $py_stdlib_not_available in #(
+  *_ctypes*) :
+    py_cv_module__ctypes=n/a ;; #(
+  *) :
+    if true; then :
+  if test "$have_libffi" = yes; then :
+  py_cv_module__ctypes=yes
+else
+  py_cv_module__ctypes=missing
+fi
+else
+  py_cv_module__ctypes=disabled
+fi
+   ;;
+esac
+  as_fn_append MODULE_BLOCK "MODULE__CTYPES=$py_cv_module__ctypes$as_nl"
+  if test "x$py_cv_module__ctypes" = xyes; then :
+
+    as_fn_append MODULE_BLOCK "MODULE__CTYPES_CFLAGS=$LIBFFI_CFLAGS$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__CTYPES_LDFLAGS=$LIBFFI_LIBS$as_nl"
+
+fi
+   if test "$py_cv_module__ctypes" = yes; then
+  MODULE__CTYPES_TRUE=
+  MODULE__CTYPES_FALSE='#'
+else
+  MODULE__CTYPES_TRUE='#'
+  MODULE__CTYPES_FALSE=
+fi
+
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__ctypes" >&5
+$as_echo "$py_cv_module__ctypes" >&6; }
+
+
   { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _decimal" >&5
 $as_echo_n "checking for stdlib extension module _decimal... " >&6; }
       case $py_stdlib_not_available in #(
@@ -23336,7 +23710,7 @@ $as_echo_n "checking for stdlib extension module _ctypes_test... " >&6; }
     py_cv_module__ctypes_test=n/a ;; #(
   *) :
     if test "$TEST_MODULES" = yes; then :
-  if true; then :
+  if test "$have_libffi" = yes; then :
   py_cv_module__ctypes_test=yes
 else
   py_cv_module__ctypes_test=missing
@@ -23350,7 +23724,7 @@ esac
   if test "x$py_cv_module__ctypes_test" = xyes; then :
 
 
-    as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=-lm$as_nl"
+    as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=$LIBM$as_nl"
 
 fi
    if test "$py_cv_module__ctypes_test" = yes; then
@@ -23774,6 +24148,10 @@ if test -z "${MODULE__CRYPT_TRUE}" && test -z "${MODULE__CRYPT_FALSE}"; then
   as_fn_error $? "conditional \"MODULE__CRYPT\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5
 fi
+if test -z "${MODULE__CTYPES_TRUE}" && test -z "${MODULE__CTYPES_FALSE}"; then
+  as_fn_error $? "conditional \"MODULE__CTYPES\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
 if test -z "${MODULE__DECIMAL_TRUE}" && test -z "${MODULE__DECIMAL_FALSE}"; then
   as_fn_error $? "conditional \"MODULE__DECIMAL\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5
diff --git a/configure.ac b/configure.ac
index b35c92acb85db..40dc6f425f26f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3175,6 +3175,60 @@ else
     with_system_ffi="yes"
 fi
 
+dnl detect libffi
+have_libffi=missing
+AS_VAR_IF([with_system_ffi], [yes], [
+  PKG_CHECK_MODULES([LIBFFI], [libffi], [have_libffi=yes], [
+    AC_CHECK_HEADER([ffi.h], [
+      WITH_SAVE_ENV([
+        AC_CHECK_LIB([ffi], [ffi_call], [have_libffi=yes], [have_libffi=no])
+      ])
+    ])
+  ])
+], [
+  dnl private ffi copy
+  have_libffi=yes
+  LIBFFI_CFLAGS="-I\$(srcdir)/Modules/_ctypes/darwin -DUSING_MALLOC_CLOSURE_DOT_C=1 -DMACOSX"
+  LIBFFI_LIBS=
+])
+
+dnl _ctypes needs -ldl for dlopen
+AS_CASE([$LIBS],
+  [*-ldl*], [LIBFFI_LIBS="$LIBFFI_LIBS -ldl"]
+)
+
+if test "$with_system_ffi" = yes -a "$have_libffi" = yes; then
+  WITH_SAVE_ENV([
+    CFLAGS="$LIBFFI_CFLAGS $CFLAGS"
+    LIBS="$LIBFFI_LIBS $LIBS"
+
+    AC_CACHE_CHECK([for ffi_prep_cif_var in ffi.h], [ac_cv_ffi_prep_cif_var], [
+      AC_EGREP_HEADER([ffi_prep_cif_var], [ffi.h], [ac_cv_ffi_prep_cif_var=yes], [ac_cv_ffi_prep_cif_var=no])
+    ])
+    AS_VAR_IF([ac_cv_ffi_prep_cif_var], [yes], [
+      AC_DEFINE([HAVE_FFI_PREP_CIF_VAR], [1],
+      [Define to 1 if you have the ffi_prep_cif_var function in <ffi.h> header file.])
+    ])
+
+    AC_CACHE_CHECK([for ffi_prep_closure_loc in ffi.h], [ac_cv_ffi_prep_closure_loc], [
+      AC_EGREP_HEADER([ffi_prep_closure_loc], [ffi.h], [ac_cv_ffi_prep_closure_loc=yes], [ac_cv_ffi_prep_closure_loc=no])
+    ])
+    AS_VAR_IF([ac_cv_ffi_prep_closure_loc], [yes], [
+      AC_DEFINE([HAVE_FFI_PREP_CLOSURE_LOC], [1],
+      [Define to 1 if you have the ffi_prep_closure_loc function in <ffi.h> header file.])
+    ])
+
+    AC_CACHE_CHECK([for ffi_closure_alloc in ffi.h], [ac_cv_ffi_closure_alloc], [
+      AC_EGREP_HEADER([ffi_closure_alloc], [ffi.h], [ac_cv_ffi_closure_alloc=yes], [ac_cv_ffi_closure_alloc=no])
+    ])
+    AS_VAR_IF([ac_cv_ffi_closure_alloc], [yes], [
+      AC_DEFINE([HAVE_FFI_CLOSURE_ALLOC], [1],
+      [Define to 1 if you have the ffi_closure_alloc function in <ffi.h> header file.])
+    ])
+  ])
+fi
+
+
 if test "$with_system_ffi" = "yes" && test -n "$PKG_CONFIG"; then
     LIBFFI_INCLUDEDIR="`"$PKG_CONFIG" libffi --cflags-only-I 2>/dev/null | sed -e 's/^-I//;s/ *$//'`"
 else
@@ -6423,6 +6477,9 @@ PY_STDLIB_MOD([_blake2], [test "$with_builtin_blake2" = yes])
 PY_STDLIB_MOD([_crypt],
   [], [test "$ac_cv_crypt_crypt" = yes],
   [$LIBCRYPT_CFLAGS], [$LIBCRYPT_LIBS])
+PY_STDLIB_MOD([_ctypes],
+  [], [test "$have_libffi" = yes],
+  [$LIBFFI_CFLAGS], [$LIBFFI_LIBS])
 PY_STDLIB_MOD([_decimal], [], [], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LDFLAGS])
 PY_STDLIB_MOD([_gdbm],
   [test "$have_gdbm_dbmliborder" = yes], [test "$have_gdbm" = yes],
@@ -6461,7 +6518,9 @@ PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes])
 PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes])
 PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes])
 PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes])
-PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [], [], [-lm])
+PY_STDLIB_MOD([_ctypes_test],
+  [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes],
+  [], [$LIBM])
 
 dnl Limited API template modules.
 dnl The limited C API is not compatible with the Py_TRACE_REFS macro.
diff --git a/pyconfig.h.in b/pyconfig.h.in
index efad243d0af8a..c6cc1fd377762 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -356,6 +356,18 @@
 /* Define to 1 if you have the `fexecve' function. */
 #undef HAVE_FEXECVE
 
+/* Define to 1 if you have the ffi_closure_alloc function in <ffi.h> header
+   file. */
+#undef HAVE_FFI_CLOSURE_ALLOC
+
+/* Define to 1 if you have the ffi_prep_cif_var function in <ffi.h> header
+   file. */
+#undef HAVE_FFI_PREP_CIF_VAR
+
+/* Define to 1 if you have the ffi_prep_closure_loc function in <ffi.h> header
+   file. */
+#undef HAVE_FFI_PREP_CLOSURE_LOC
+
 /* Define to 1 if you have the `flock' function. */
 #undef HAVE_FLOCK
 
diff --git a/setup.py b/setup.py
index 572f2ac1783b8..ca5d0774e001a 100644
--- a/setup.py
+++ b/setup.py
@@ -669,12 +669,6 @@ def print_three_column(lst):
             raise RuntimeError("Failed to build some stdlib modules")
 
     def build_extension(self, ext):
-
-        if ext.name == '_ctypes':
-            if not self.configure_ctypes(ext):
-                self.failed.append(ext.name)
-                return
-
         try:
             build_ext.build_extension(self, ext)
         except (CCompilerError, DistutilsError) as why:
@@ -1738,10 +1732,26 @@ def detect_tkinter(self):
                            library_dirs=added_lib_dirs))
         return True
 
-    def configure_ctypes(self, ext):
-        return True
-
     def detect_ctypes(self):
+        ext = Extension(
+            '_ctypes',
+            [
+                '_ctypes/_ctypes.c',
+                '_ctypes/callbacks.c',
+                '_ctypes/callproc.c',
+                '_ctypes/stgdict.c',
+                '_ctypes/cfield.c',
+            ]
+        )
+        if MACOS:
+            self._build_ctypes_macos(ext)
+        else:
+            self.use_system_libffi = True
+            self.addext(ext)
+
+        self.addext(Extension('_ctypes_test', ['_ctypes/_ctypes_test.c']))
+
+    def _build_ctypes_macos(self, ext):
         # Thomas Heller's _ctypes module
 
         if (not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and MACOS):
@@ -1749,20 +1759,11 @@ def detect_ctypes(self):
         else:
             self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS")
 
-        include_dirs = []
-        extra_compile_args = []
-        extra_link_args = []
-        sources = ['_ctypes/_ctypes.c',
-                   '_ctypes/callbacks.c',
-                   '_ctypes/callproc.c',
-                   '_ctypes/stgdict.c',
-                   '_ctypes/cfield.c']
-
         if MACOS:
-            sources.append('_ctypes/malloc_closure.c')
-            extra_compile_args.append('-DUSING_MALLOC_CLOSURE_DOT_C=1')
-            extra_compile_args.append('-DMACOSX')
-            include_dirs.append('_ctypes/darwin')
+            ext.sources.append('_ctypes/malloc_closure.c')
+            ext.extra_compile_args.append('-DUSING_MALLOC_CLOSURE_DOT_C=1')
+            ext.extra_compile_args.append('-DMACOSX')
+            ext.include_dirs.append('_ctypes/darwin')
 
         elif HOST_PLATFORM == 'sunos5':
             # XXX This shouldn't be necessary; it appears that some
@@ -1773,20 +1774,12 @@ def detect_ctypes(self):
             # this option. If you want to compile ctypes with the Sun
             # compiler, please research a proper solution, instead of
             # finding some -z option for the Sun compiler.
-            extra_link_args.append('-mimpure-text')
+            ext.extra_link_args.append('-mimpure-text')
 
         elif HOST_PLATFORM.startswith('hp-ux'):
-            extra_link_args.append('-fPIC')
-
-        ext = Extension('_ctypes',
-                        include_dirs=include_dirs,
-                        extra_compile_args=extra_compile_args,
-                        extra_link_args=extra_link_args,
-                        libraries=[],
-                        sources=sources)
+            ext.extra_link_args.append('-fPIC')
+
         self.add(ext)
-        # function my_sqrt() needs libm for sqrt()
-        self.addext(Extension('_ctypes_test', ['_ctypes/_ctypes_test.c']))
 
         ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR")
         ffi_lib = None


From webhook-mailer at python.org  Sat Dec  4 09:14:57 2021
From: webhook-mailer at python.org (tiran)
Date: Sat, 04 Dec 2021 14:14:57 -0000
Subject: [Python-checkins] bpo-45847: Update whatsnew and add place holder
 entries for missing extensions (GH-29914)
Message-ID: <mailman.35.1638627298.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/64be8d369b7e3878078dbef466804ae8be49c1cf
commit: 64be8d369b7e3878078dbef466804ae8be49c1cf
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-04T15:14:48+01:00
summary:

bpo-45847: Update whatsnew and add place holder entries for missing extensions (GH-29914)

files:
M Doc/whatsnew/3.11.rst
M Modules/Setup.stdlib.in
M configure.ac

diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index c498225591a74..10dc30939414c 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -583,13 +583,12 @@ Build Changes
   (Contributed by Brett Cannon and Christian Heimes in :issue:`45548`,
   :issue:`45570`, :issue:`45571`, and :issue:`43974`.)
 
-* The build dependencies for :mod:`zlib`, :mod:`bz2`, and :mod:`lzma` are now
-  detected by :program:`configure`.
-  (Contributed by Christian Heimes in :issue:`45763`.)
-
-* Build dependencies for :mod:`dbm` are now detected by :program:`configure`.
-  ``libdb`` 3.x and 4.x are no longer supported.
-  (Contributed by Christian Heimes in :issue:`45747`.)
+* Build dependencies, compiler flags, and linker flags for most stdlib
+  extension modules are now detected by :program:`configure`. libffi, libnsl,
+  libsqlite3, zlib, bzip2, liblzma, libcrypt, and uuid flags are detected by
+  ``pkg-config`` (when available).
+  (Contributed by Christian Heimes and Erlend Egeberg Aasland in
+  :issue:`bpo-45847`, :issue:`45747`, and :issue:`45763`.)
 
 * CPython now has experimental support for cross compiling to WebAssembly
   platform ``wasm32-emscripten``. The effort is inspired by previous work
diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in
index ff92db3cc3784..eaf624d8bc579 100644
--- a/Modules/Setup.stdlib.in
+++ b/Modules/Setup.stdlib.in
@@ -68,9 +68,13 @@
 
 # dbm/gdbm
 # dbm needs either libndbm, libgdbm_compat, or libdb 5.x
+#@MODULE__DBM_TRUE at _dbm _dbmmodule.c
 # gdbm module needs -lgdbm
 @MODULE__GDBM_TRUE at _gdbm _gdbmmodule.c
 
+# needs -lreadline or -leditline, sometimes termcap, termlib, or tinfo
+#@MODULE_READLINE_TRUE at readline readline.c
+
 # hashing builtins, can be disabled with --without-builtin-hashlib-hashes
 @MODULE__MD5_TRUE at _md5 md5module.c
 @MODULE__SHA1_TRUE at _sha1 sha1module.c
@@ -134,6 +138,11 @@
 # needs -lffi and -ldl
 @MODULE__CTYPES_TRUE at _ctypes _ctypes/_ctypes.c _ctypes/callbacks.c _ctypes/callproc.c _ctypes/stgdict.c _ctypes/cfield.c
 
+# needs -lncurses, -lncursesw or -lcurses, sometimes -ltermcap
+#@MODULE__CURSES_TRUE at _curses _cursesmodule.c
+# needs -lncurses and -lpanel
+#@MODULE__CURSES_PANEL_TRUE at _curses_panel _curses_panel.c
+
 @MODULE__SQLITE3_TRUE at _sqlite3 _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c
 
 # needs -lssl and -lcrypt
@@ -141,6 +150,9 @@
 # needs -lcrypt
 @MODULE__HASHLIB_TRUE at _hashlib _hashopenssl.c
 
+# needs -ltk, -ltcl, and sometimes -lX11
+#@MODULE__TKINTER_TRUE at _tkinter _tkinter.c tkappinit.c
+
 # Linux: -luuid, BSD/AIX: libc's uuid_create()
 @MODULE__UUID_TRUE at _uuid _uuidmodule.c
 
diff --git a/configure.ac b/configure.ac
index 40dc6f425f26f..39890fe38f0a1 100644
--- a/configure.ac
+++ b/configure.ac
@@ -6480,17 +6480,22 @@ PY_STDLIB_MOD([_crypt],
 PY_STDLIB_MOD([_ctypes],
   [], [test "$have_libffi" = yes],
   [$LIBFFI_CFLAGS], [$LIBFFI_LIBS])
+dnl PY_STDLIB_MOD([_curses], [], [], [], [])
+dnl PY_STDLIB_MOD([_curses_panel], [], [], [], [])
 PY_STDLIB_MOD([_decimal], [], [], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LDFLAGS])
+dnl PY_STDLIB_MOD([_dbm], [], [], [], [])
 PY_STDLIB_MOD([_gdbm],
   [test "$have_gdbm_dbmliborder" = yes], [test "$have_gdbm" = yes],
   [$GDBM_CFLAGS], [$GDBM_LIBS])
 PY_STDLIB_MOD([nis],
   [], [test "$have_nis" = yes -a "$ac_cv_header_rpc_rpc_h" = yes],
   [$LIBNSL_CFLAGS], [$LIBNSL_LIBS])
+dnl PY_STDLIB_MOD([readline], [], [], [], [])
 PY_STDLIB_MOD([_sqlite3],
   [test "$have_sqlite3" = "yes"],
   [test "$have_supported_sqlite3" = "yes"],
   [$LIBSQLITE3_CFLAGS], [$LIBSQLITE3_LIBS])
+dnl PY_STDLIB_MOD([_tkinter], [], [], [], [])
 PY_STDLIB_MOD([_uuid],
   [], [test "$have_uuid" = "yes"],
   [$LIBUUID_CFLAGS], [$LIBUUID_LIBS])


From webhook-mailer at python.org  Sat Dec  4 10:38:26 2021
From: webhook-mailer at python.org (corona10)
Date: Sat, 04 Dec 2021 15:38:26 -0000
Subject: [Python-checkins] Fixed documentation typo in compileall.py
 (GH-29912)
Message-ID: <mailman.36.1638632306.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/87a18deda43a9e512e8ca85daf363e9924f6db5f
commit: 87a18deda43a9e512e8ca85daf363e9924f6db5f
branch: main
author: Vishal Pandey <vishalpandeyviptsk at gmail.com>
committer: corona10 <donghee.na92 at gmail.com>
date: 2021-12-05T00:38:17+09:00
summary:

Fixed documentation typo in compileall.py (GH-29912)

files:
M Lib/compileall.py

diff --git a/Lib/compileall.py b/Lib/compileall.py
index 3755e76ba813f..330a90786efc5 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -4,7 +4,7 @@
 given as arguments recursively; the -l option prevents it from
 recursing into directories.
 
-Without arguments, if compiles all modules on sys.path, without
+Without arguments, it compiles all modules on sys.path, without
 recursing into subdirectories.  (Even though it should do so for
 packages -- for now, you'll have to deal with packages separately.)
 


From webhook-mailer at python.org  Sat Dec  4 13:57:28 2021
From: webhook-mailer at python.org (serhiy-storchaka)
Date: Sat, 04 Dec 2021 18:57:28 -0000
Subject: [Python-checkins] bpo-13236: Flush the output stream more often in
 unittest (GH-29864)
Message-ID: <mailman.37.1638644249.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/f42a06ba279c916fb67289e47f9bc60dc5dee4ee
commit: f42a06ba279c916fb67289e47f9bc60dc5dee4ee
branch: main
author: Serhiy Storchaka <storchaka at gmail.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2021-12-04T20:57:20+02:00
summary:

bpo-13236: Flush the output stream more often in unittest (GH-29864)

It can prevent some losses when output to buffered stream.

files:
A Misc/NEWS.d/next/Library/2021-11-30-13-52-02.bpo-13236.FmJIkO.rst
M Lib/unittest/runner.py
M Lib/unittest/test/test_program.py
M Lib/unittest/test/test_result.py

diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py
index b51def8882fc1..120ee6aba3570 100644
--- a/Lib/unittest/runner.py
+++ b/Lib/unittest/runner.py
@@ -68,6 +68,7 @@ def _write_status(self, test, status):
             self.stream.write(self.getDescription(test))
             self.stream.write(" ... ")
         self.stream.writeln(status)
+        self.stream.flush()
         self._newline = True
 
     def addSubTest(self, test, subtest, err):
@@ -121,6 +122,7 @@ def addExpectedFailure(self, test, err):
         super(TextTestResult, self).addExpectedFailure(test, err)
         if self.showAll:
             self.stream.writeln("expected failure")
+            self.stream.flush()
         elif self.dots:
             self.stream.write("x")
             self.stream.flush()
@@ -129,6 +131,7 @@ def addUnexpectedSuccess(self, test):
         super(TextTestResult, self).addUnexpectedSuccess(test)
         if self.showAll:
             self.stream.writeln("unexpected success")
+            self.stream.flush()
         elif self.dots:
             self.stream.write("u")
             self.stream.flush()
@@ -136,6 +139,7 @@ def addUnexpectedSuccess(self, test):
     def printErrors(self):
         if self.dots or self.showAll:
             self.stream.writeln()
+            self.stream.flush()
         self.printErrorList('ERROR', self.errors)
         self.printErrorList('FAIL', self.failures)
 
@@ -145,6 +149,7 @@ def printErrorList(self, flavour, errors):
             self.stream.writeln("%s: %s" % (flavour,self.getDescription(test)))
             self.stream.writeln(self.separator2)
             self.stream.writeln("%s" % err)
+            self.stream.flush()
 
 
 class TextTestRunner(object):
@@ -239,4 +244,5 @@ def run(self, test):
             self.stream.writeln(" (%s)" % (", ".join(infos),))
         else:
             self.stream.write("\n")
+        self.stream.flush()
         return result
diff --git a/Lib/unittest/test/test_program.py b/Lib/unittest/test/test_program.py
index 93a61097e93c7..cc02b227e1da8 100644
--- a/Lib/unittest/test/test_program.py
+++ b/Lib/unittest/test/test_program.py
@@ -6,6 +6,7 @@
 from test import support
 import unittest
 import unittest.test
+from .test_result import BufferedWriter
 
 
 class Test_TestProgram(unittest.TestCase):
@@ -104,30 +105,39 @@ def run(self, test):
                           program.testNames)
 
     def test_NonExit(self):
+        stream = BufferedWriter()
         program = unittest.main(exit=False,
                                 argv=["foobar"],
-                                testRunner=unittest.TextTestRunner(stream=io.StringIO()),
+                                testRunner=unittest.TextTestRunner(stream=stream),
                                 testLoader=self.FooBarLoader())
         self.assertTrue(hasattr(program, 'result'))
+        self.assertIn('\nFAIL: testFail ', stream.getvalue())
+        self.assertTrue(stream.getvalue().endswith('\n\nFAILED (failures=1)\n'))
 
 
     def test_Exit(self):
+        stream = BufferedWriter()
         self.assertRaises(
             SystemExit,
             unittest.main,
             argv=["foobar"],
-            testRunner=unittest.TextTestRunner(stream=io.StringIO()),
+            testRunner=unittest.TextTestRunner(stream=stream),
             exit=True,
             testLoader=self.FooBarLoader())
+        self.assertIn('\nFAIL: testFail ', stream.getvalue())
+        self.assertTrue(stream.getvalue().endswith('\n\nFAILED (failures=1)\n'))
 
 
     def test_ExitAsDefault(self):
+        stream = BufferedWriter()
         self.assertRaises(
             SystemExit,
             unittest.main,
             argv=["foobar"],
-            testRunner=unittest.TextTestRunner(stream=io.StringIO()),
+            testRunner=unittest.TextTestRunner(stream=stream),
             testLoader=self.FooBarLoader())
+        self.assertIn('\nFAIL: testFail ', stream.getvalue())
+        self.assertTrue(stream.getvalue().endswith('\n\nFAILED (failures=1)\n'))
 
 
 class InitialisableProgram(unittest.TestProgram):
diff --git a/Lib/unittest/test/test_result.py b/Lib/unittest/test/test_result.py
index c4a49b48752dc..224a78463edc9 100644
--- a/Lib/unittest/test/test_result.py
+++ b/Lib/unittest/test/test_result.py
@@ -33,6 +33,22 @@ def bad_cleanup2():
     raise ValueError('bad cleanup2')
 
 
+class BufferedWriter:
+    def __init__(self):
+        self.result = ''
+        self.buffer = ''
+
+    def write(self, arg):
+        self.buffer += arg
+
+    def flush(self):
+        self.result += self.buffer
+        self.buffer = ''
+
+    def getvalue(self):
+        return self.result
+
+
 class Test_TestResult(unittest.TestCase):
     # Note: there are not separate tests for TestResult.wasSuccessful(),
     # TestResult.errors, TestResult.failures, TestResult.testsRun or
@@ -335,10 +351,13 @@ def testFailFast(self):
         self.assertTrue(result.shouldStop)
 
     def testFailFastSetByRunner(self):
-        runner = unittest.TextTestRunner(stream=io.StringIO(), failfast=True)
+        stream = BufferedWriter()
+        runner = unittest.TextTestRunner(stream=stream, failfast=True)
         def test(result):
             self.assertTrue(result.failfast)
         result = runner.run(test)
+        stream.flush()
+        self.assertTrue(stream.getvalue().endswith('\n\nOK\n'))
 
 
 class Test_TextTestResult(unittest.TestCase):
@@ -462,6 +481,12 @@ def testFail(self):
             self.fail('fail')
         def testError(self):
             raise Exception('error')
+        @unittest.expectedFailure
+        def testExpectedFailure(self):
+            self.fail('fail')
+        @unittest.expectedFailure
+        def testUnexpectedSuccess(self):
+            pass
         def testSubTestSuccess(self):
             with self.subTest('one', a=1):
                 pass
@@ -483,7 +508,7 @@ def tearDown(self):
                 raise self.tearDownError
 
     def _run_test(self, test_name, verbosity, tearDownError=None):
-        stream = io.StringIO()
+        stream = BufferedWriter()
         stream = unittest.runner._WritelnDecorator(stream)
         result = unittest.TextTestResult(stream, True, verbosity)
         test = self.Test(test_name)
@@ -496,6 +521,8 @@ def testDotsOutput(self):
         self.assertEqual(self._run_test('testSkip', 1), 's')
         self.assertEqual(self._run_test('testFail', 1), 'F')
         self.assertEqual(self._run_test('testError', 1), 'E')
+        self.assertEqual(self._run_test('testExpectedFailure', 1), 'x')
+        self.assertEqual(self._run_test('testUnexpectedSuccess', 1), 'u')
 
     def testLongOutput(self):
         classname = f'{__name__}.{self.Test.__qualname__}'
@@ -507,6 +534,10 @@ def testLongOutput(self):
                          f'testFail ({classname}) ... FAIL\n')
         self.assertEqual(self._run_test('testError', 2),
                          f'testError ({classname}) ... ERROR\n')
+        self.assertEqual(self._run_test('testExpectedFailure', 2),
+                         f'testExpectedFailure ({classname}) ... expected failure\n')
+        self.assertEqual(self._run_test('testUnexpectedSuccess', 2),
+                         f'testUnexpectedSuccess ({classname}) ... unexpected success\n')
 
     def testDotsOutputSubTestSuccess(self):
         self.assertEqual(self._run_test('testSubTestSuccess', 1), '.')
diff --git a/Misc/NEWS.d/next/Library/2021-11-30-13-52-02.bpo-13236.FmJIkO.rst b/Misc/NEWS.d/next/Library/2021-11-30-13-52-02.bpo-13236.FmJIkO.rst
new file mode 100644
index 0000000000000..bfea8d4fca0e0
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-11-30-13-52-02.bpo-13236.FmJIkO.rst
@@ -0,0 +1,2 @@
+:class:`unittest.TextTestResult` and :class:`unittest.TextTestRunner` flush
+now the output stream more often.


From webhook-mailer at python.org  Sun Dec  5 07:23:25 2021
From: webhook-mailer at python.org (serhiy-storchaka)
Date: Sun, 05 Dec 2021 12:23:25 -0000
Subject: [Python-checkins] bpo-27946: Fix possible crash in
 ElementTree.Element (GH-29915)
Message-ID: <mailman.38.1638707006.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/d15cdb2f32f572ce56d7120135da24b9fdce4c99
commit: d15cdb2f32f572ce56d7120135da24b9fdce4c99
branch: main
author: Serhiy Storchaka <storchaka at gmail.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2021-12-05T14:22:54+02:00
summary:

bpo-27946: Fix possible crash in ElementTree.Element (GH-29915)

Getting an attribute via attrib.get() simultaneously with replacing
the attrib dict can lead to access to deallocated dict.

files:
A Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
M Lib/test/test_xml_etree_c.py
M Modules/_elementtree.c

diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py
index e68613bb910dd..bec8208571902 100644
--- a/Lib/test/test_xml_etree_c.py
+++ b/Lib/test/test_xml_etree_c.py
@@ -169,6 +169,18 @@ def test_xmlpullparser_leaks(self):
         del parser
         support.gc_collect()
 
+    def test_dict_disappearing_during_get_item(self):
+        # test fix for seg fault reported in issue 27946
+        class X:
+            def __hash__(self):
+                e.attrib = {} # this frees e->extra->attrib
+                [{i: i} for i in range(1000)] # exhaust the dict keys cache
+                return 13
+
+        e = cET.Element("elem", {1: 2})
+        r = e.get(X())
+        self.assertIsNone(r)
+
 
 @unittest.skipUnless(cET, 'requires _elementtree')
 class TestAliasWorking(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst b/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
new file mode 100644
index 0000000000000..0378efca746bb
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
@@ -0,0 +1,3 @@
+Fix possible crash when getting an attribute of
+class:`xml.etree.ElementTree.Element` simultaneously with
+replacing the ``attrib`` dict.
diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
index b4528a90b3e09..9dadeef712938 100644
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -1393,22 +1393,19 @@ _elementtree_Element_get_impl(ElementObject *self, PyObject *key,
                               PyObject *default_value)
 /*[clinic end generated code: output=523c614142595d75 input=ee153bbf8cdb246e]*/
 {
-    PyObject* value;
-
-    if (!self->extra || !self->extra->attrib)
-        value = default_value;
-    else {
-        value = PyDict_GetItemWithError(self->extra->attrib, key);
-        if (!value) {
-            if (PyErr_Occurred()) {
-                return NULL;
-            }
-            value = default_value;
+    if (self->extra && self->extra->attrib) {
+        PyObject *attrib = self->extra->attrib;
+        Py_INCREF(attrib);
+        PyObject *value = PyDict_GetItemWithError(attrib, key);
+        Py_XINCREF(value);
+        Py_DECREF(attrib);
+        if (value != NULL || PyErr_Occurred()) {
+            return value;
         }
     }
 
-    Py_INCREF(value);
-    return value;
+    Py_INCREF(default_value);
+    return default_value;
 }
 
 static PyObject *


From webhook-mailer at python.org  Sun Dec  5 12:04:16 2021
From: webhook-mailer at python.org (gvanrossum)
Date: Sun, 05 Dec 2021 17:04:16 -0000
Subject: [Python-checkins] Delete orphaned comment (#29917)
Message-ID: <mailman.39.1638723857.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/605f4bf7838dbef86b52c955dc21ab489219490f
commit: 605f4bf7838dbef86b52c955dc21ab489219490f
branch: main
author: Guido van Rossum <guido at python.org>
committer: gvanrossum <gvanrossum at gmail.com>
date: 2021-12-05T09:03:58-08:00
summary:

Delete orphaned comment (#29917)

(The function this described was deleted by PR #23743, the comment was accidentally retained.)

files:
M Python/compile.c

diff --git a/Python/compile.c b/Python/compile.c
index 87de7baab4819..6138031833ac9 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -5064,11 +5064,6 @@ compiler_visit_keyword(struct compiler *c, keyword_ty k)
     return 1;
 }
 
-/* Test whether expression is constant.  For constants, report
-   whether they are true or false.
-
-   Return values: 1 for true, 0 for false, -1 for non-constant.
- */
 
 static int
 compiler_with_except_finish(struct compiler *c, basicblock * cleanup) {


From webhook-mailer at python.org  Sun Dec  5 12:41:55 2021
From: webhook-mailer at python.org (tiran)
Date: Sun, 05 Dec 2021 17:41:55 -0000
Subject: [Python-checkins] bpo-45582: Fix signature of
 _Py_Get_Getpath_CodeObject (GH-29921)
Message-ID: <mailman.40.1638726116.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/628abe4463ed40cd54ca952a2b4cc2d6e74073f7
commit: 628abe4463ed40cd54ca952a2b4cc2d6e74073f7
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-05T18:41:46+01:00
summary:

bpo-45582: Fix signature of _Py_Get_Getpath_CodeObject (GH-29921)

files:
M Modules/getpath.c

diff --git a/Modules/getpath.c b/Modules/getpath.c
index c8c85a8540d39..f77b18eee95b6 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -783,7 +783,7 @@ library_to_dict(PyObject *dict, const char *key)
 
 
 PyObject *
-_Py_Get_Getpath_CodeObject()
+_Py_Get_Getpath_CodeObject(void)
 {
     return PyMarshal_ReadObjectFromString(
         (const char*)_Py_M__getpath, sizeof(_Py_M__getpath));


From webhook-mailer at python.org  Sun Dec  5 14:05:07 2021
From: webhook-mailer at python.org (miss-islington)
Date: Sun, 05 Dec 2021 19:05:07 -0000
Subject: [Python-checkins] bpo-27946: Fix possible crash in
 ElementTree.Element (GH-29915)
Message-ID: <mailman.41.1638731108.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/52a9a71fe682e47f6c78a9c34aa9a797ca632c86
commit: 52a9a71fe682e47f6c78a9c34aa9a797ca632c86
branch: 3.9
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-05T11:04:59-08:00
summary:

bpo-27946: Fix possible crash in ElementTree.Element (GH-29915)


Getting an attribute via attrib.get() simultaneously with replacing
the attrib dict can lead to access to deallocated dict.
(cherry picked from commit d15cdb2f32f572ce56d7120135da24b9fdce4c99)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
M Lib/test/test_xml_etree_c.py
M Modules/_elementtree.c

diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py
index e26e1714a540b..82cedfd00a55d 100644
--- a/Lib/test/test_xml_etree_c.py
+++ b/Lib/test/test_xml_etree_c.py
@@ -169,6 +169,18 @@ def test_xmlpullparser_leaks(self):
         del parser
         support.gc_collect()
 
+    def test_dict_disappearing_during_get_item(self):
+        # test fix for seg fault reported in issue 27946
+        class X:
+            def __hash__(self):
+                e.attrib = {} # this frees e->extra->attrib
+                [{i: i} for i in range(1000)] # exhaust the dict keys cache
+                return 13
+
+        e = cET.Element("elem", {1: 2})
+        r = e.get(X())
+        self.assertIsNone(r)
+
 
 @unittest.skipUnless(cET, 'requires _elementtree')
 class TestAliasWorking(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst b/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
new file mode 100644
index 0000000000000..0378efca746bb
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
@@ -0,0 +1,3 @@
+Fix possible crash when getting an attribute of
+class:`xml.etree.ElementTree.Element` simultaneously with
+replacing the ``attrib`` dict.
diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
index 67a00a3413b1d..d9c2a23ccb858 100644
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -1393,22 +1393,19 @@ _elementtree_Element_get_impl(ElementObject *self, PyObject *key,
                               PyObject *default_value)
 /*[clinic end generated code: output=523c614142595d75 input=ee153bbf8cdb246e]*/
 {
-    PyObject* value;
-
-    if (!self->extra || !self->extra->attrib)
-        value = default_value;
-    else {
-        value = PyDict_GetItemWithError(self->extra->attrib, key);
-        if (!value) {
-            if (PyErr_Occurred()) {
-                return NULL;
-            }
-            value = default_value;
+    if (self->extra && self->extra->attrib) {
+        PyObject *attrib = self->extra->attrib;
+        Py_INCREF(attrib);
+        PyObject *value = PyDict_GetItemWithError(attrib, key);
+        Py_XINCREF(value);
+        Py_DECREF(attrib);
+        if (value != NULL || PyErr_Occurred()) {
+            return value;
         }
     }
 
-    Py_INCREF(value);
-    return value;
+    Py_INCREF(default_value);
+    return default_value;
 }
 
 static PyObject *


From webhook-mailer at python.org  Sun Dec  5 14:05:07 2021
From: webhook-mailer at python.org (miss-islington)
Date: Sun, 05 Dec 2021 19:05:07 -0000
Subject: [Python-checkins] bpo-27946: Fix possible crash in
 ElementTree.Element (GH-29915)
Message-ID: <mailman.42.1638731108.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/beb834292db54fea129dd073cc822179430cee52
commit: beb834292db54fea129dd073cc822179430cee52
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-05T11:04:52-08:00
summary:

bpo-27946: Fix possible crash in ElementTree.Element (GH-29915)


Getting an attribute via attrib.get() simultaneously with replacing
the attrib dict can lead to access to deallocated dict.
(cherry picked from commit d15cdb2f32f572ce56d7120135da24b9fdce4c99)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
M Lib/test/test_xml_etree_c.py
M Modules/_elementtree.c

diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py
index e68613bb910dd..bec8208571902 100644
--- a/Lib/test/test_xml_etree_c.py
+++ b/Lib/test/test_xml_etree_c.py
@@ -169,6 +169,18 @@ def test_xmlpullparser_leaks(self):
         del parser
         support.gc_collect()
 
+    def test_dict_disappearing_during_get_item(self):
+        # test fix for seg fault reported in issue 27946
+        class X:
+            def __hash__(self):
+                e.attrib = {} # this frees e->extra->attrib
+                [{i: i} for i in range(1000)] # exhaust the dict keys cache
+                return 13
+
+        e = cET.Element("elem", {1: 2})
+        r = e.get(X())
+        self.assertIsNone(r)
+
 
 @unittest.skipUnless(cET, 'requires _elementtree')
 class TestAliasWorking(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst b/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
new file mode 100644
index 0000000000000..0378efca746bb
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
@@ -0,0 +1,3 @@
+Fix possible crash when getting an attribute of
+class:`xml.etree.ElementTree.Element` simultaneously with
+replacing the ``attrib`` dict.
diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c
index b4528a90b3e09..9dadeef712938 100644
--- a/Modules/_elementtree.c
+++ b/Modules/_elementtree.c
@@ -1393,22 +1393,19 @@ _elementtree_Element_get_impl(ElementObject *self, PyObject *key,
                               PyObject *default_value)
 /*[clinic end generated code: output=523c614142595d75 input=ee153bbf8cdb246e]*/
 {
-    PyObject* value;
-
-    if (!self->extra || !self->extra->attrib)
-        value = default_value;
-    else {
-        value = PyDict_GetItemWithError(self->extra->attrib, key);
-        if (!value) {
-            if (PyErr_Occurred()) {
-                return NULL;
-            }
-            value = default_value;
+    if (self->extra && self->extra->attrib) {
+        PyObject *attrib = self->extra->attrib;
+        Py_INCREF(attrib);
+        PyObject *value = PyDict_GetItemWithError(attrib, key);
+        Py_XINCREF(value);
+        Py_DECREF(attrib);
+        if (value != NULL || PyErr_Occurred()) {
+            return value;
         }
     }
 
-    Py_INCREF(value);
-    return value;
+    Py_INCREF(default_value);
+    return default_value;
 }
 
 static PyObject *


From webhook-mailer at python.org  Sun Dec  5 15:26:20 2021
From: webhook-mailer at python.org (serhiy-storchaka)
Date: Sun, 05 Dec 2021 20:26:20 -0000
Subject: [Python-checkins] bpo-37295: Optimize math.comb() and math.perm()
 (GH-29090)
Message-ID: <mailman.43.1638735981.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/60c320c38e4e95877cde0b1d8562ebd6bc02ac61
commit: 60c320c38e4e95877cde0b1d8562ebd6bc02ac61
branch: main
author: Serhiy Storchaka <storchaka at gmail.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2021-12-05T22:26:10+02:00
summary:

bpo-37295: Optimize math.comb() and math.perm() (GH-29090)

For very large numbers use divide-and-conquer algorithm for getting
benefit of Karatsuba multiplication of large numbers.

Do calculations completely in C unsigned long long instead of Python
integers if possible.

files:
A Misc/NEWS.d/next/Library/2021-10-18-16-08-55.bpo-37295.wBEWH2.rst
M Doc/whatsnew/3.11.rst
M Modules/mathmodule.c

diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index 10dc30939414c..b06d8d4215033 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -351,6 +351,11 @@ Optimizations
 * Pure ASCII strings are now normalized in constant time by :func:`unicodedata.normalize`.
   (Contributed by Dong-hee Na in :issue:`44987`.)
 
+* :mod:`math` functions :func:`~math.comb` and :func:`~math.perm` are now up
+  to 10 times or more faster for large arguments (the speed up is larger for
+  larger *k*).
+  (Contributed by Serhiy Storchaka in :issue:`37295`.)
+
 
 CPython bytecode changes
 ========================
diff --git a/Misc/NEWS.d/next/Library/2021-10-18-16-08-55.bpo-37295.wBEWH2.rst b/Misc/NEWS.d/next/Library/2021-10-18-16-08-55.bpo-37295.wBEWH2.rst
new file mode 100644
index 0000000000000..634f0c453884e
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-18-16-08-55.bpo-37295.wBEWH2.rst
@@ -0,0 +1 @@
+Optimize :func:`math.comb` and :func:`math.perm`.
diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c
index 64ce4e6a13fd5..84b5b954b1051 100644
--- a/Modules/mathmodule.c
+++ b/Modules/mathmodule.c
@@ -3221,6 +3221,138 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start)
 }
 
 
+/* Number of permutations and combinations.
+ * P(n, k) = n! / (n-k)!
+ * C(n, k) = P(n, k) / k!
+ */
+
+/* Calculate C(n, k) for n in the 63-bit range. */
+static PyObject *
+perm_comb_small(unsigned long long n, unsigned long long k, int iscomb)
+{
+    /* long long is at least 64 bit */
+    static const unsigned long long fast_comb_limits[] = {
+        0, ULLONG_MAX, 4294967296ULL, 3329022, 102570, 13467, 3612, 1449,  // 0-7
+        746, 453, 308, 227, 178, 147, 125, 110,  // 8-15
+        99, 90, 84, 79, 75, 72, 69, 68,  // 16-23
+        66, 65, 64, 63, 63, 62, 62, 62,  // 24-31
+    };
+    static const unsigned long long fast_perm_limits[] = {
+        0, ULLONG_MAX, 4294967296ULL, 2642246, 65537, 7133, 1627, 568,  // 0-7
+        259, 142, 88, 61, 45, 36, 30,  // 8-14
+    };
+
+    if (k == 0) {
+        return PyLong_FromLong(1);
+    }
+
+    /* For small enough n and k the result fits in the 64-bit range and can
+     * be calculated without allocating intermediate PyLong objects. */
+    if (iscomb
+        ? (k < Py_ARRAY_LENGTH(fast_comb_limits)
+           && n <= fast_comb_limits[k])
+        : (k < Py_ARRAY_LENGTH(fast_perm_limits)
+            && n <= fast_perm_limits[k]))
+    {
+        unsigned long long result = n;
+        if (iscomb) {
+            for (unsigned long long i = 1; i < k;) {
+                result *= --n;
+                result /= ++i;
+            }
+        }
+        else {
+            for (unsigned long long i = 1; i < k;) {
+                result *= --n;
+                ++i;
+            }
+        }
+        return PyLong_FromUnsignedLongLong(result);
+    }
+
+    /* For larger n use recursive formula. */
+    /* C(n, k) = C(n, j) * C(n-j, k-j) // C(k, j) */
+    unsigned long long j = k / 2;
+    PyObject *a, *b;
+    a = perm_comb_small(n, j, iscomb);
+    if (a == NULL) {
+        return NULL;
+    }
+    b = perm_comb_small(n - j, k - j, iscomb);
+    if (b == NULL) {
+        goto error;
+    }
+    Py_SETREF(a, PyNumber_Multiply(a, b));
+    Py_DECREF(b);
+    if (iscomb && a != NULL) {
+        b = perm_comb_small(k, j, 1);
+        if (b == NULL) {
+            goto error;
+        }
+        Py_SETREF(a, PyNumber_FloorDivide(a, b));
+        Py_DECREF(b);
+    }
+    return a;
+
+error:
+    Py_DECREF(a);
+    return NULL;
+}
+
+/* Calculate P(n, k) or C(n, k) using recursive formulas.
+ * It is more efficient than sequential multiplication thanks to
+ * Karatsuba multiplication.
+ */
+static PyObject *
+perm_comb(PyObject *n, unsigned long long k, int iscomb)
+{
+    if (k == 0) {
+        return PyLong_FromLong(1);
+    }
+    if (k == 1) {
+        Py_INCREF(n);
+        return n;
+    }
+
+    /* P(n, k) = P(n, j) * P(n-j, k-j) */
+    /* C(n, k) = C(n, j) * C(n-j, k-j) // C(k, j) */
+    unsigned long long j = k / 2;
+    PyObject *a, *b;
+    a = perm_comb(n, j, iscomb);
+    if (a == NULL) {
+        return NULL;
+    }
+    PyObject *t = PyLong_FromUnsignedLongLong(j);
+    if (t == NULL) {
+        goto error;
+    }
+    n = PyNumber_Subtract(n, t);
+    Py_DECREF(t);
+    if (n == NULL) {
+        goto error;
+    }
+    b = perm_comb(n, k - j, iscomb);
+    Py_DECREF(n);
+    if (b == NULL) {
+        goto error;
+    }
+    Py_SETREF(a, PyNumber_Multiply(a, b));
+    Py_DECREF(b);
+    if (iscomb && a != NULL) {
+        b = perm_comb_small(k, j, 1);
+        if (b == NULL) {
+            goto error;
+        }
+        Py_SETREF(a, PyNumber_FloorDivide(a, b));
+        Py_DECREF(b);
+    }
+    return a;
+
+error:
+    Py_DECREF(a);
+    return NULL;
+}
+
 /*[clinic input]
 math.perm
 
@@ -3244,9 +3376,9 @@ static PyObject *
 math_perm_impl(PyObject *module, PyObject *n, PyObject *k)
 /*[clinic end generated code: output=e021a25469653e23 input=5311c5a00f359b53]*/
 {
-    PyObject *result = NULL, *factor = NULL;
+    PyObject *result = NULL;
     int overflow, cmp;
-    long long i, factors;
+    long long ki, ni;
 
     if (k == Py_None) {
         return math_factorial(module, n);
@@ -3260,6 +3392,7 @@ math_perm_impl(PyObject *module, PyObject *n, PyObject *k)
         Py_DECREF(n);
         return NULL;
     }
+    assert(PyLong_CheckExact(n) && PyLong_CheckExact(k));
 
     if (Py_SIZE(n) < 0) {
         PyErr_SetString(PyExc_ValueError,
@@ -3281,42 +3414,26 @@ math_perm_impl(PyObject *module, PyObject *n, PyObject *k)
         goto error;
     }
 
-    factors = PyLong_AsLongLongAndOverflow(k, &overflow);
+    ki = PyLong_AsLongLongAndOverflow(k, &overflow);
+    assert(overflow >= 0 && !PyErr_Occurred());
     if (overflow > 0) {
         PyErr_Format(PyExc_OverflowError,
                      "k must not exceed %lld",
                      LLONG_MAX);
         goto error;
     }
-    else if (factors == -1) {
-        /* k is nonnegative, so a return value of -1 can only indicate error */
-        goto error;
-    }
+    assert(ki >= 0);
 
-    if (factors == 0) {
-        result = PyLong_FromLong(1);
-        goto done;
+    ni = PyLong_AsLongLongAndOverflow(n, &overflow);
+    assert(overflow >= 0 && !PyErr_Occurred());
+    if (!overflow && ki > 1) {
+        assert(ni >= 0);
+        result = perm_comb_small((unsigned long long)ni,
+                                 (unsigned long long)ki, 0);
     }
-
-    result = n;
-    Py_INCREF(result);
-    if (factors == 1) {
-        goto done;
-    }
-
-    factor = Py_NewRef(n);
-    PyObject *one = _PyLong_GetOne();  // borrowed ref
-    for (i = 1; i < factors; ++i) {
-        Py_SETREF(factor, PyNumber_Subtract(factor, one));
-        if (factor == NULL) {
-            goto error;
-        }
-        Py_SETREF(result, PyNumber_Multiply(result, factor));
-        if (result == NULL) {
-            goto error;
-        }
+    else {
+        result = perm_comb(n, (unsigned long long)ki, 0);
     }
-    Py_DECREF(factor);
 
 done:
     Py_DECREF(n);
@@ -3324,14 +3441,11 @@ math_perm_impl(PyObject *module, PyObject *n, PyObject *k)
     return result;
 
 error:
-    Py_XDECREF(factor);
-    Py_XDECREF(result);
     Py_DECREF(n);
     Py_DECREF(k);
     return NULL;
 }
 
-
 /*[clinic input]
 math.comb
 
@@ -3357,9 +3471,9 @@ static PyObject *
 math_comb_impl(PyObject *module, PyObject *n, PyObject *k)
 /*[clinic end generated code: output=bd2cec8d854f3493 input=9a05315af2518709]*/
 {
-    PyObject *result = NULL, *factor = NULL, *temp;
+    PyObject *result = NULL, *temp;
     int overflow, cmp;
-    long long i, factors;
+    long long ki, ni;
 
     n = PyNumber_Index(n);
     if (n == NULL) {
@@ -3370,6 +3484,7 @@ math_comb_impl(PyObject *module, PyObject *n, PyObject *k)
         Py_DECREF(n);
         return NULL;
     }
+    assert(PyLong_CheckExact(n) && PyLong_CheckExact(k));
 
     if (Py_SIZE(n) < 0) {
         PyErr_SetString(PyExc_ValueError,
@@ -3382,73 +3497,59 @@ math_comb_impl(PyObject *module, PyObject *n, PyObject *k)
         goto error;
     }
 
-    /* k = min(k, n - k) */
-    temp = PyNumber_Subtract(n, k);
-    if (temp == NULL) {
-        goto error;
-    }
-    if (Py_SIZE(temp) < 0) {
-        Py_DECREF(temp);
-        result = PyLong_FromLong(0);
-        goto done;
-    }
-    cmp = PyObject_RichCompareBool(temp, k, Py_LT);
-    if (cmp > 0) {
-        Py_SETREF(k, temp);
+    ni = PyLong_AsLongLongAndOverflow(n, &overflow);
+    assert(overflow >= 0 && !PyErr_Occurred());
+    if (!overflow) {
+        assert(ni >= 0);
+        ki = PyLong_AsLongLongAndOverflow(k, &overflow);
+        assert(overflow >= 0 && !PyErr_Occurred());
+        if (overflow || ki > ni) {
+            result = PyLong_FromLong(0);
+            goto done;
+        }
+        assert(ki >= 0);
+        ki = Py_MIN(ki, ni - ki);
+        if (ki > 1) {
+            result = perm_comb_small((unsigned long long)ni,
+                                     (unsigned long long)ki, 1);
+            goto done;
+        }
+        /* For k == 1 just return the original n in perm_comb(). */
     }
     else {
-        Py_DECREF(temp);
-        if (cmp < 0) {
+        /* k = min(k, n - k) */
+        temp = PyNumber_Subtract(n, k);
+        if (temp == NULL) {
             goto error;
         }
-    }
-
-    factors = PyLong_AsLongLongAndOverflow(k, &overflow);
-    if (overflow > 0) {
-        PyErr_Format(PyExc_OverflowError,
-                     "min(n - k, k) must not exceed %lld",
-                     LLONG_MAX);
-        goto error;
-    }
-    if (factors == -1) {
-        /* k is nonnegative, so a return value of -1 can only indicate error */
-        goto error;
-    }
-
-    if (factors == 0) {
-        result = PyLong_FromLong(1);
-        goto done;
-    }
-
-    result = n;
-    Py_INCREF(result);
-    if (factors == 1) {
-        goto done;
-    }
-
-    factor = Py_NewRef(n);
-    PyObject *one = _PyLong_GetOne();  // borrowed ref
-    for (i = 1; i < factors; ++i) {
-        Py_SETREF(factor, PyNumber_Subtract(factor, one));
-        if (factor == NULL) {
-            goto error;
+        if (Py_SIZE(temp) < 0) {
+            Py_DECREF(temp);
+            result = PyLong_FromLong(0);
+            goto done;
         }
-        Py_SETREF(result, PyNumber_Multiply(result, factor));
-        if (result == NULL) {
-            goto error;
+        cmp = PyObject_RichCompareBool(temp, k, Py_LT);
+        if (cmp > 0) {
+            Py_SETREF(k, temp);
         }
-
-        temp = PyLong_FromUnsignedLongLong((unsigned long long)i + 1);
-        if (temp == NULL) {
-            goto error;
+        else {
+            Py_DECREF(temp);
+            if (cmp < 0) {
+                goto error;
+            }
         }
-        Py_SETREF(result, PyNumber_FloorDivide(result, temp));
-        Py_DECREF(temp);
-        if (result == NULL) {
+
+        ki = PyLong_AsLongLongAndOverflow(k, &overflow);
+        assert(overflow >= 0 && !PyErr_Occurred());
+        if (overflow) {
+            PyErr_Format(PyExc_OverflowError,
+                         "min(n - k, k) must not exceed %lld",
+                         LLONG_MAX);
             goto error;
         }
+        assert(ki >= 0);
     }
-    Py_DECREF(factor);
+
+    result = perm_comb(n, (unsigned long long)ki, 1);
 
 done:
     Py_DECREF(n);
@@ -3456,8 +3557,6 @@ math_comb_impl(PyObject *module, PyObject *n, PyObject *k)
     return result;
 
 error:
-    Py_XDECREF(factor);
-    Py_XDECREF(result);
     Py_DECREF(n);
     Py_DECREF(k);
     return NULL;


From webhook-mailer at python.org  Sun Dec  5 15:42:06 2021
From: webhook-mailer at python.org (serhiy-storchaka)
Date: Sun, 05 Dec 2021 20:42:06 -0000
Subject: [Python-checkins] bpo-45662: Fix the repr of InitVar with a type
 alias to the built-in class (GH-29291)
Message-ID: <mailman.44.1638736927.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/1fd4de5bddbbf2a97cdbac4d298c89e1156bdc6c
commit: 1fd4de5bddbbf2a97cdbac4d298c89e1156bdc6c
branch: main
author: Serhiy Storchaka <storchaka at gmail.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2021-12-05T22:41:58+02:00
summary:

bpo-45662: Fix the repr of InitVar with a type alias to the built-in class (GH-29291)

For example, InitVar[list[int]].

files:
A Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst
M Lib/dataclasses.py
M Lib/test/test_dataclasses.py

diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py
index 8643589077a4a..da06aa69148b7 100644
--- a/Lib/dataclasses.py
+++ b/Lib/dataclasses.py
@@ -229,7 +229,7 @@ def __init__(self, type):
         self.type = type
 
     def __repr__(self):
-        if isinstance(self.type, type):
+        if isinstance(self.type, type) and not isinstance(self.type, GenericAlias):
             type_name = self.type.__name__
         else:
             # typing objects, e.g. List[int]
diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py
index bcd004f4ec3aa..47075df8d59f3 100644
--- a/Lib/test/test_dataclasses.py
+++ b/Lib/test/test_dataclasses.py
@@ -1126,6 +1126,10 @@ def test_init_var_preserve_type(self):
         self.assertEqual(repr(InitVar[int]), 'dataclasses.InitVar[int]')
         self.assertEqual(repr(InitVar[List[int]]),
                          'dataclasses.InitVar[typing.List[int]]')
+        self.assertEqual(repr(InitVar[list[int]]),
+                         'dataclasses.InitVar[list[int]]')
+        self.assertEqual(repr(InitVar[int|str]),
+                         'dataclasses.InitVar[int | str]')
 
     def test_init_var_inheritance(self):
         # Note that this deliberately tests that a dataclass need not
diff --git a/Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst b/Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst
new file mode 100644
index 0000000000000..050b443dd7cb2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst
@@ -0,0 +1,2 @@
+Fix the repr of :data:`dataclasses.InitVar` with a type alias to the
+built-in class, e.g. ``InitVar[list[int]]``.


From webhook-mailer at python.org  Sun Dec  5 15:42:54 2021
From: webhook-mailer at python.org (serhiy-storchaka)
Date: Sun, 05 Dec 2021 20:42:54 -0000
Subject: [Python-checkins] bpo-45663: Fix is_dataclass() for dataclasses
 which are subclasses of types.GenericAlias (GH-29294)
Message-ID: <mailman.45.1638736976.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/446be166861b2f08f87f74018113dd98ca5fca02
commit: 446be166861b2f08f87f74018113dd98ca5fca02
branch: main
author: Serhiy Storchaka <storchaka at gmail.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2021-12-05T22:42:50+02:00
summary:

bpo-45663: Fix is_dataclass() for dataclasses which are subclasses of types.GenericAlias (GH-29294)

files:
A Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
M Lib/dataclasses.py
M Lib/test/test_dataclasses.py

diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py
index da06aa69148b7..3f85d859b1642 100644
--- a/Lib/dataclasses.py
+++ b/Lib/dataclasses.py
@@ -1211,7 +1211,7 @@ def _is_dataclass_instance(obj):
 def is_dataclass(obj):
     """Returns True if obj is a dataclass or an instance of a
     dataclass."""
-    cls = obj if isinstance(obj, type) else type(obj)
+    cls = obj if isinstance(obj, type) and not isinstance(obj, GenericAlias) else type(obj)
     return hasattr(cls, _FIELDS)
 
 
diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py
index 47075df8d59f3..ef5009ab11677 100644
--- a/Lib/test/test_dataclasses.py
+++ b/Lib/test/test_dataclasses.py
@@ -8,6 +8,7 @@
 import pickle
 import inspect
 import builtins
+import types
 import unittest
 from unittest.mock import Mock
 from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol
@@ -1354,6 +1355,17 @@ class B:
                 with self.assertRaisesRegex(TypeError, 'should be called on dataclass instances'):
                     replace(obj, x=0)
 
+    def test_is_dataclass_genericalias(self):
+        @dataclass
+        class A(types.GenericAlias):
+            origin: type
+            args: type
+        self.assertTrue(is_dataclass(A))
+        a = A(list, int)
+        self.assertTrue(is_dataclass(type(a)))
+        self.assertTrue(is_dataclass(a))
+
+
     def test_helper_fields_with_class_instance(self):
         # Check that we can call fields() on either a class or instance,
         #  and get back the same thing.
diff --git a/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst b/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
new file mode 100644
index 0000000000000..f246f67cf80e5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
@@ -0,0 +1,2 @@
+Fix :func:`dataclasses.is_dataclass` for dataclasses which are subclasses of
+:class:`types.GenericAlias`.


From webhook-mailer at python.org  Sun Dec  5 15:44:09 2021
From: webhook-mailer at python.org (serhiy-storchaka)
Date: Sun, 05 Dec 2021 20:44:09 -0000
Subject: [Python-checkins] bpo-45664: Fix resolve_bases() and new_class()
 for GenericAlias instance as a base (GH-29298)
Message-ID: <mailman.46.1638737050.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/2b318ce1c988b7b6e3caf293d55f289e066b6e0f
commit: 2b318ce1c988b7b6e3caf293d55f289e066b6e0f
branch: main
author: Serhiy Storchaka <storchaka at gmail.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2021-12-05T22:44:01+02:00
summary:

bpo-45664: Fix resolve_bases() and new_class() for GenericAlias instance as a base (GH-29298)

files:
A Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
M Lib/test/test_types.py
M Lib/types.py

diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py
index b1218abc8af5e..3dfda5cb95663 100644
--- a/Lib/test/test_types.py
+++ b/Lib/test/test_types.py
@@ -1236,6 +1236,17 @@ def __mro_entries__(self, bases):
         self.assertEqual(D.__orig_bases__, (c,))
         self.assertEqual(D.__mro__, (D, A, object))
 
+    def test_new_class_with_mro_entry_genericalias(self):
+        L1 = types.new_class('L1', (typing.List[int],), {})
+        self.assertEqual(L1.__bases__, (list, typing.Generic))
+        self.assertEqual(L1.__orig_bases__, (typing.List[int],))
+        self.assertEqual(L1.__mro__, (L1, list, typing.Generic, object))
+
+        L2 = types.new_class('L2', (list[int],), {})
+        self.assertEqual(L2.__bases__, (list,))
+        self.assertEqual(L2.__orig_bases__, (list[int],))
+        self.assertEqual(L2.__mro__, (L2, list, object))
+
     def test_new_class_with_mro_entry_none(self):
         class A: pass
         class B: pass
@@ -1351,6 +1362,11 @@ def __mro_entries__(self, bases):
         for bases in [x, y, z, t]:
             self.assertIs(types.resolve_bases(bases), bases)
 
+    def test_resolve_bases_with_mro_entry(self):
+        self.assertEqual(types.resolve_bases((typing.List[int],)),
+                         (list, typing.Generic))
+        self.assertEqual(types.resolve_bases((list[int],)), (list,))
+
     def test_metaclass_derivation(self):
         # issue1294232: correct metaclass calculation
         new_calls = []  # to check the order of __new__ calls
diff --git a/Lib/types.py b/Lib/types.py
index ce1c28d5625a2..679c7f638b310 100644
--- a/Lib/types.py
+++ b/Lib/types.py
@@ -82,7 +82,7 @@ def resolve_bases(bases):
     updated = False
     shift = 0
     for i, base in enumerate(bases):
-        if isinstance(base, type):
+        if isinstance(base, type) and not isinstance(base, GenericAlias):
             continue
         if not hasattr(base, "__mro_entries__"):
             continue
diff --git a/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst b/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
new file mode 100644
index 0000000000000..573a569845a87
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
@@ -0,0 +1,2 @@
+Fix :func:`types.resolve_bases` and :func:`types.new_class` for
+:class:`types.GenericAlias` instance as a base.


From webhook-mailer at python.org  Sun Dec  5 15:49:45 2021
From: webhook-mailer at python.org (serhiy-storchaka)
Date: Sun, 05 Dec 2021 20:49:45 -0000
Subject: [Python-checkins] bpo-45840: Improve cross-references in the data
 model documentation (GH-29633)
Message-ID: <mailman.47.1638737386.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/c0521fe49fd75e794a38a216813658ab40185834
commit: c0521fe49fd75e794a38a216813658ab40185834
branch: main
author: Alex Waygood <Alex.Waygood at Gmail.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2021-12-05T22:49:36+02:00
summary:

bpo-45840: Improve cross-references in the data model documentation (GH-29633)

files:
A Misc/NEWS.d/next/Documentation/2021-11-19-02-02-32.bpo-45840.A51B2S.rst
M Doc/reference/datamodel.rst

diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 41092458102de..597c8ec9f176b 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -188,7 +188,7 @@ Ellipsis
    representation in computers.
 
    The string representations of the numeric classes, computed by
-   :meth:`__repr__` and :meth:`__str__`, have the following
+   :meth:`~object.__repr__` and :meth:`~object.__str__`, have the following
    properties:
 
    * They are valid numeric literals which, when passed to their
@@ -677,7 +677,8 @@ Callable types
       returns an :term:`asynchronous iterator` object which can be used in an
       :keyword:`async for` statement to execute the body of the function.
 
-      Calling the asynchronous iterator's :meth:`aiterator.__anext__` method
+      Calling the asynchronous iterator's
+      :meth:`aiterator.__anext__ <object.__anext__>` method
       will return an :term:`awaitable` which when awaited
       will execute until it provides a value using the :keyword:`yield`
       expression.  When the function executes an empty :keyword:`return`
@@ -715,13 +716,13 @@ Callable types
    Classes
       Classes are callable.  These objects normally act as factories for new
       instances of themselves, but variations are possible for class types that
-      override :meth:`__new__`.  The arguments of the call are passed to
-      :meth:`__new__` and, in the typical case, to :meth:`__init__` to
+      override :meth:`~object.__new__`.  The arguments of the call are passed to
+      :meth:`__new__` and, in the typical case, to :meth:`~object.__init__` to
       initialize the new instance.
 
    Class Instances
       Instances of arbitrary classes can be made callable by defining a
-      :meth:`__call__` method in their class.
+      :meth:`~object.__call__` method in their class.
 
 
 Modules
@@ -880,14 +881,14 @@ Class instances
    section :ref:`descriptors` for another way in which attributes of a class
    retrieved via its instances may differ from the objects actually stored in
    the class's :attr:`~object.__dict__`.  If no class attribute is found, and the
-   object's class has a :meth:`__getattr__` method, that is called to satisfy
+   object's class has a :meth:`~object.__getattr__` method, that is called to satisfy
    the lookup.
 
    .. index:: triple: class instance; attribute; assignment
 
    Attribute assignments and deletions update the instance's dictionary, never a
-   class's dictionary.  If the class has a :meth:`__setattr__` or
-   :meth:`__delattr__` method, this is called instead of updating the instance
+   class's dictionary.  If the class has a :meth:`~object.__setattr__` or
+   :meth:`~object.__delattr__` method, this is called instead of updating the instance
    dictionary directly.
 
    .. index::
@@ -1176,7 +1177,8 @@ Internal types
    Slice objects
       .. index:: builtin: slice
 
-      Slice objects are used to represent slices for :meth:`__getitem__`
+      Slice objects are used to represent slices for
+      :meth:`~object.__getitem__`
       methods.  They are also created by the built-in :func:`slice` function.
 
       .. index::
@@ -1229,7 +1231,8 @@ A class can implement certain operations that are invoked by special syntax
 (such as arithmetic operations or subscripting and slicing) by defining methods
 with special names. This is Python's approach to :dfn:`operator overloading`,
 allowing classes to define their own behavior with respect to language
-operators.  For instance, if a class defines a method named :meth:`__getitem__`,
+operators.  For instance, if a class defines a method named
+:meth:`~object.__getitem__`,
 and ``x`` is an instance of this class, then ``x[i]`` is roughly equivalent
 to ``type(x).__getitem__(x, i)``.  Except where mentioned, attempts to execute an
 operation raise an exception when no appropriate method is defined (typically
@@ -1237,9 +1240,9 @@ operation raise an exception when no appropriate method is defined (typically
 
 Setting a special method to ``None`` indicates that the corresponding
 operation is not available.  For example, if a class sets
-:meth:`__iter__` to ``None``, the class is not iterable, so calling
+:meth:`~object.__iter__` to ``None``, the class is not iterable, so calling
 :func:`iter` on its instances will raise a :exc:`TypeError` (without
-falling back to :meth:`__getitem__`). [#]_
+falling back to :meth:`~object.__getitem__`). [#]_
 
 When implementing a class that emulates any built-in type, it is important that
 the emulation only be implemented to the degree that it makes sense for the
@@ -1789,7 +1792,8 @@ Invoking Descriptors
 
 In general, a descriptor is an object attribute with "binding behavior", one
 whose attribute access has been overridden by methods in the descriptor
-protocol:  :meth:`__get__`, :meth:`__set__`, and :meth:`__delete__`. If any of
+protocol:  :meth:`~object.__get__`, :meth:`~object.__set__`, and
+:meth:`~object.__delete__`. If any of
 those methods are defined for an object, it is said to be a descriptor.
 
 The default behavior for attribute access is to get, set, or delete the
@@ -1853,7 +1857,8 @@ Super Binding
 
 For instance bindings, the precedence of descriptor invocation depends on
 which descriptor methods are defined.  A descriptor can define any combination
-of :meth:`__get__`, :meth:`__set__` and :meth:`__delete__`.  If it does not
+of :meth:`~object.__get__`, :meth:`~object.__set__` and
+:meth:`~object.__delete__`.  If it does not
 define :meth:`__get__`, then accessing the attribute will return the descriptor
 object itself unless there is a value in the object's instance dictionary.  If
 the descriptor defines :meth:`__set__` and/or :meth:`__delete__`, it is a data
@@ -1864,7 +1869,8 @@ descriptors have just the :meth:`__get__` method.  Data descriptors with
 instance dictionary.  In contrast, non-data descriptors can be overridden by
 instances.
 
-Python methods (including :func:`staticmethod` and :func:`classmethod`) are
+Python methods (including those decorated with
+:func:`@staticmethod <staticmethod>` and :func:`@classmethod <classmethod>`) are
 implemented as non-data descriptors.  Accordingly, instances can redefine and
 override methods.  This allows individual instances to acquire behaviors that
 differ from other instances of the same class.
@@ -1879,46 +1885,50 @@ __slots__
 ^^^^^^^^^
 
 *__slots__* allow us to explicitly declare data members (like
-properties) and deny the creation of *__dict__* and *__weakref__*
+properties) and deny the creation of :attr:`~object.__dict__` and *__weakref__*
 (unless explicitly declared in *__slots__* or available in a parent.)
 
-The space saved over using *__dict__* can be significant.
+The space saved over using :attr:`~object.__dict__` can be significant.
 Attribute lookup speed can be significantly improved as well.
 
 .. data:: object.__slots__
 
    This class variable can be assigned a string, iterable, or sequence of
    strings with variable names used by instances.  *__slots__* reserves space
-   for the declared variables and prevents the automatic creation of *__dict__*
+   for the declared variables and prevents the automatic creation of
+   :attr:`~object.__dict__`
    and *__weakref__* for each instance.
 
 
 Notes on using *__slots__*
 """"""""""""""""""""""""""
 
-* When inheriting from a class without *__slots__*, the *__dict__* and
+* When inheriting from a class without *__slots__*, the
+  :attr:`~object.__dict__` and
   *__weakref__* attribute of the instances will always be accessible.
 
-* Without a *__dict__* variable, instances cannot be assigned new variables not
+* Without a :attr:`~object.__dict__` variable, instances cannot be assigned new
+  variables not
   listed in the *__slots__* definition.  Attempts to assign to an unlisted
   variable name raises :exc:`AttributeError`. If dynamic assignment of new
   variables is desired, then add ``'__dict__'`` to the sequence of strings in
   the *__slots__* declaration.
 
 * Without a *__weakref__* variable for each instance, classes defining
-  *__slots__* do not support weak references to its instances. If weak reference
+  *__slots__* do not support :mod:`weak references <weakref>` to its instances.
+  If weak reference
   support is needed, then add ``'__weakref__'`` to the sequence of strings in the
   *__slots__* declaration.
 
-* *__slots__* are implemented at the class level by creating descriptors
-  (:ref:`descriptors`) for each variable name.  As a result, class attributes
+* *__slots__* are implemented at the class level by creating :ref:`descriptors <descriptors>`
+  for each variable name.  As a result, class attributes
   cannot be used to set default values for instance variables defined by
   *__slots__*; otherwise, the class attribute would overwrite the descriptor
   assignment.
 
 * The action of a *__slots__* declaration is not limited to the class
   where it is defined.  *__slots__* declared in parents are available in
-  child classes. However, child subclasses will get a *__dict__*  and
+  child classes. However, child subclasses will get a :attr:`~object.__dict__` and
   *__weakref__* unless they also define *__slots__* (which should only
   contain names of any *additional* slots).
 
@@ -1934,14 +1944,17 @@ Notes on using *__slots__*
   used; however, in the future, special meaning may be assigned to the values
   corresponding to each key.
 
-* *__class__* assignment works only if both classes have the same *__slots__*.
+* :attr:`~instance.__class__` assignment works only if both classes have the
+  same *__slots__*.
 
-* Multiple inheritance with multiple slotted parent classes can be used,
+* :ref:`Multiple inheritance <tut-multiple>` with multiple slotted parent
+  classes can be used,
   but only one parent is allowed to have attributes created by slots
   (the other bases must have empty slot layouts) - violations raise
   :exc:`TypeError`.
 
-* If an iterator is used for *__slots__* then a descriptor is created for each
+* If an :term:`iterator` is used for *__slots__* then a :term:`descriptor` is
+  created for each
   of the iterator's values. However, the *__slots__* attribute will be an empty
   iterator.
 
@@ -1950,7 +1963,7 @@ Notes on using *__slots__*
 Customizing class creation
 --------------------------
 
-Whenever a class inherits from another class, *__init_subclass__* is
+Whenever a class inherits from another class, :meth:`~object.__init_subclass__` is
 called on that class. This way, it is possible to write classes which
 change the behavior of subclasses. This is closely related to class
 decorators, but where class decorators only affect the specific class they're
@@ -1991,7 +2004,7 @@ class defining the method.
 
 
 When a class is created, :meth:`type.__new__` scans the class variables
-and makes callbacks to those with a :meth:`__set_name__` hook.
+and makes callbacks to those with a :meth:`~object.__set_name__` hook.
 
 .. method:: object.__set_name__(self, owner, name)
 
@@ -2103,7 +2116,8 @@ Once the appropriate metaclass has been identified, then the class namespace
 is prepared. If the metaclass has a ``__prepare__`` attribute, it is called
 as ``namespace = metaclass.__prepare__(name, bases, **kwds)`` (where the
 additional keyword arguments, if any, come from the class definition). The
-``__prepare__`` method should be implemented as a :func:`classmethod`. The
+``__prepare__`` method should be implemented as a
+:func:`classmethod <classmethod>`. The
 namespace returned by ``__prepare__`` is passed in to ``__new__``, but when
 the final class object is created the namespace is copied into a new ``dict``.
 
@@ -2401,31 +2415,36 @@ Emulating container types
 -------------------------
 
 The following methods can be defined to implement container objects.  Containers
-usually are sequences (such as lists or tuples) or mappings (like dictionaries),
+usually are :term:`sequences <sequence>` (such as :class:`lists <list>` or
+:class:`tuples <tuple>`) or :term:`mappings <mapping>` (like
+:class:`dictionaries <dict>`),
 but can represent other containers as well.  The first set of methods is used
 either to emulate a sequence or to emulate a mapping; the difference is that for
 a sequence, the allowable keys should be the integers *k* for which ``0 <= k <
-N`` where *N* is the length of the sequence, or slice objects, which define a
+N`` where *N* is the length of the sequence, or :class:`slice` objects, which define a
 range of items.  It is also recommended that mappings provide the methods
 :meth:`keys`, :meth:`values`, :meth:`items`, :meth:`get`, :meth:`clear`,
 :meth:`setdefault`, :meth:`pop`, :meth:`popitem`, :meth:`!copy`, and
-:meth:`update` behaving similar to those for Python's standard dictionary
+:meth:`update` behaving similar to those for Python's standard :class:`dictionary <dict>`
 objects.  The :mod:`collections.abc` module provides a
 :class:`~collections.abc.MutableMapping`
-abstract base class to help create those methods from a base set of
-:meth:`__getitem__`, :meth:`__setitem__`, :meth:`__delitem__`, and :meth:`keys`.
+:term:`abstract base class` to help create those methods from a base set of
+:meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__delitem__`, and :meth:`keys`.
 Mutable sequences should provide methods :meth:`append`, :meth:`count`,
 :meth:`index`, :meth:`extend`, :meth:`insert`, :meth:`pop`, :meth:`remove`,
-:meth:`reverse` and :meth:`sort`, like Python standard list objects.  Finally,
+:meth:`reverse` and :meth:`sort`, like Python standard :class:`list`
+objects. Finally,
 sequence types should implement addition (meaning concatenation) and
-multiplication (meaning repetition) by defining the methods :meth:`__add__`,
-:meth:`__radd__`, :meth:`__iadd__`, :meth:`__mul__`, :meth:`__rmul__` and
-:meth:`__imul__` described below; they should not define other numerical
+multiplication (meaning repetition) by defining the methods
+:meth:`~object.__add__`, :meth:`~object.__radd__`, :meth:`~object.__iadd__`,
+:meth:`~object.__mul__`, :meth:`~object.__rmul__` and :meth:`~object.__imul__`
+described below; they should not define other numerical
 operators.  It is recommended that both mappings and sequences implement the
-:meth:`__contains__` method to allow efficient use of the ``in`` operator; for
+:meth:`~object.__contains__` method to allow efficient use of the ``in``
+operator; for
 mappings, ``in`` should search the mapping's keys; for sequences, it should
 search through the values.  It is further recommended that both mappings and
-sequences implement the :meth:`__iter__` method to allow efficient iteration
+sequences implement the :meth:`~object.__iter__` method to allow efficient iteration
 through the container; for mappings, :meth:`__iter__` should iterate
 through the object's keys; for sequences, it should iterate through the values.
 
@@ -2838,7 +2857,8 @@ exception::
    TypeError: object of type 'C' has no len()
 
 The rationale behind this behaviour lies with a number of special methods such
-as :meth:`__hash__` and :meth:`__repr__` that are implemented by all objects,
+as :meth:`~object.__hash__` and :meth:`~object.__repr__` that are implemented
+by all objects,
 including type objects. If the implicit lookup of these methods used the
 conventional lookup process, they would fail when invoked on the type object
 itself::
@@ -2861,7 +2881,7 @@ the instance when looking up special methods::
 
 In addition to bypassing any instance attributes in the interest of
 correctness, implicit special method lookup generally also bypasses the
-:meth:`__getattribute__` method even of the object's metaclass::
+:meth:`~object.__getattribute__` method even of the object's metaclass::
 
    >>> class Meta(type):
    ...     def __getattribute__(*args):
@@ -2885,7 +2905,7 @@ correctness, implicit special method lookup generally also bypasses the
    >>> len(c)                      # Implicit lookup
    10
 
-Bypassing the :meth:`__getattribute__` machinery in this fashion
+Bypassing the :meth:`~object.__getattribute__` machinery in this fashion
 provides significant scope for speed optimisations within the
 interpreter, at the cost of some flexibility in the handling of
 special methods (the special method *must* be set on the class
@@ -2902,7 +2922,7 @@ Coroutines
 Awaitable Objects
 -----------------
 
-An :term:`awaitable` object generally implements an :meth:`__await__` method.
+An :term:`awaitable` object generally implements an :meth:`~object.__await__` method.
 :term:`Coroutine objects <coroutine>` returned from :keyword:`async def` functions
 are awaitable.
 
@@ -2910,7 +2930,7 @@ are awaitable.
 
    The :term:`generator iterator` objects returned from generators
    decorated with :func:`types.coroutine`
-   are also awaitable, but they do not implement :meth:`__await__`.
+   are also awaitable, but they do not implement :meth:`~object.__await__`.
 
 .. method:: object.__await__(self)
 
@@ -2929,7 +2949,7 @@ Coroutine Objects
 -----------------
 
 :term:`Coroutine objects <coroutine>` are :term:`awaitable` objects.
-A coroutine's execution can be controlled by calling :meth:`__await__` and
+A coroutine's execution can be controlled by calling :meth:`~object.__await__` and
 iterating over the result.  When the coroutine has finished executing and
 returns, the iterator raises :exc:`StopIteration`, and the exception's
 :attr:`~StopIteration.value` attribute holds the return value.  If the
@@ -2948,7 +2968,7 @@ generators, coroutines do not directly support iteration.
 
    Starts or resumes execution of the coroutine.  If *value* is ``None``,
    this is equivalent to advancing the iterator returned by
-   :meth:`__await__`.  If *value* is not ``None``, this method delegates
+   :meth:`~object.__await__`.  If *value* is not ``None``, this method delegates
    to the :meth:`~generator.send` method of the iterator that caused
    the coroutine to suspend.  The result (return value,
    :exc:`StopIteration`, or other exception) is the same as when
@@ -2961,7 +2981,7 @@ generators, coroutines do not directly support iteration.
    the coroutine to suspend, if it has such a method.  Otherwise,
    the exception is raised at the suspension point.  The result
    (return value, :exc:`StopIteration`, or other exception) is the same as
-   when iterating over the :meth:`__await__` return value, described
+   when iterating over the :meth:`~object.__await__` return value, described
    above.  If the exception is not caught in the coroutine, it propagates
    back to the caller.
 
@@ -3015,11 +3035,11 @@ An example of an asynchronous iterable object::
 .. versionadded:: 3.5
 
 .. versionchanged:: 3.7
-   Prior to Python 3.7, ``__aiter__`` could return an *awaitable*
+   Prior to Python 3.7, :meth:`~object.__aiter__` could return an *awaitable*
    that would resolve to an
    :term:`asynchronous iterator <asynchronous iterator>`.
 
-   Starting with Python 3.7, ``__aiter__`` must return an
+   Starting with Python 3.7, :meth:`~object.__aiter__` must return an
    asynchronous iterator object.  Returning anything else
    will result in a :exc:`TypeError` error.
 
@@ -3062,8 +3082,9 @@ An example of an asynchronous context manager class::
    controlled conditions. It generally isn't a good idea though, since it can
    lead to some very strange behaviour if it is handled incorrectly.
 
-.. [#] The :meth:`__hash__`, :meth:`__iter__`, :meth:`__reversed__`, and
-   :meth:`__contains__` methods have special handling for this; others
+.. [#] The :meth:`~object.__hash__`, :meth:`~object.__iter__`,
+   :meth:`~object.__reversed__`, and :meth:`~object.__contains__` methods have
+   special handling for this; others
    will still raise a :exc:`TypeError`, but may do so by relying on
    the behavior that ``None`` is not callable.
 
@@ -3074,5 +3095,6 @@ An example of an asynchronous context manager class::
    *blocking* such fallback.
 
 .. [#] For operands of the same type, it is assumed that if the non-reflected
-   method -- such as :meth:`__add__` -- fails then the overall operation is not
+   method -- such as :meth:`~object.__add__` -- fails then the overall
+   operation is not
    supported, which is why the reflected method is not called.
diff --git a/Misc/NEWS.d/next/Documentation/2021-11-19-02-02-32.bpo-45840.A51B2S.rst b/Misc/NEWS.d/next/Documentation/2021-11-19-02-02-32.bpo-45840.A51B2S.rst
new file mode 100644
index 0000000000000..87371e5b76bc1
--- /dev/null
+++ b/Misc/NEWS.d/next/Documentation/2021-11-19-02-02-32.bpo-45840.A51B2S.rst
@@ -0,0 +1 @@
+Improve cross-references in the documentation for the data model.


From webhook-mailer at python.org  Sun Dec  5 16:02:52 2021
From: webhook-mailer at python.org (miss-islington)
Date: Sun, 05 Dec 2021 21:02:52 -0000
Subject: [Python-checkins] bpo-45662: Fix the repr of InitVar with a type
 alias to the built-in class (GH-29291)
Message-ID: <mailman.48.1638738174.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/f1dd5ed1f35a7ed5c3833c822e9965de2400d77e
commit: f1dd5ed1f35a7ed5c3833c822e9965de2400d77e
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-05T13:02:47-08:00
summary:

bpo-45662: Fix the repr of InitVar with a type alias to the built-in class (GH-29291)


For example, InitVar[list[int]].
(cherry picked from commit 1fd4de5bddbbf2a97cdbac4d298c89e1156bdc6c)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst
M Lib/dataclasses.py
M Lib/test/test_dataclasses.py

diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py
index b3a9194d9d068..fe4094b891b4a 100644
--- a/Lib/dataclasses.py
+++ b/Lib/dataclasses.py
@@ -229,7 +229,7 @@ def __init__(self, type):
         self.type = type
 
     def __repr__(self):
-        if isinstance(self.type, type):
+        if isinstance(self.type, type) and not isinstance(self.type, GenericAlias):
             type_name = self.type.__name__
         else:
             # typing objects, e.g. List[int]
diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py
index bcd004f4ec3aa..47075df8d59f3 100644
--- a/Lib/test/test_dataclasses.py
+++ b/Lib/test/test_dataclasses.py
@@ -1126,6 +1126,10 @@ def test_init_var_preserve_type(self):
         self.assertEqual(repr(InitVar[int]), 'dataclasses.InitVar[int]')
         self.assertEqual(repr(InitVar[List[int]]),
                          'dataclasses.InitVar[typing.List[int]]')
+        self.assertEqual(repr(InitVar[list[int]]),
+                         'dataclasses.InitVar[list[int]]')
+        self.assertEqual(repr(InitVar[int|str]),
+                         'dataclasses.InitVar[int | str]')
 
     def test_init_var_inheritance(self):
         # Note that this deliberately tests that a dataclass need not
diff --git a/Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst b/Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst
new file mode 100644
index 0000000000000..050b443dd7cb2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst
@@ -0,0 +1,2 @@
+Fix the repr of :data:`dataclasses.InitVar` with a type alias to the
+built-in class, e.g. ``InitVar[list[int]]``.


From webhook-mailer at python.org  Sun Dec  5 16:04:38 2021
From: webhook-mailer at python.org (miss-islington)
Date: Sun, 05 Dec 2021 21:04:38 -0000
Subject: [Python-checkins] bpo-45663: Fix is_dataclass() for dataclasses
 which are subclasses of types.GenericAlias (GH-29294)
Message-ID: <mailman.49.1638738279.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/abceb66c7e33d165361d8a26efb3770faa721aff
commit: abceb66c7e33d165361d8a26efb3770faa721aff
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-05T13:04:29-08:00
summary:

bpo-45663: Fix is_dataclass() for dataclasses which are subclasses of types.GenericAlias (GH-29294)

(cherry picked from commit 446be166861b2f08f87f74018113dd98ca5fca02)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
M Lib/dataclasses.py
M Lib/test/test_dataclasses.py

diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py
index fe4094b891b4a..105a95b95540e 100644
--- a/Lib/dataclasses.py
+++ b/Lib/dataclasses.py
@@ -1211,7 +1211,7 @@ def _is_dataclass_instance(obj):
 def is_dataclass(obj):
     """Returns True if obj is a dataclass or an instance of a
     dataclass."""
-    cls = obj if isinstance(obj, type) else type(obj)
+    cls = obj if isinstance(obj, type) and not isinstance(obj, GenericAlias) else type(obj)
     return hasattr(cls, _FIELDS)
 
 
diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py
index 47075df8d59f3..ef5009ab11677 100644
--- a/Lib/test/test_dataclasses.py
+++ b/Lib/test/test_dataclasses.py
@@ -8,6 +8,7 @@
 import pickle
 import inspect
 import builtins
+import types
 import unittest
 from unittest.mock import Mock
 from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol
@@ -1354,6 +1355,17 @@ class B:
                 with self.assertRaisesRegex(TypeError, 'should be called on dataclass instances'):
                     replace(obj, x=0)
 
+    def test_is_dataclass_genericalias(self):
+        @dataclass
+        class A(types.GenericAlias):
+            origin: type
+            args: type
+        self.assertTrue(is_dataclass(A))
+        a = A(list, int)
+        self.assertTrue(is_dataclass(type(a)))
+        self.assertTrue(is_dataclass(a))
+
+
     def test_helper_fields_with_class_instance(self):
         # Check that we can call fields() on either a class or instance,
         #  and get back the same thing.
diff --git a/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst b/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
new file mode 100644
index 0000000000000..f246f67cf80e5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
@@ -0,0 +1,2 @@
+Fix :func:`dataclasses.is_dataclass` for dataclasses which are subclasses of
+:class:`types.GenericAlias`.


From webhook-mailer at python.org  Sun Dec  5 16:25:55 2021
From: webhook-mailer at python.org (miss-islington)
Date: Sun, 05 Dec 2021 21:25:55 -0000
Subject: [Python-checkins] bpo-45663: Fix is_dataclass() for dataclasses
 which are subclasses of types.GenericAlias (GH-29294)
Message-ID: <mailman.50.1638739555.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/19050711f5a68e50b942b3b7f1f4cf398f27efff
commit: 19050711f5a68e50b942b3b7f1f4cf398f27efff
branch: 3.9
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-05T13:25:43-08:00
summary:

bpo-45663: Fix is_dataclass() for dataclasses which are subclasses of types.GenericAlias (GH-29294)

(cherry picked from commit 446be166861b2f08f87f74018113dd98ca5fca02)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
M Lib/dataclasses.py
M Lib/test/test_dataclasses.py

diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py
index c98e74d4ff9cc..68907058403fd 100644
--- a/Lib/dataclasses.py
+++ b/Lib/dataclasses.py
@@ -1047,7 +1047,7 @@ def _is_dataclass_instance(obj):
 def is_dataclass(obj):
     """Returns True if obj is a dataclass or an instance of a
     dataclass."""
-    cls = obj if isinstance(obj, type) else type(obj)
+    cls = obj if isinstance(obj, type) and not isinstance(obj, GenericAlias) else type(obj)
     return hasattr(cls, _FIELDS)
 
 
diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py
index ad981d1cc654f..fa5adfcea1b43 100644
--- a/Lib/test/test_dataclasses.py
+++ b/Lib/test/test_dataclasses.py
@@ -7,6 +7,7 @@
 import pickle
 import inspect
 import builtins
+import types
 import unittest
 from unittest.mock import Mock
 from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol
@@ -1348,6 +1349,17 @@ class B:
                 with self.assertRaisesRegex(TypeError, 'should be called on dataclass instances'):
                     replace(obj, x=0)
 
+    def test_is_dataclass_genericalias(self):
+        @dataclass
+        class A(types.GenericAlias):
+            origin: type
+            args: type
+        self.assertTrue(is_dataclass(A))
+        a = A(list, int)
+        self.assertTrue(is_dataclass(type(a)))
+        self.assertTrue(is_dataclass(a))
+
+
     def test_helper_fields_with_class_instance(self):
         # Check that we can call fields() on either a class or instance,
         #  and get back the same thing.
diff --git a/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst b/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
new file mode 100644
index 0000000000000..f246f67cf80e5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
@@ -0,0 +1,2 @@
+Fix :func:`dataclasses.is_dataclass` for dataclasses which are subclasses of
+:class:`types.GenericAlias`.


From webhook-mailer at python.org  Sun Dec  5 16:26:47 2021
From: webhook-mailer at python.org (miss-islington)
Date: Sun, 05 Dec 2021 21:26:47 -0000
Subject: [Python-checkins] bpo-45664: Fix resolve_bases() and new_class()
 for GenericAlias instance as a base (GH-29298)
Message-ID: <mailman.51.1638739608.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/cb68c0a3a4aeb4ec58ab1f71b70bc8bfecbceef6
commit: cb68c0a3a4aeb4ec58ab1f71b70bc8bfecbceef6
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-05T13:26:37-08:00
summary:

bpo-45664: Fix resolve_bases() and new_class() for GenericAlias instance as a base (GH-29298)

(cherry picked from commit 2b318ce1c988b7b6e3caf293d55f289e066b6e0f)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
M Lib/test/test_types.py
M Lib/types.py

diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py
index b1218abc8af5e..3dfda5cb95663 100644
--- a/Lib/test/test_types.py
+++ b/Lib/test/test_types.py
@@ -1236,6 +1236,17 @@ def __mro_entries__(self, bases):
         self.assertEqual(D.__orig_bases__, (c,))
         self.assertEqual(D.__mro__, (D, A, object))
 
+    def test_new_class_with_mro_entry_genericalias(self):
+        L1 = types.new_class('L1', (typing.List[int],), {})
+        self.assertEqual(L1.__bases__, (list, typing.Generic))
+        self.assertEqual(L1.__orig_bases__, (typing.List[int],))
+        self.assertEqual(L1.__mro__, (L1, list, typing.Generic, object))
+
+        L2 = types.new_class('L2', (list[int],), {})
+        self.assertEqual(L2.__bases__, (list,))
+        self.assertEqual(L2.__orig_bases__, (list[int],))
+        self.assertEqual(L2.__mro__, (L2, list, object))
+
     def test_new_class_with_mro_entry_none(self):
         class A: pass
         class B: pass
@@ -1351,6 +1362,11 @@ def __mro_entries__(self, bases):
         for bases in [x, y, z, t]:
             self.assertIs(types.resolve_bases(bases), bases)
 
+    def test_resolve_bases_with_mro_entry(self):
+        self.assertEqual(types.resolve_bases((typing.List[int],)),
+                         (list, typing.Generic))
+        self.assertEqual(types.resolve_bases((list[int],)), (list,))
+
     def test_metaclass_derivation(self):
         # issue1294232: correct metaclass calculation
         new_calls = []  # to check the order of __new__ calls
diff --git a/Lib/types.py b/Lib/types.py
index c2dc97dd57ae3..62122a994866f 100644
--- a/Lib/types.py
+++ b/Lib/types.py
@@ -82,7 +82,7 @@ def resolve_bases(bases):
     updated = False
     shift = 0
     for i, base in enumerate(bases):
-        if isinstance(base, type):
+        if isinstance(base, type) and not isinstance(base, GenericAlias):
             continue
         if not hasattr(base, "__mro_entries__"):
             continue
diff --git a/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst b/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
new file mode 100644
index 0000000000000..573a569845a87
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
@@ -0,0 +1,2 @@
+Fix :func:`types.resolve_bases` and :func:`types.new_class` for
+:class:`types.GenericAlias` instance as a base.


From webhook-mailer at python.org  Mon Dec  6 02:39:16 2021
From: webhook-mailer at python.org (ned-deily)
Date: Mon, 06 Dec 2021 07:39:16 -0000
Subject: [Python-checkins] bpo-45732: Update python.org macOS installer to
 use Tcl/Tk 8.6.12. (GH-29932)
Message-ID: <mailman.52.1638776357.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/20801982fa696a01ad2c116925bddf2fda106338
commit: 20801982fa696a01ad2c116925bddf2fda106338
branch: 3.10
author: Ned Deily <nad at python.org>
committer: ned-deily <nad at python.org>
date: 2021-12-06T02:39:11-05:00
summary:

bpo-45732: Update python.org macOS installer to use Tcl/Tk 8.6.12. (GH-29932)

files:
A Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
D Mac/BuildScript/bpo-44828-filedialog-crash-monterey.patch
M Mac/BuildScript/build-installer.py

diff --git a/Mac/BuildScript/bpo-44828-filedialog-crash-monterey.patch b/Mac/BuildScript/bpo-44828-filedialog-crash-monterey.patch
deleted file mode 100644
index 059149250f8c8..0000000000000
--- a/Mac/BuildScript/bpo-44828-filedialog-crash-monterey.patch
+++ /dev/null
@@ -1,203 +0,0 @@
---- tk8.6.11/macosx/tkMacOSXDialog.c	2020-12-31 01:46:07.000000000 +0000
-+++ tk8.6.11-patched/macosx/tkMacOSXDialog.c	2021-10-28 15:13:03.000000000 +0000
-@@ -221,7 +221,7 @@
- 		returnCode: (NSModalResponse) returnCode
- 	       contextInfo: (void *) contextInfo
- {
--    FilePanelCallbackInfo *callbackInfo = contextInfo;
-+    FilePanelCallbackInfo *callbackInfo = (FilePanelCallbackInfo *)contextInfo;
- 
-     if (returnCode == modalOK) {
- 	Tcl_Obj *resultObj;
-@@ -266,7 +266,7 @@
- - (void) tkAlertDidEnd: (NSAlert *) alert returnCode: (NSInteger) returnCode
- 	contextInfo: (void *) contextInfo
- {
--    AlertCallbackInfo *callbackInfo = contextInfo;
-+    AlertCallbackInfo *callbackInfo = (AlertCallbackInfo *)contextInfo;
- 
-     if (returnCode >= NSAlertFirstButtonReturn) {
- 	Tcl_Obj *resultObj = Tcl_NewStringObj(alertButtonStrings[
-@@ -350,49 +350,42 @@
-     FilePanelCallbackInfo *callbackInfo)
- {
-     NSInteger modalReturnCode;
-+    int OSVersion = [NSApp macOSVersion];
- 
--    if (parent && ![parent attachedSheet]) {
--	[panel beginSheetModalForWindow:parent
--	       completionHandler:^(NSModalResponse returnCode) {
--	    [NSApp tkFilePanelDidEnd:panel
--		       returnCode:returnCode
--		       contextInfo:callbackInfo ];
--	    }];
--
--	/*
--	 * The sheet has been prepared, so now we have to run it as a modal
--	 * window.  Using [NSApp runModalForWindow:] on macOS 10.15 or later
--	 * generates warnings on stderr.  But using [NSOpenPanel runModal] or
--	 * [NSSavePanel runModal] on 10.14 or earler does not cause the
--	 * completion handler to run when the panel is closed.
--	 */
-+    /*
-+     * Use a sheet if -parent is specified (unless there is already a sheet).
-+     */
- 
--	if ([NSApp macOSVersion] > 101400) {
--	    modalReturnCode = [panel runModal];
--	} else {
-+    if (parent && ![parent attachedSheet]) {
-+	if (OSVersion < 101500) {
-+	    [panel beginSheetModalForWindow:parent
-+			  completionHandler:^(NSModalResponse returnCode) {
-+		    [NSApp tkFilePanelDidEnd:panel
-+				  returnCode:returnCode
-+				 contextInfo:callbackInfo ];
-+		}];
- 	    modalReturnCode = [NSApp runModalForWindow:panel];
--	}
--    } else {
--
--	/*
--	 * For the standalone file dialog, completion handlers do not work
--	 * at all on macOS 10.14 and earlier.
--	 */
--
--	if ([NSApp macOSVersion] > 101400) {
--	    [panel beginWithCompletionHandler:^(NSModalResponse returnCode) {
-+	} else if (OSVersion < 110000) {
-+	    [panel beginSheetModalForWindow:parent
-+			  completionHandler:^(NSModalResponse returnCode) {
- 		    [NSApp tkFilePanelDidEnd:panel
--			          returnCode:returnCode
--			         contextInfo:callbackInfo ];
--	    }];
-+				  returnCode:returnCode
-+				 contextInfo:callbackInfo ];
-+		}];
- 	    modalReturnCode = [panel runModal];
- 	} else {
-+	    [parent beginSheet: panel completionHandler:nil];
- 	    modalReturnCode = [panel runModal];
- 	    [NSApp tkFilePanelDidEnd:panel
--		   	  returnCode:modalReturnCode
--		         contextInfo:callbackInfo ];
--	    [panel close];
-+			  returnCode:modalReturnCode
-+			 contextInfo:callbackInfo ];
-+		[parent endSheet:panel];
- 	}
-+    } else {
-+	modalReturnCode = [panel runModal];
-+	[NSApp tkFilePanelDidEnd:panel
-+		      returnCode:modalReturnCode
-+		     contextInfo:callbackInfo ];
-     }
-     return callbackInfo->cmdObj ? modalOther : modalReturnCode;
- }
-@@ -422,7 +414,7 @@
-     Tcl_Obj *const objv[])	/* Argument objects. */
- {
-     int result = TCL_ERROR;
--    Tk_Window parent, tkwin = clientData;
-+    Tk_Window parent, tkwin = (Tk_Window)clientData;
-     const char *title = NULL;
-     int i;
-     NSColor *color = nil, *initialColor = nil;
-@@ -677,7 +669,7 @@
-     int objc,			/* Number of arguments. */
-     Tcl_Obj *const objv[])	/* Argument objects. */
- {
--    Tk_Window tkwin = clientData;
-+    Tk_Window tkwin = (Tk_Window)clientData;
-     char *str;
-     int i, result = TCL_ERROR, haveParentOption = 0;
-     int index, len, multiple = 0;
-@@ -1679,10 +1671,10 @@
-     if (!fontchooserInterp) {
- 	return;
-     }
--    fcdPtr = Tcl_GetAssocData(fontchooserInterp, "::tk::fontchooser", NULL);
-+    fcdPtr = (FontchooserData *)Tcl_GetAssocData(fontchooserInterp, "::tk::fontchooser", NULL);
-     switch (kind) {
-     case FontchooserClosed:
--	if (fcdPtr->parent != None) {
-+	if (fcdPtr->parent != NULL) {
- 	    TkSendVirtualEvent(fcdPtr->parent, "TkFontchooserVisibility", NULL);
- 	    fontchooserInterp = NULL;
- 	}
-@@ -1738,7 +1730,7 @@
- 
-     switch(optionIndex) {
-     case FontchooserParent:
--	if (fcdPtr->parent != None) {
-+	if (fcdPtr->parent != NULL) {
- 	    resObj = Tcl_NewStringObj(
- 		    ((TkWindow *)fcdPtr->parent)->pathName, -1);
- 	} else {
-@@ -1801,7 +1793,7 @@
-     Tcl_Obj *const objv[])
- {
-     Tk_Window tkwin = (Tk_Window)clientData;
--    FontchooserData *fcdPtr = Tcl_GetAssocData(interp, "::tk::fontchooser",
-+    FontchooserData *fcdPtr = (FontchooserData *)Tcl_GetAssocData(interp, "::tk::fontchooser",
- 	    NULL);
-     int i, r = TCL_OK;
- 
-@@ -1858,7 +1850,7 @@
- 	    Tk_Window parent = Tk_NameToWindow(interp,
- 		    Tcl_GetString(objv[i+1]), tkwin);
- 
--	    if (parent == None) {
-+	    if (parent == NULL) {
- 		return TCL_ERROR;
- 	    }
- 	    if (fcdPtr->parent) {
-@@ -1885,7 +1877,7 @@
- 		fcdPtr->titleObj = NULL;
- 	    }
- 	    break;
--	case FontchooserFont:
-+	case FontchooserFont: {
- 	    Tcl_GetStringFromObj(objv[i+1], &len);
- 	    if (len) {
- 		Tk_Font f = Tk_AllocFontFromObj(interp, tkwin, objv[i+1]);
-@@ -1919,6 +1911,7 @@
- 			"TkFontchooserFontChanged", NULL);
- 	    }
- 	    break;
-+	}
- 	case FontchooserCmd:
- 	    if (fcdPtr->cmdObj) {
- 		Tcl_DecrRefCount(fcdPtr->cmdObj);
-@@ -1964,10 +1957,10 @@
-     TCL_UNUSED(int),
-     TCL_UNUSED(Tcl_Obj *const *))
- {
--    FontchooserData *fcdPtr = Tcl_GetAssocData(interp, "::tk::fontchooser",
-+    FontchooserData *fcdPtr = (FontchooserData *)Tcl_GetAssocData(interp, "::tk::fontchooser",
- 	    NULL);
- 
--    if (fcdPtr->parent == None) {
-+    if (fcdPtr->parent == NULL) {
- 	fcdPtr->parent = (Tk_Window)clientData;
- 	Tk_CreateEventHandler(fcdPtr->parent, StructureNotifyMask,
- 		FontchooserParentEventHandler, fcdPtr);
-@@ -2042,7 +2035,7 @@
-     ClientData clientData,
-     XEvent *eventPtr)
- {
--    FontchooserData *fcdPtr = clientData;
-+    FontchooserData *fcdPtr = (FontchooserData *)clientData;
- 
-     if (eventPtr->type == DestroyNotify) {
- 	Tk_DeleteEventHandler(fcdPtr->parent, StructureNotifyMask,
-@@ -2074,7 +2067,7 @@
-     ClientData clientData,
-     Tcl_Interp *interp)
- {
--    FontchooserData *fcdPtr = clientData;
-+    FontchooserData *fcdPtr = (FontchooserData *)clientData;
- 
-     if (fcdPtr->titleObj) {
- 	Tcl_DecrRefCount(fcdPtr->titleObj);
diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
index 86d31688da75c..9fefe67753666 100755
--- a/Mac/BuildScript/build-installer.py
+++ b/Mac/BuildScript/build-installer.py
@@ -265,11 +265,11 @@ def library_recipes():
             tk_patches = ['tk868_on_10_8_10_9.patch']
 
         else:
-            tcl_tk_ver='8.6.11'
-            tcl_checksum='8a4c004f48984a03a7747e9ba06e4da4'
+            tcl_tk_ver='8.6.12'
+            tcl_checksum='87ea890821d2221f2ab5157bc5eb885f'
 
-            tk_checksum='c7ee71a2d05bba78dfffd76528dc17c6'
-            tk_patches = ['bpo-44828-filedialog-crash-monterey.patch']
+            tk_checksum='1d6dcf6120356e3d211e056dff5e462a'
+            tk_patches = [ ]
 
 
         result.extend([
diff --git a/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst b/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
new file mode 100644
index 0000000000000..eb47985f86f95
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
@@ -0,0 +1 @@
+Update python.org macOS installer to use Tcl/Tk 8.6.12.


From webhook-mailer at python.org  Mon Dec  6 02:40:01 2021
From: webhook-mailer at python.org (ned-deily)
Date: Mon, 06 Dec 2021 07:40:01 -0000
Subject: [Python-checkins] bpo-45732: Update python.org macOS installer to
 use Tcl/Tk 8.6.12. (GH-29933)
Message-ID: <mailman.53.1638776402.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/7accb4fdb3dc1b4140089edee290998aff496fe2
commit: 7accb4fdb3dc1b4140089edee290998aff496fe2
branch: 3.9
author: Ned Deily <nad at python.org>
committer: ned-deily <nad at python.org>
date: 2021-12-06T02:39:56-05:00
summary:

bpo-45732: Update python.org macOS installer to use Tcl/Tk 8.6.12. (GH-29933)

files:
A Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
D Mac/BuildScript/bpo-44828-filedialog-crash-monterey.patch
M Mac/BuildScript/build-installer.py

diff --git a/Mac/BuildScript/bpo-44828-filedialog-crash-monterey.patch b/Mac/BuildScript/bpo-44828-filedialog-crash-monterey.patch
deleted file mode 100644
index 059149250f8c8..0000000000000
--- a/Mac/BuildScript/bpo-44828-filedialog-crash-monterey.patch
+++ /dev/null
@@ -1,203 +0,0 @@
---- tk8.6.11/macosx/tkMacOSXDialog.c	2020-12-31 01:46:07.000000000 +0000
-+++ tk8.6.11-patched/macosx/tkMacOSXDialog.c	2021-10-28 15:13:03.000000000 +0000
-@@ -221,7 +221,7 @@
- 		returnCode: (NSModalResponse) returnCode
- 	       contextInfo: (void *) contextInfo
- {
--    FilePanelCallbackInfo *callbackInfo = contextInfo;
-+    FilePanelCallbackInfo *callbackInfo = (FilePanelCallbackInfo *)contextInfo;
- 
-     if (returnCode == modalOK) {
- 	Tcl_Obj *resultObj;
-@@ -266,7 +266,7 @@
- - (void) tkAlertDidEnd: (NSAlert *) alert returnCode: (NSInteger) returnCode
- 	contextInfo: (void *) contextInfo
- {
--    AlertCallbackInfo *callbackInfo = contextInfo;
-+    AlertCallbackInfo *callbackInfo = (AlertCallbackInfo *)contextInfo;
- 
-     if (returnCode >= NSAlertFirstButtonReturn) {
- 	Tcl_Obj *resultObj = Tcl_NewStringObj(alertButtonStrings[
-@@ -350,49 +350,42 @@
-     FilePanelCallbackInfo *callbackInfo)
- {
-     NSInteger modalReturnCode;
-+    int OSVersion = [NSApp macOSVersion];
- 
--    if (parent && ![parent attachedSheet]) {
--	[panel beginSheetModalForWindow:parent
--	       completionHandler:^(NSModalResponse returnCode) {
--	    [NSApp tkFilePanelDidEnd:panel
--		       returnCode:returnCode
--		       contextInfo:callbackInfo ];
--	    }];
--
--	/*
--	 * The sheet has been prepared, so now we have to run it as a modal
--	 * window.  Using [NSApp runModalForWindow:] on macOS 10.15 or later
--	 * generates warnings on stderr.  But using [NSOpenPanel runModal] or
--	 * [NSSavePanel runModal] on 10.14 or earler does not cause the
--	 * completion handler to run when the panel is closed.
--	 */
-+    /*
-+     * Use a sheet if -parent is specified (unless there is already a sheet).
-+     */
- 
--	if ([NSApp macOSVersion] > 101400) {
--	    modalReturnCode = [panel runModal];
--	} else {
-+    if (parent && ![parent attachedSheet]) {
-+	if (OSVersion < 101500) {
-+	    [panel beginSheetModalForWindow:parent
-+			  completionHandler:^(NSModalResponse returnCode) {
-+		    [NSApp tkFilePanelDidEnd:panel
-+				  returnCode:returnCode
-+				 contextInfo:callbackInfo ];
-+		}];
- 	    modalReturnCode = [NSApp runModalForWindow:panel];
--	}
--    } else {
--
--	/*
--	 * For the standalone file dialog, completion handlers do not work
--	 * at all on macOS 10.14 and earlier.
--	 */
--
--	if ([NSApp macOSVersion] > 101400) {
--	    [panel beginWithCompletionHandler:^(NSModalResponse returnCode) {
-+	} else if (OSVersion < 110000) {
-+	    [panel beginSheetModalForWindow:parent
-+			  completionHandler:^(NSModalResponse returnCode) {
- 		    [NSApp tkFilePanelDidEnd:panel
--			          returnCode:returnCode
--			         contextInfo:callbackInfo ];
--	    }];
-+				  returnCode:returnCode
-+				 contextInfo:callbackInfo ];
-+		}];
- 	    modalReturnCode = [panel runModal];
- 	} else {
-+	    [parent beginSheet: panel completionHandler:nil];
- 	    modalReturnCode = [panel runModal];
- 	    [NSApp tkFilePanelDidEnd:panel
--		   	  returnCode:modalReturnCode
--		         contextInfo:callbackInfo ];
--	    [panel close];
-+			  returnCode:modalReturnCode
-+			 contextInfo:callbackInfo ];
-+		[parent endSheet:panel];
- 	}
-+    } else {
-+	modalReturnCode = [panel runModal];
-+	[NSApp tkFilePanelDidEnd:panel
-+		      returnCode:modalReturnCode
-+		     contextInfo:callbackInfo ];
-     }
-     return callbackInfo->cmdObj ? modalOther : modalReturnCode;
- }
-@@ -422,7 +414,7 @@
-     Tcl_Obj *const objv[])	/* Argument objects. */
- {
-     int result = TCL_ERROR;
--    Tk_Window parent, tkwin = clientData;
-+    Tk_Window parent, tkwin = (Tk_Window)clientData;
-     const char *title = NULL;
-     int i;
-     NSColor *color = nil, *initialColor = nil;
-@@ -677,7 +669,7 @@
-     int objc,			/* Number of arguments. */
-     Tcl_Obj *const objv[])	/* Argument objects. */
- {
--    Tk_Window tkwin = clientData;
-+    Tk_Window tkwin = (Tk_Window)clientData;
-     char *str;
-     int i, result = TCL_ERROR, haveParentOption = 0;
-     int index, len, multiple = 0;
-@@ -1679,10 +1671,10 @@
-     if (!fontchooserInterp) {
- 	return;
-     }
--    fcdPtr = Tcl_GetAssocData(fontchooserInterp, "::tk::fontchooser", NULL);
-+    fcdPtr = (FontchooserData *)Tcl_GetAssocData(fontchooserInterp, "::tk::fontchooser", NULL);
-     switch (kind) {
-     case FontchooserClosed:
--	if (fcdPtr->parent != None) {
-+	if (fcdPtr->parent != NULL) {
- 	    TkSendVirtualEvent(fcdPtr->parent, "TkFontchooserVisibility", NULL);
- 	    fontchooserInterp = NULL;
- 	}
-@@ -1738,7 +1730,7 @@
- 
-     switch(optionIndex) {
-     case FontchooserParent:
--	if (fcdPtr->parent != None) {
-+	if (fcdPtr->parent != NULL) {
- 	    resObj = Tcl_NewStringObj(
- 		    ((TkWindow *)fcdPtr->parent)->pathName, -1);
- 	} else {
-@@ -1801,7 +1793,7 @@
-     Tcl_Obj *const objv[])
- {
-     Tk_Window tkwin = (Tk_Window)clientData;
--    FontchooserData *fcdPtr = Tcl_GetAssocData(interp, "::tk::fontchooser",
-+    FontchooserData *fcdPtr = (FontchooserData *)Tcl_GetAssocData(interp, "::tk::fontchooser",
- 	    NULL);
-     int i, r = TCL_OK;
- 
-@@ -1858,7 +1850,7 @@
- 	    Tk_Window parent = Tk_NameToWindow(interp,
- 		    Tcl_GetString(objv[i+1]), tkwin);
- 
--	    if (parent == None) {
-+	    if (parent == NULL) {
- 		return TCL_ERROR;
- 	    }
- 	    if (fcdPtr->parent) {
-@@ -1885,7 +1877,7 @@
- 		fcdPtr->titleObj = NULL;
- 	    }
- 	    break;
--	case FontchooserFont:
-+	case FontchooserFont: {
- 	    Tcl_GetStringFromObj(objv[i+1], &len);
- 	    if (len) {
- 		Tk_Font f = Tk_AllocFontFromObj(interp, tkwin, objv[i+1]);
-@@ -1919,6 +1911,7 @@
- 			"TkFontchooserFontChanged", NULL);
- 	    }
- 	    break;
-+	}
- 	case FontchooserCmd:
- 	    if (fcdPtr->cmdObj) {
- 		Tcl_DecrRefCount(fcdPtr->cmdObj);
-@@ -1964,10 +1957,10 @@
-     TCL_UNUSED(int),
-     TCL_UNUSED(Tcl_Obj *const *))
- {
--    FontchooserData *fcdPtr = Tcl_GetAssocData(interp, "::tk::fontchooser",
-+    FontchooserData *fcdPtr = (FontchooserData *)Tcl_GetAssocData(interp, "::tk::fontchooser",
- 	    NULL);
- 
--    if (fcdPtr->parent == None) {
-+    if (fcdPtr->parent == NULL) {
- 	fcdPtr->parent = (Tk_Window)clientData;
- 	Tk_CreateEventHandler(fcdPtr->parent, StructureNotifyMask,
- 		FontchooserParentEventHandler, fcdPtr);
-@@ -2042,7 +2035,7 @@
-     ClientData clientData,
-     XEvent *eventPtr)
- {
--    FontchooserData *fcdPtr = clientData;
-+    FontchooserData *fcdPtr = (FontchooserData *)clientData;
- 
-     if (eventPtr->type == DestroyNotify) {
- 	Tk_DeleteEventHandler(fcdPtr->parent, StructureNotifyMask,
-@@ -2074,7 +2067,7 @@
-     ClientData clientData,
-     Tcl_Interp *interp)
- {
--    FontchooserData *fcdPtr = clientData;
-+    FontchooserData *fcdPtr = (FontchooserData *)clientData;
- 
-     if (fcdPtr->titleObj) {
- 	Tcl_DecrRefCount(fcdPtr->titleObj);
diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
index 10112fe330061..cc24d54753924 100755
--- a/Mac/BuildScript/build-installer.py
+++ b/Mac/BuildScript/build-installer.py
@@ -263,11 +263,11 @@ def library_recipes():
             tk_patches = ['tk868_on_10_8_10_9.patch']
 
         else:
-            tcl_tk_ver='8.6.11'
-            tcl_checksum='8a4c004f48984a03a7747e9ba06e4da4'
+            tcl_tk_ver='8.6.12'
+            tcl_checksum='87ea890821d2221f2ab5157bc5eb885f'
 
-            tk_checksum='c7ee71a2d05bba78dfffd76528dc17c6'
-            tk_patches = ['bpo-44828-filedialog-crash-monterey.patch']
+            tk_checksum='1d6dcf6120356e3d211e056dff5e462a'
+            tk_patches = [ ]
 
 
         result.extend([
diff --git a/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst b/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
new file mode 100644
index 0000000000000..eb47985f86f95
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
@@ -0,0 +1 @@
+Update python.org macOS installer to use Tcl/Tk 8.6.12.


From webhook-mailer at python.org  Mon Dec  6 02:44:21 2021
From: webhook-mailer at python.org (ned-deily)
Date: Mon, 06 Dec 2021 07:44:21 -0000
Subject: [Python-checkins] bpo-45732: Update python.org macOS installer to
 use Tcl/Tk 8.6.12. (GH-29931)
Message-ID: <mailman.54.1638776662.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/f34d181fa15e1f082140a4e4a8fa3b77118f8e98
commit: f34d181fa15e1f082140a4e4a8fa3b77118f8e98
branch: main
author: Ned Deily <nad at python.org>
committer: ned-deily <nad at python.org>
date: 2021-12-06T02:44:09-05:00
summary:

bpo-45732: Update python.org macOS installer to use Tcl/Tk 8.6.12. (GH-29931)

files:
A Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
D Mac/BuildScript/bpo-44828-filedialog-crash-monterey-8612rc1.patch
M Mac/BuildScript/build-installer.py

diff --git a/Mac/BuildScript/bpo-44828-filedialog-crash-monterey-8612rc1.patch b/Mac/BuildScript/bpo-44828-filedialog-crash-monterey-8612rc1.patch
deleted file mode 100644
index e70e2c35dc113..0000000000000
--- a/Mac/BuildScript/bpo-44828-filedialog-crash-monterey-8612rc1.patch
+++ /dev/null
@@ -1,10 +0,0 @@
---- tk8.6.12/macosx/tkMacOSXDialog.c	2021-10-29 17:08:23.000000000 +0000
-+++ tk8.6.12-patched/macosx/tkMacOSXDialog.c	2021-11-02 19:04:59.000000000 +0000
-@@ -379,6 +379,7 @@
- 	    [NSApp tkFilePanelDidEnd:panel
- 			  returnCode:modalReturnCode
- 			 contextInfo:callbackInfo ];
-+	    [parent endSheet:panel];
- 	}
-     } else {
- 	modalReturnCode = [panel runModal];
diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py
index 1f6fcafc6f92c..5365f62bf3d0e 100755
--- a/Mac/BuildScript/build-installer.py
+++ b/Mac/BuildScript/build-installer.py
@@ -265,11 +265,11 @@ def library_recipes():
             tk_patches = ['tk868_on_10_8_10_9.patch']
 
         else:
-            tcl_tk_ver='8.6.12rc1'
-            tcl_checksum='82fd1637c0f7d4b76cb909f8abc373ec'
+            tcl_tk_ver='8.6.12'
+            tcl_checksum='87ea890821d2221f2ab5157bc5eb885f'
 
-            tk_checksum='d63c3b91b86cd8b6fa54e83ef2c5153e'
-            tk_patches = ['bpo-44828-filedialog-crash-monterey-8612rc1.patch']
+            tk_checksum='1d6dcf6120356e3d211e056dff5e462a'
+            tk_patches = [ ]
 
 
         result.extend([
diff --git a/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst b/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
new file mode 100644
index 0000000000000..eb47985f86f95
--- /dev/null
+++ b/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
@@ -0,0 +1 @@
+Update python.org macOS installer to use Tcl/Tk 8.6.12.


From webhook-mailer at python.org  Mon Dec  6 05:14:00 2021
From: webhook-mailer at python.org (markshannon)
Date: Mon, 06 Dec 2021 10:14:00 -0000
Subject: [Python-checkins] bpo-45963: Make space for the InterpreterFrame of
 a generator in that generator. (GH-29891)
Message-ID: <mailman.55.1638785641.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/299483c95d601ddcfdce2f96418b6499c1fc7b9f
commit: 299483c95d601ddcfdce2f96418b6499c1fc7b9f
branch: main
author: Mark Shannon <mark at hotpy.org>
committer: markshannon <mark at hotpy.org>
date: 2021-12-06T10:13:49Z
summary:

bpo-45963: Make space for the InterpreterFrame of a generator in that generator. (GH-29891)

* Make generator, coroutine and async gen structs all the same size.

* Store interpreter frame in generator (and coroutine). Reduces the number of allocations neeeded for a generator from two to one.

files:
M Include/cpython/genobject.h
M Include/internal/pycore_ceval.h
M Include/internal/pycore_frame.h
M Lib/test/test_sys.py
M Objects/genobject.c
M Python/ceval.c
M Python/frame.c

diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h
index 8f87cf5fff757..ad2818e881667 100644
--- a/Include/cpython/genobject.h
+++ b/Include/cpython/genobject.h
@@ -14,7 +14,6 @@ extern "C" {
 #define _PyGenObject_HEAD(prefix)                                           \
     PyObject_HEAD                                                           \
     /* Note: gi_frame can be NULL if the generator is "finished" */         \
-    struct _interpreter_frame *prefix##_xframe;                             \
     /* The code object backing the generator */                             \
     PyCodeObject *prefix##_code;                                            \
     /* List of weak reference. */                                           \
@@ -23,7 +22,14 @@ extern "C" {
     PyObject *prefix##_name;                                                \
     /* Qualified name of the generator. */                                  \
     PyObject *prefix##_qualname;                                            \
-    _PyErr_StackItem prefix##_exc_state;
+    _PyErr_StackItem prefix##_exc_state;                                    \
+    PyObject *prefix##_origin_or_finalizer;                                 \
+    char prefix##_hooks_inited;                                             \
+    char prefix##_closed;                                                   \
+    char prefix##_running_async;                                            \
+    /* The frame */                                                         \
+    char prefix##_frame_valid;                                              \
+    PyObject *prefix##_iframe[1];
 
 typedef struct {
     /* The gi_ prefix is intended to remind of generator-iterator. */
@@ -48,7 +54,6 @@ PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self);
 
 typedef struct {
     _PyGenObject_HEAD(cr)
-    PyObject *cr_origin;
 } PyCoroObject;
 
 PyAPI_DATA(PyTypeObject) PyCoro_Type;
@@ -64,18 +69,6 @@ PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *,
 
 typedef struct {
     _PyGenObject_HEAD(ag)
-    PyObject *ag_finalizer;
-
-    /* Flag is set to 1 when hooks set up by sys.set_asyncgen_hooks
-       were called on the generator, to avoid calling them more
-       than once. */
-    int ag_hooks_inited;
-
-    /* Flag is set to 1 when aclose() is called for the first time, or
-       when a StopAsyncIteration exception is raised. */
-    int ag_closed;
-
-    int ag_running_async;
 } PyAsyncGenObject;
 
 PyAPI_DATA(PyTypeObject) PyAsyncGen_Type;
diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index 9987f2076cd59..26d5677b12837 100644
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -113,7 +113,7 @@ static inline void _Py_LeaveRecursiveCall_inline(void)  {
 
 struct _interpreter_frame *_PyEval_GetFrame(void);
 
-PyObject *_Py_MakeCoro(PyFunctionObject *func, struct _interpreter_frame *);
+PyObject *_Py_MakeCoro(PyFunctionObject *func);
 
 #ifdef __cplusplus
 }
diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h
index b0e51a6ddc461..f4f7ab942c1ac 100644
--- a/Include/internal/pycore_frame.h
+++ b/Include/internal/pycore_frame.h
@@ -74,7 +74,7 @@ static inline void _PyFrame_StackPush(InterpreterFrame *f, PyObject *value) {
 
 #define FRAME_SPECIALS_SIZE ((sizeof(InterpreterFrame)-1)/sizeof(PyObject *))
 
-InterpreterFrame *_PyFrame_Copy(InterpreterFrame *frame);
+void _PyFrame_Copy(InterpreterFrame *src, InterpreterFrame *dest);
 
 static inline void
 _PyFrame_InitializeSpecials(
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index 2b1ba2457f50d..81802893a8ee5 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -1340,7 +1340,7 @@ def bar(cls):
             check(bar, size('PP'))
         # generator
         def get_gen(): yield 1
-        check(get_gen(), size('P2PPP4P'))
+        check(get_gen(), size('P2PPP4P4c8P2iciP'))
         # iterator
         check(iter('abc'), size('lP'))
         # callable-iterator
diff --git a/Objects/genobject.c b/Objects/genobject.c
index 04d98a2b4edf1..c4ba660530cae 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -36,8 +36,8 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg)
     Py_VISIT(gen->gi_code);
     Py_VISIT(gen->gi_name);
     Py_VISIT(gen->gi_qualname);
-    InterpreterFrame *frame = gen->gi_xframe;
-    if (frame != NULL) {
+    if (gen->gi_frame_valid) {
+        InterpreterFrame *frame = (InterpreterFrame *)(gen->gi_iframe);
         assert(frame->frame_obj == NULL || frame->frame_obj->f_owns_frame == 0);
         int err = _PyFrame_Traverse(frame, visit, arg);
         if (err) {
@@ -56,14 +56,14 @@ _PyGen_Finalize(PyObject *self)
     PyObject *res = NULL;
     PyObject *error_type, *error_value, *error_traceback;
 
-    if (gen->gi_xframe == NULL ||  _PyFrameHasCompleted(gen->gi_xframe)) {
+    if (gen->gi_frame_valid == 0 || _PyFrameHasCompleted((InterpreterFrame *)gen->gi_iframe)) {
         /* Generator isn't paused, so no need to close */
         return;
     }
 
     if (PyAsyncGen_CheckExact(self)) {
         PyAsyncGenObject *agen = (PyAsyncGenObject*)self;
-        PyObject *finalizer = agen->ag_finalizer;
+        PyObject *finalizer = agen->ag_origin_or_finalizer;
         if (finalizer && !agen->ag_closed) {
             /* Save the current exception, if any. */
             PyErr_Fetch(&error_type, &error_value, &error_traceback);
@@ -88,7 +88,7 @@ _PyGen_Finalize(PyObject *self)
        issue a RuntimeWarning. */
     if (gen->gi_code != NULL &&
         ((PyCodeObject *)gen->gi_code)->co_flags & CO_COROUTINE &&
-        gen->gi_xframe->f_lasti == -1)
+        ((InterpreterFrame *)gen->gi_iframe)->f_lasti == -1)
     {
         _PyErr_WarnUnawaitedCoroutine((PyObject *)gen);
     }
@@ -129,18 +129,17 @@ gen_dealloc(PyGenObject *gen)
         /* We have to handle this case for asynchronous generators
            right here, because this code has to be between UNTRACK
            and GC_Del. */
-        Py_CLEAR(((PyAsyncGenObject*)gen)->ag_finalizer);
+        Py_CLEAR(((PyAsyncGenObject*)gen)->ag_origin_or_finalizer);
     }
-    InterpreterFrame *frame = gen->gi_xframe;
-    if (frame != NULL) {
-        gen->gi_xframe = NULL;
+    if (gen->gi_frame_valid) {
+        InterpreterFrame *frame = (InterpreterFrame *)gen->gi_iframe;
+        gen->gi_frame_valid = 0;
         frame->generator = NULL;
         frame->previous = NULL;
         _PyFrame_Clear(frame);
-        PyMem_Free(frame);
     }
     if (((PyCodeObject *)gen->gi_code)->co_flags & CO_COROUTINE) {
-        Py_CLEAR(((PyCoroObject *)gen)->cr_origin);
+        Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer);
     }
     Py_CLEAR(gen->gi_code);
     Py_CLEAR(gen->gi_name);
@@ -154,11 +153,11 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
              int exc, int closing)
 {
     PyThreadState *tstate = _PyThreadState_GET();
-    InterpreterFrame *frame = gen->gi_xframe;
+    InterpreterFrame *frame = (InterpreterFrame *)gen->gi_iframe;
     PyObject *result;
 
     *presult = NULL;
-    if (frame != NULL && _PyFrame_IsExecuting(frame)) {
+    if (gen->gi_frame_valid && _PyFrame_IsExecuting(frame)) {
         const char *msg = "generator already executing";
         if (PyCoro_CheckExact(gen)) {
             msg = "coroutine already executing";
@@ -169,7 +168,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
         PyErr_SetString(PyExc_ValueError, msg);
         return PYGEN_ERROR;
     }
-    if (frame == NULL || _PyFrameHasCompleted(frame)) {
+    if (gen->gi_frame_valid == 0 || _PyFrameHasCompleted(frame)) {
         if (PyCoro_CheckExact(gen) && !closing) {
             /* `gen` is an exhausted coroutine: raise an error,
                except when called from gen_close(), which should
@@ -188,6 +187,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
         return PYGEN_ERROR;
     }
 
+    assert(gen->gi_frame_valid);
     assert(_PyFrame_IsRunnable(frame));
     /* Push arg onto the frame's value stack */
     result = arg ? arg : Py_None;
@@ -254,9 +254,8 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     _PyErr_ClearExcState(&gen->gi_exc_state);
 
     frame->generator = NULL;
-    gen->gi_xframe = NULL;
+    gen->gi_frame_valid = 0;
     _PyFrame_Clear(frame);
-    PyMem_Free(frame);
     *presult = result;
     return result ? PYGEN_RETURN : PYGEN_ERROR;
 }
@@ -337,8 +336,8 @@ _PyGen_yf(PyGenObject *gen)
 {
     PyObject *yf = NULL;
 
-    if (gen->gi_xframe) {
-        InterpreterFrame *frame = gen->gi_xframe;
+    if (gen->gi_frame_valid) {
+        InterpreterFrame *frame = (InterpreterFrame *)gen->gi_iframe;
         PyObject *bytecode = gen->gi_code->co_code;
         unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
 
@@ -367,10 +366,11 @@ gen_close(PyGenObject *gen, PyObject *args)
     int err = 0;
 
     if (yf) {
-        PyFrameState state = gen->gi_xframe->f_state;
-        gen->gi_xframe->f_state = FRAME_EXECUTING;
+        InterpreterFrame *frame = (InterpreterFrame *)gen->gi_iframe;
+        PyFrameState state = frame->f_state;
+        frame->f_state = FRAME_EXECUTING;
         err = gen_close_iter(yf);
-        gen->gi_xframe->f_state = state;
+        frame->f_state = state;
         Py_DECREF(yf);
     }
     if (err == 0)
@@ -408,6 +408,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
     _Py_IDENTIFIER(throw);
 
     if (yf) {
+        InterpreterFrame *frame = (InterpreterFrame *)gen->gi_iframe;
         PyObject *ret;
         int err;
         if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) &&
@@ -417,10 +418,10 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
                We have to allow some awaits to work it through, hence the
                `close_on_genexit` parameter here.
             */
-            PyFrameState state = gen->gi_xframe->f_state;
-            gen->gi_xframe->f_state = FRAME_EXECUTING;
+            PyFrameState state = frame->f_state;
+            frame->f_state = FRAME_EXECUTING;
             err = gen_close_iter(yf);
-            gen->gi_xframe->f_state = state;
+            frame->f_state = state;
             Py_DECREF(yf);
             if (err < 0)
                 return gen_send_ex(gen, Py_None, 1, 0);
@@ -429,9 +430,6 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
         if (PyGen_CheckExact(yf) || PyCoro_CheckExact(yf)) {
             /* `yf` is a generator or a coroutine. */
             PyThreadState *tstate = _PyThreadState_GET();
-            InterpreterFrame *frame = gen->gi_xframe;
-
-
             /* Since we are fast-tracking things by skipping the eval loop,
                we need to update the current frame so the stack trace
                will be reported correctly to the user. */
@@ -442,11 +440,11 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
             tstate->cframe->current_frame = frame;
             /* Close the generator that we are currently iterating with
                'yield from' or awaiting on with 'await'. */
-            PyFrameState state = gen->gi_xframe->f_state;
-            gen->gi_xframe->f_state = FRAME_EXECUTING;
+            PyFrameState state = frame->f_state;
+            frame->f_state = FRAME_EXECUTING;
             ret = _gen_throw((PyGenObject *)yf, close_on_genexit,
                              typ, val, tb);
-            gen->gi_xframe->f_state = state;
+            frame->f_state = state;
             tstate->cframe->current_frame = prev;
             frame->previous = NULL;
         } else {
@@ -460,22 +458,23 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
                 Py_DECREF(yf);
                 goto throw_here;
             }
-            PyFrameState state = gen->gi_xframe->f_state;
-            gen->gi_xframe->f_state = FRAME_EXECUTING;
+            PyFrameState state = frame->f_state;
+            frame->f_state = FRAME_EXECUTING;
             ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL);
-            gen->gi_xframe->f_state = state;
+            frame->f_state = state;
             Py_DECREF(meth);
         }
         Py_DECREF(yf);
         if (!ret) {
             PyObject *val;
             /* Pop subiterator from stack */
-            ret = _PyFrame_StackPop(gen->gi_xframe);
+            assert(gen->gi_frame_valid);
+            ret = _PyFrame_StackPop((InterpreterFrame *)gen->gi_iframe);
             assert(ret == yf);
             Py_DECREF(ret);
             /* Termination repetition of YIELD_FROM */
-            assert(gen->gi_xframe->f_lasti >= 0);
-            gen->gi_xframe->f_lasti += 1;
+            assert(frame->f_lasti >= 0);
+            frame->f_lasti += 1;
             if (_PyGen_FetchStopIterationValue(&val) == 0) {
                 ret = gen_send(gen, val);
                 Py_DECREF(val);
@@ -732,10 +731,10 @@ gen_getyieldfrom(PyGenObject *gen, void *Py_UNUSED(ignored))
 static PyObject *
 gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored))
 {
-    if (gen->gi_xframe == NULL) {
+    if (gen->gi_frame_valid == 0) {
         Py_RETURN_FALSE;
     }
-    return PyBool_FromLong(_PyFrame_IsExecuting(gen->gi_xframe));
+    return PyBool_FromLong(_PyFrame_IsExecuting((InterpreterFrame *)gen->gi_iframe));
 }
 
 static PyObject *
@@ -744,10 +743,10 @@ _gen_getframe(PyGenObject *gen, const char *const name)
     if (PySys_Audit("object.__getattr__", "Os", gen, name) < 0) {
         return NULL;
     }
-    if (gen->gi_xframe == NULL) {
+    if (gen->gi_frame_valid == 0) {
         Py_RETURN_NONE;
     }
-    return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject(gen->gi_xframe));
+    return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject((InterpreterFrame *)gen->gi_iframe));
 }
 
 static PyObject *
@@ -773,10 +772,24 @@ static PyMemberDef gen_memberlist[] = {
     {NULL}      /* Sentinel */
 };
 
+static PyObject *
+gen_sizeof(PyGenObject *gen, PyObject *Py_UNUSED(ignored))
+{
+    Py_ssize_t res;
+    res = offsetof(PyGenObject, gi_iframe) + offsetof(InterpreterFrame, localsplus);
+    PyCodeObject *code = gen->gi_code;
+    res += (code->co_nlocalsplus+code->co_stacksize) * sizeof(PyObject *);
+    return PyLong_FromSsize_t(res);
+}
+
+PyDoc_STRVAR(sizeof__doc__,
+"gen.__sizeof__() -> size of gen in memory, in bytes");
+
 static PyMethodDef gen_methods[] = {
     {"send",(PyCFunction)gen_send, METH_O, send_doc},
     {"throw",(PyCFunction)gen_throw, METH_VARARGS, throw_doc},
     {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc},
+    {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__},
     {NULL, NULL}        /* Sentinel */
 };
 
@@ -791,8 +804,9 @@ static PyAsyncMethods gen_as_async = {
 PyTypeObject PyGen_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "generator",                                /* tp_name */
-    sizeof(PyGenObject),                        /* tp_basicsize */
-    0,                                          /* tp_itemsize */
+    offsetof(PyGenObject, gi_iframe) +
+    offsetof(InterpreterFrame, localsplus),       /* tp_basicsize */
+    sizeof(PyObject *),                         /* tp_itemsize */
     /* methods */
     (destructor)gen_dealloc,                    /* tp_dealloc */
     0,                                          /* tp_vectorcall_offset */
@@ -842,18 +856,16 @@ PyTypeObject PyGen_Type = {
 };
 
 static PyObject *
-make_gen(PyTypeObject *type, PyFunctionObject *func, InterpreterFrame *frame)
+make_gen(PyTypeObject *type, PyFunctionObject *func)
 {
-    PyGenObject *gen = PyObject_GC_New(PyGenObject, type);
+    PyCodeObject *code = (PyCodeObject *)func->func_code;
+    int slots = code->co_nlocalsplus + code->co_stacksize;
+    PyGenObject *gen = PyObject_GC_NewVar(PyGenObject, type, slots);
     if (gen == NULL) {
-        assert(frame->frame_obj == NULL);
-        _PyFrame_Clear(frame);
-        PyMem_Free(frame);
         return NULL;
     }
-    gen->gi_xframe = frame;
-    frame->generator = (PyObject *)gen;
-    gen->gi_code = frame->f_code;
+    gen->gi_frame_valid = 0;
+    gen->gi_code = (PyCodeObject *)func->func_code;
     Py_INCREF(gen->gi_code);
     gen->gi_weakreflist = NULL;
     gen->gi_exc_state.exc_type = NULL;
@@ -878,28 +890,28 @@ static PyObject *
 compute_cr_origin(int origin_depth);
 
 PyObject *
-_Py_MakeCoro(PyFunctionObject *func, InterpreterFrame *frame)
+_Py_MakeCoro(PyFunctionObject *func)
 {
     int coro_flags = ((PyCodeObject *)func->func_code)->co_flags &
         (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR);
     assert(coro_flags);
     if (coro_flags == CO_GENERATOR) {
-        return make_gen(&PyGen_Type, func, frame);
+        return make_gen(&PyGen_Type, func);
     }
     if (coro_flags == CO_ASYNC_GENERATOR) {
         PyAsyncGenObject *o;
-        o = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func, frame);
+        o = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func);
         if (o == NULL) {
             return NULL;
         }
-        o->ag_finalizer = NULL;
+        o->ag_origin_or_finalizer = NULL;
         o->ag_closed = 0;
         o->ag_hooks_inited = 0;
         o->ag_running_async = 0;
         return (PyObject*)o;
     }
     assert (coro_flags == CO_COROUTINE);
-    PyObject *coro = make_gen(&PyCoro_Type, func, frame);
+    PyObject *coro = make_gen(&PyCoro_Type, func);
     if (!coro) {
         return NULL;
     }
@@ -907,16 +919,15 @@ _Py_MakeCoro(PyFunctionObject *func, InterpreterFrame *frame)
     int origin_depth = tstate->coroutine_origin_tracking_depth;
 
     if (origin_depth == 0) {
-        ((PyCoroObject *)coro)->cr_origin = NULL;
+        ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL;
     } else {
         PyObject *cr_origin = compute_cr_origin(origin_depth);
-        ((PyCoroObject *)coro)->cr_origin = cr_origin;
+        ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin;
         if (!cr_origin) {
             Py_DECREF(coro);
             return NULL;
         }
     }
-
     return coro;
 }
 
@@ -924,27 +935,27 @@ static PyObject *
 gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f,
                       PyObject *name, PyObject *qualname)
 {
-    PyGenObject *gen = PyObject_GC_New(PyGenObject, type);
+    PyCodeObject *code = f->f_frame->f_code;
+    int size = code->co_nlocalsplus + code->co_stacksize;
+    PyGenObject *gen = PyObject_GC_NewVar(PyGenObject, type, size);
     if (gen == NULL) {
         Py_DECREF(f);
         return NULL;
     }
-
-    /* Take ownership of the frame */
+    /* Copy the frame */
     assert(f->f_frame->frame_obj == NULL);
     assert(f->f_owns_frame);
-    gen->gi_xframe = _PyFrame_Copy((InterpreterFrame *)f->_f_frame_data);
-    if (gen->gi_xframe == NULL) {
-        Py_DECREF(f);
-        Py_DECREF(gen);
-        return NULL;
-    }
-    gen->gi_xframe->frame_obj = f;
+    InterpreterFrame *frame = (InterpreterFrame *)gen->gi_iframe;
+    _PyFrame_Copy((InterpreterFrame *)f->_f_frame_data, frame);
+    gen->gi_frame_valid = 1;
+    assert(frame->frame_obj == f);
     f->f_owns_frame = 0;
-    gen->gi_xframe->generator = (PyObject *) gen;
+    f->f_frame = frame;
+    frame->generator = (PyObject *) gen;
     assert(PyObject_GC_IsTracked((PyObject *)f));
-
     gen->gi_code = PyFrame_GetCode(f);
+    Py_INCREF(gen->gi_code);
+    Py_DECREF(f);
     gen->gi_weakreflist = NULL;
     gen->gi_exc_state.exc_type = NULL;
     gen->gi_exc_state.exc_value = NULL;
@@ -1077,10 +1088,10 @@ coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored))
 static PyObject *
 cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored))
 {
-    if (coro->cr_xframe == NULL) {
+    if (coro->cr_frame_valid == 0) {
         Py_RETURN_FALSE;
     }
-    return PyBool_FromLong(_PyFrame_IsExecuting(coro->cr_xframe));
+    return PyBool_FromLong(_PyFrame_IsExecuting((InterpreterFrame *)coro->cr_iframe));
 }
 
 static PyObject *
@@ -1104,7 +1115,7 @@ static PyGetSetDef coro_getsetlist[] = {
 
 static PyMemberDef coro_memberlist[] = {
     {"cr_code",      T_OBJECT, offsetof(PyCoroObject, cr_code),     READONLY|PY_AUDIT_READ},
-    {"cr_origin",    T_OBJECT, offsetof(PyCoroObject, cr_origin),   READONLY},
+    {"cr_origin",    T_OBJECT, offsetof(PyCoroObject, cr_origin_or_finalizer),   READONLY},
     {NULL}      /* Sentinel */
 };
 
@@ -1123,6 +1134,7 @@ static PyMethodDef coro_methods[] = {
     {"send",(PyCFunction)gen_send, METH_O, coro_send_doc},
     {"throw",(PyCFunction)gen_throw, METH_VARARGS, coro_throw_doc},
     {"close",(PyCFunction)gen_close, METH_NOARGS, coro_close_doc},
+    {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__},
     {NULL, NULL}        /* Sentinel */
 };
 
@@ -1136,8 +1148,9 @@ static PyAsyncMethods coro_as_async = {
 PyTypeObject PyCoro_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "coroutine",                                /* tp_name */
-    sizeof(PyCoroObject),                       /* tp_basicsize */
-    0,                                          /* tp_itemsize */
+    offsetof(PyCoroObject, cr_iframe) +
+    offsetof(InterpreterFrame, localsplus),       /* tp_basicsize */
+    sizeof(PyObject *),                         /* tp_itemsize */
     /* methods */
     (destructor)gen_dealloc,                    /* tp_dealloc */
     0,                                          /* tp_vectorcall_offset */
@@ -1318,10 +1331,10 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
     int origin_depth = tstate->coroutine_origin_tracking_depth;
 
     if (origin_depth == 0) {
-        ((PyCoroObject *)coro)->cr_origin = NULL;
+        ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL;
     } else {
         PyObject *cr_origin = compute_cr_origin(origin_depth);
-        ((PyCoroObject *)coro)->cr_origin = cr_origin;
+        ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin;
         if (!cr_origin) {
             Py_DECREF(coro);
             return NULL;
@@ -1382,7 +1395,7 @@ typedef struct _PyAsyncGenWrappedValue {
 static int
 async_gen_traverse(PyAsyncGenObject *gen, visitproc visit, void *arg)
 {
-    Py_VISIT(gen->ag_finalizer);
+    Py_VISIT(gen->ag_origin_or_finalizer);
     return gen_traverse((PyGenObject*)gen, visit, arg);
 }
 
@@ -1413,7 +1426,7 @@ async_gen_init_hooks(PyAsyncGenObject *o)
     finalizer = tstate->async_gen_finalizer;
     if (finalizer) {
         Py_INCREF(finalizer);
-        o->ag_finalizer = finalizer;
+        o->ag_origin_or_finalizer = finalizer;
     }
 
     firstiter = tstate->async_gen_firstiter;
@@ -1508,6 +1521,7 @@ static PyMethodDef async_gen_methods[] = {
     {"asend", (PyCFunction)async_gen_asend, METH_O, async_asend_doc},
     {"athrow",(PyCFunction)async_gen_athrow, METH_VARARGS, async_athrow_doc},
     {"aclose", (PyCFunction)async_gen_aclose, METH_NOARGS, async_aclose_doc},
+    {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__},
     {"__class_getitem__",    Py_GenericAlias,
     METH_O|METH_CLASS,       PyDoc_STR("See PEP 585")},
     {NULL, NULL}        /* Sentinel */
@@ -1525,8 +1539,9 @@ static PyAsyncMethods async_gen_as_async = {
 PyTypeObject PyAsyncGen_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "async_generator",                          /* tp_name */
-    sizeof(PyAsyncGenObject),                   /* tp_basicsize */
-    0,                                          /* tp_itemsize */
+    offsetof(PyAsyncGenObject, ag_iframe) +
+    offsetof(InterpreterFrame, localsplus),       /* tp_basicsize */
+    sizeof(PyObject *),                         /* tp_itemsize */
     /* methods */
     (destructor)gen_dealloc,                    /* tp_dealloc */
     0,                                          /* tp_vectorcall_offset */
@@ -1594,7 +1609,7 @@ PyAsyncGen_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
     if (o == NULL) {
         return NULL;
     }
-    o->ag_finalizer = NULL;
+    o->ag_origin_or_finalizer = NULL;
     o->ag_closed = 0;
     o->ag_hooks_inited = 0;
     o->ag_running_async = 0;
@@ -2011,7 +2026,7 @@ static PyObject *
 async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
 {
     PyGenObject *gen = (PyGenObject*)o->agt_gen;
-    InterpreterFrame *frame = gen->gi_xframe;
+    InterpreterFrame *frame = (InterpreterFrame *)gen->gi_iframe;
     PyObject *retval;
 
     if (o->agt_state == AWAITABLE_STATE_CLOSED) {
@@ -2021,7 +2036,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg)
         return NULL;
     }
 
-    if (frame == NULL || _PyFrameHasCompleted(frame)) {
+    if (gen->gi_frame_valid == 0 || _PyFrameHasCompleted(frame)) {
         o->agt_state = AWAITABLE_STATE_CLOSED;
         PyErr_SetNone(PyExc_StopIteration);
         return NULL;
diff --git a/Python/ceval.c b/Python/ceval.c
index 05897c561a16e..a8bbad33552e4 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -5854,8 +5854,8 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func,
     return -1;
 }
 
-static InterpreterFrame *
-make_coro_frame(PyThreadState *tstate,
+static int
+initialize_coro_frame(InterpreterFrame *frame, PyThreadState *tstate,
            PyFunctionObject *func, PyObject *locals,
            PyObject *const *args, Py_ssize_t argcount,
            PyObject *kwnames)
@@ -5863,37 +5863,15 @@ make_coro_frame(PyThreadState *tstate,
     assert(is_tstate_valid(tstate));
     assert(func->func_defaults == NULL || PyTuple_CheckExact(func->func_defaults));
     PyCodeObject *code = (PyCodeObject *)func->func_code;
-    int size = code->co_nlocalsplus+code->co_stacksize + FRAME_SPECIALS_SIZE;
-    InterpreterFrame *frame = (InterpreterFrame *)PyMem_Malloc(sizeof(PyObject *)*size);
-    if (frame == NULL) {
-        goto fail_no_memory;
-    }
     _PyFrame_InitializeSpecials(frame, func, locals, code->co_nlocalsplus);
     for (int i = 0; i < code->co_nlocalsplus; i++) {
         frame->localsplus[i] = NULL;
     }
     assert(frame->frame_obj == NULL);
-    if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) {
-        _PyFrame_Clear(frame);
-        PyMem_Free(frame);
-        return NULL;
-    }
-    return frame;
-fail_no_memory:
-    /* Consume the references */
-    for (Py_ssize_t i = 0; i < argcount; i++) {
-        Py_DECREF(args[i]);
-    }
-    if (kwnames) {
-        Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames);
-        for (Py_ssize_t i = 0; i < kwcount; i++) {
-            Py_DECREF(args[i+argcount]);
-        }
-    }
-    PyErr_NoMemory();
-    return NULL;
+    return initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames);
 }
 
+
 /* Consumes all the references to the args */
 static PyObject *
 make_coro(PyThreadState *tstate, PyFunctionObject *func,
@@ -5902,14 +5880,17 @@ make_coro(PyThreadState *tstate, PyFunctionObject *func,
           PyObject *kwnames)
 {
     assert (((PyCodeObject *)func->func_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR));
-    InterpreterFrame *frame = make_coro_frame(tstate, func, locals, args, argcount, kwnames);
-    if (frame == NULL) {
+    PyObject *gen = _Py_MakeCoro(func);
+    if (gen == NULL) {
         return NULL;
     }
-    PyObject *gen = _Py_MakeCoro(func, frame);
-    if (gen == NULL) {
+    InterpreterFrame *frame = (InterpreterFrame *)((PyGenObject *)gen)->gi_iframe;
+    if (initialize_coro_frame(frame, tstate, func, locals, args, argcount, kwnames)) {
+        Py_DECREF(gen);
         return NULL;
     }
+    frame->generator = gen;
+    ((PyGenObject *)gen)->gi_frame_valid = 1;
     return gen;
 }
 
diff --git a/Python/frame.c b/Python/frame.c
index 21f2ced959d80..da2c1c4aec075 100644
--- a/Python/frame.c
+++ b/Python/frame.c
@@ -43,18 +43,12 @@ _PyFrame_MakeAndSetFrameObject(InterpreterFrame *frame)
     return f;
 }
 
-InterpreterFrame *
-_PyFrame_Copy(InterpreterFrame *frame)
+void
+_PyFrame_Copy(InterpreterFrame *src, InterpreterFrame *dest)
 {
-    assert(frame->stacktop >= frame->f_code->co_nlocalsplus);
-    Py_ssize_t size = ((char*)&frame->localsplus[frame->stacktop]) - (char *)frame;
-    InterpreterFrame *copy = PyMem_Malloc(size);
-    if (copy == NULL) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-    memcpy(copy, frame, size);
-    return copy;
+    assert(src->stacktop >= src->f_code->co_nlocalsplus);
+    Py_ssize_t size = ((char*)&src->localsplus[src->stacktop]) - (char *)src;
+    memcpy(dest, src, size);
 }
 
 static inline void
@@ -112,7 +106,7 @@ _PyFrame_Clear(InterpreterFrame * frame)
         }
         Py_DECREF(f);
     }
-    assert(_PyFrame_GetStackPointer(frame) >= _PyFrame_Stackbase(frame));
+    assert(frame->stacktop >= 0);
     for (int i = 0; i < frame->stacktop; i++) {
         Py_XDECREF(frame->localsplus[i]);
     }


From webhook-mailer at python.org  Mon Dec  6 07:19:31 2021
From: webhook-mailer at python.org (tiran)
Date: Mon, 06 Dec 2021 12:19:31 -0000
Subject: [Python-checkins] bpo-44035: Check autoconf files thoroughly
 (GH-29935)
Message-ID: <mailman.56.1638793172.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/98fac8bc183c113903403793ffe411358ee40d8a
commit: 98fac8bc183c113903403793ffe411358ee40d8a
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-06T13:18:56+01:00
summary:

bpo-44035: Check autoconf files thoroughly (GH-29935)

Check that users don't push changes with outdated or patched autoconf.
The presence of runstatedir option and aclocal 1.16.3 are good markers.

Use my container image to regenerate autoconf files. "Check for changes"
will fail later when any file is regenerated.

Use ccache in check_generated_files to speed up testing.

files:
A Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
M .github/workflows/build.yml
M .gitignore

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fbf7bb6fe2128..99f05f3abec93 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -64,6 +64,18 @@ jobs:
       - uses: actions/setup-python at v2
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
+      - name: Add ccache to PATH
+        run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+      - name: Configure ccache action
+        uses: hendrikmuhs/ccache-action at v1
+      - name: Check Autoconf version 2.69 and aclocal 1.16.3
+        run: |
+          grep "Generated by GNU Autoconf 2.69" configure
+          grep "aclocal 1.16.3" aclocal.m4
+          grep -q "runstatedir" configure
+          grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
+      - name: Regenerate autoconf files
+        run: docker run --rm -v $(pwd):/src quay.io/tiran/cpython_autoconf:269
       - name: Build CPython
         run: |
           # Build Python with the libpython dynamic library
@@ -75,9 +87,10 @@ jobs:
           git add -u
           changes=$(git status --porcelain)
           # Check for changes in regenerated files
-          if ! test -z "$changes"
-          then
-            echo "Generated files not up to date. Perhaps you forgot to run make regen-all or build.bat --regen ;)"
+          if test -n "$changes"; then
+            echo "Generated files not up to date."
+            echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)"
+            echo "configure files must be regenerated with a specific, unpatched version of autoconf."
             echo "$changes"
             exit 1
           fi
@@ -85,10 +98,6 @@ jobs:
         run: make smelly
       - name: Check limited ABI symbols
         run: make check-limited-abi
-      - name: Check Autoconf version 2.69
-        run: |
-          grep "Generated by GNU Autoconf 2.69" configure
-          grep "PKG_PROG_PKG_CONFIG" aclocal.m4
 
   build_win32:
     name: 'Windows (x86)'
diff --git a/.gitignore b/.gitignore
index cfd3163cd1818..39de8410200ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -115,6 +115,8 @@ Tools/unicode/data/
 /config.log
 /config.status
 /config.status.lineno
+# hendrikmuhs/ccache-action at v1
+/.ccache
 /platform
 /profile-clean-stamp
 /profile-run-stamp
diff --git a/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst b/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
new file mode 100644
index 0000000000000..7530587b73d14
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
@@ -0,0 +1,2 @@
+CI now verifies that autoconf files have been regenerated with a current and
+unpatched autoconf package.


From webhook-mailer at python.org  Mon Dec  6 07:47:51 2021
From: webhook-mailer at python.org (tiran)
Date: Mon, 06 Dec 2021 12:47:51 -0000
Subject: [Python-checkins] [3.10] bpo-44035: Check autoconf files thoroughly
 (GH-29935) (GH-29937)
Message-ID: <mailman.57.1638794872.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/1528d249a5f8492801d09755f402618688337006
commit: 1528d249a5f8492801d09755f402618688337006
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: tiran <christian at python.org>
date: 2021-12-06T13:47:42+01:00
summary:

[3.10] bpo-44035: Check autoconf files thoroughly (GH-29935) (GH-29937)

Co-authored-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
M .github/workflows/build.yml
M .gitignore

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 01b75aedafed5..c91fcc6d05fdb 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -85,6 +85,18 @@ jobs:
       - uses: actions/setup-python at v2
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
+      - name: Add ccache to PATH
+        run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+      - name: Configure ccache action
+        uses: hendrikmuhs/ccache-action at v1
+      - name: Check Autoconf version 2.69 and aclocal 1.16.3
+        run: |
+          grep "Generated by GNU Autoconf 2.69" configure
+          grep "aclocal 1.16.3" aclocal.m4
+          grep -q "runstatedir" configure
+          grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
+      - name: Regenerate autoconf files
+        run: docker run --rm -v $(pwd):/src quay.io/tiran/cpython_autoconf:269
       - name: Build CPython
         run: |
           # Build Python with the libpython dynamic library
@@ -95,9 +107,10 @@ jobs:
         run: |
           changes=$(git status --porcelain)
           # Check for changes in regenerated files
-          if ! test -z "$changes"
-          then
-            echo "Generated files not up to date. Perhaps you forgot to run make regen-all or build.bat --regen ;)"
+          if test -n "$changes"; then
+            echo "Generated files not up to date."
+            echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)"
+            echo "configure files must be regenerated with a specific, unpatched version of autoconf."
             echo "$changes"
             exit 1
           fi
@@ -105,10 +118,6 @@ jobs:
         run: make smelly
       - name: Check limited ABI symbols
         run: make check-limited-abi
-      - name: Check Autoconf version 2.69
-        run: |
-          grep "Generated by GNU Autoconf 2.69" configure
-          grep "PKG_PROG_PKG_CONFIG" aclocal.m4
 
   build_win32:
     name: 'Windows (x86)'
diff --git a/.gitignore b/.gitignore
index 19b4214a9aea0..09d08c8050cb2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -112,6 +112,8 @@ Tools/unicode/data/
 /config.log
 /config.status
 /config.status.lineno
+# hendrikmuhs/ccache-action at v1
+/.ccache
 /platform
 /profile-clean-stamp
 /profile-run-stamp
diff --git a/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst b/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
new file mode 100644
index 0000000000000..7530587b73d14
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
@@ -0,0 +1,2 @@
+CI now verifies that autoconf files have been regenerated with a current and
+unpatched autoconf package.


From webhook-mailer at python.org  Mon Dec  6 07:48:58 2021
From: webhook-mailer at python.org (tiran)
Date: Mon, 06 Dec 2021 12:48:58 -0000
Subject: [Python-checkins] [3.9] bpo-44035: Check autoconf files thoroughly
 (GH-29935) (GH-29938)
Message-ID: <mailman.58.1638794939.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/f147248795bf22be05bbc891053d60ef6a2f035d
commit: f147248795bf22be05bbc891053d60ef6a2f035d
branch: 3.9
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-06T13:48:54+01:00
summary:

[3.9] bpo-44035: Check autoconf files thoroughly (GH-29935) (GH-29938)

Co-authored-by: Christian Heimes <christian at python.org>

files:
A Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
M .github/workflows/build.yml
M .gitignore
M aclocal.m4
M configure
M pyconfig.h.in

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index bb56a42a5e340..295b24c2cf136 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -83,6 +83,18 @@ jobs:
       - uses: actions/setup-python at v2
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
+      - name: Add ccache to PATH
+        run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+      - name: Configure ccache action
+        uses: hendrikmuhs/ccache-action at v1
+      - name: Check Autoconf version 2.69 and aclocal 1.16.3
+        run: |
+          grep "Generated by GNU Autoconf 2.69" configure
+          grep "aclocal 1.16.3" aclocal.m4
+          grep -q "runstatedir" configure
+          grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
+      - name: Regenerate autoconf files
+        run: docker run --rm -v $(pwd):/src quay.io/tiran/cpython_autoconf:269
       - name: Build CPython
         run: |
           ./configure --with-pydebug
@@ -91,16 +103,15 @@ jobs:
         run: |
           changes=$(git status --porcelain)
           # Check for changes in regenerated files
-          if ! test -z "$changes"
-          then
-            echo "Generated files not up to date. Perhaps you forgot to run make regen-all or build.bat --regen ;)"
+          if test -n "$changes"; then
+            echo "Generated files not up to date."
+            echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)"
+            echo "configure files must be regenerated with a specific, unpatched version of autoconf."
             echo "$changes"
             exit 1
           fi
       - name: Check exported libpython symbols
         run: make smelly
-      - name: Check Autoconf version 2.69
-        run: grep "Generated by GNU Autoconf 2.69" configure
 
   build_win32:
     name: 'Windows (x86)'
diff --git a/.gitignore b/.gitignore
index 0dd3aee5c3151..1ba44ec5d635b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -112,6 +112,8 @@ Tools/unicode/data/
 /config.log
 /config.status
 /config.status.lineno
+# hendrikmuhs/ccache-action at v1
+/.ccache
 /platform
 /profile-clean-stamp
 /profile-run-stamp
diff --git a/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst b/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
new file mode 100644
index 0000000000000..7530587b73d14
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
@@ -0,0 +1,2 @@
+CI now verifies that autoconf files have been regenerated with a current and
+unpatched autoconf package.
diff --git a/aclocal.m4 b/aclocal.m4
index 04342a4982e2d..e5e804276f501 100644
--- a/aclocal.m4
+++ b/aclocal.m4
@@ -1,4 +1,4 @@
-# generated automatically by aclocal 1.16.2 -*- Autoconf -*-
+# generated automatically by aclocal 1.16.3 -*- Autoconf -*-
 
 # Copyright (C) 1996-2020 Free Software Foundation, Inc.
 
diff --git a/configure b/configure
index 33ecb16f71452..1b1caf84de7d5 100755
--- a/configure
+++ b/configure
@@ -788,6 +788,7 @@ infodir
 docdir
 oldincludedir
 includedir
+runstatedir
 localstatedir
 sharedstatedir
 sysconfdir
@@ -905,6 +906,7 @@ datadir='${datarootdir}'
 sysconfdir='${prefix}/etc'
 sharedstatedir='${prefix}/com'
 localstatedir='${prefix}/var'
+runstatedir='${localstatedir}/run'
 includedir='${prefix}/include'
 oldincludedir='/usr/include'
 docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@@ -1157,6 +1159,15 @@ do
   | -silent | --silent | --silen | --sile | --sil)
     silent=yes ;;
 
+  -runstatedir | --runstatedir | --runstatedi | --runstated \
+  | --runstate | --runstat | --runsta | --runst | --runs \
+  | --run | --ru | --r)
+    ac_prev=runstatedir ;;
+  -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
+  | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
+  | --run=* | --ru=* | --r=*)
+    runstatedir=$ac_optarg ;;
+
   -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
     ac_prev=sbindir ;;
   -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@@ -1294,7 +1305,7 @@ fi
 for ac_var in	exec_prefix prefix bindir sbindir libexecdir datarootdir \
 		datadir sysconfdir sharedstatedir localstatedir includedir \
 		oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
-		libdir localedir mandir
+		libdir localedir mandir runstatedir
 do
   eval ac_val=\$$ac_var
   # Remove trailing slashes.
@@ -1447,6 +1458,7 @@ Fine tuning of the installation directories:
   --sysconfdir=DIR        read-only single-machine data [PREFIX/etc]
   --sharedstatedir=DIR    modifiable architecture-independent data [PREFIX/com]
   --localstatedir=DIR     modifiable single-machine data [PREFIX/var]
+  --runstatedir=DIR       modifiable per-process data [LOCALSTATEDIR/run]
   --libdir=DIR            object code libraries [EPREFIX/lib]
   --includedir=DIR        C header files [PREFIX/include]
   --oldincludedir=DIR     C header files for non-gcc [/usr/include]
@@ -9920,13 +9932,15 @@ $as_echo "no" >&6; }
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
 
+# check for libuuid from util-linux
 save_LIBS=$LIBS
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing uuid_generate_time_safe" >&5
-$as_echo_n "checking for library containing uuid_generate_time_safe... " >&6; }
-if ${ac_cv_search_uuid_generate_time_safe+:} false; then :
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for uuid_generate_time in -luuid" >&5
+$as_echo_n "checking for uuid_generate_time in -luuid... " >&6; }
+if ${ac_cv_lib_uuid_uuid_generate_time+:} false; then :
   $as_echo_n "(cached) " >&6
 else
-  ac_func_search_save_LIBS=$LIBS
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-luuid  $LIBS"
 cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
 
@@ -9936,61 +9950,39 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 #ifdef __cplusplus
 extern "C"
 #endif
-char uuid_generate_time_safe ();
+char uuid_generate_time ();
 int
 main ()
 {
-return uuid_generate_time_safe ();
+return uuid_generate_time ();
   ;
   return 0;
 }
 _ACEOF
-for ac_lib in '' uuid; do
-  if test -z "$ac_lib"; then
-    ac_res="none required"
-  else
-    ac_res=-l$ac_lib
-    LIBS="-l$ac_lib  $ac_func_search_save_LIBS"
-  fi
-  if ac_fn_c_try_link "$LINENO"; then :
-  ac_cv_search_uuid_generate_time_safe=$ac_res
-fi
-rm -f core conftest.err conftest.$ac_objext \
-    conftest$ac_exeext
-  if ${ac_cv_search_uuid_generate_time_safe+:} false; then :
-  break
-fi
-done
-if ${ac_cv_search_uuid_generate_time_safe+:} false; then :
-
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_uuid_uuid_generate_time=yes
 else
-  ac_cv_search_uuid_generate_time_safe=no
+  ac_cv_lib_uuid_uuid_generate_time=no
 fi
-rm conftest.$ac_ext
-LIBS=$ac_func_search_save_LIBS
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
 fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_uuid_generate_time_safe" >&5
-$as_echo "$ac_cv_search_uuid_generate_time_safe" >&6; }
-ac_res=$ac_cv_search_uuid_generate_time_safe
-if test "$ac_res" != no; then :
-  test "$ac_res" = "none required" || LIBS="$ac_res $LIBS"
-
-
-$as_echo "#define HAVE_LIBUUID 1" >>confdefs.h
-,
-    { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_uuid_uuid_generate_time" >&5
+$as_echo "$ac_cv_lib_uuid_uuid_generate_time" >&6; }
+if test "x$ac_cv_lib_uuid_uuid_generate_time" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBUUID 1
+_ACEOF
 
-else
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
+  LIBS="-luuid $LIBS"
 
 fi
 
 LIBS=$save_LIBS
 
 # AIX provides support for RFC4122 (uuid) in libc.a starting with AIX 6.1 (anno 2007)
-# FreeBSD and OpenBSD provides support as well
+# FreeBSD and OpenBSD provides support in libc as well.
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for uuid_create" >&5
 $as_echo_n "checking for uuid_create... " >&6; }
 cat confdefs.h - <<_ACEOF >conftest.$ac_ext
diff --git a/pyconfig.h.in b/pyconfig.h.in
index 63b013823cbf1..188faeeb12020 100644
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@@ -613,7 +613,7 @@
 /* Define to 1 if you have the <libutil.h> header file. */
 #undef HAVE_LIBUTIL_H
 
-/* Define you have libuuid. */
+/* Define to 1 if you have the `uuid' library (-luuid). */
 #undef HAVE_LIBUUID
 
 /* Define if you have the 'link' function. */


From webhook-mailer at python.org  Mon Dec  6 11:14:04 2021
From: webhook-mailer at python.org (tiran)
Date: Mon, 06 Dec 2021 16:14:04 -0000
Subject: [Python-checkins] bpo-45950: Fix macOS framework builds of
 _bootstrap_python (GH-29936)
Message-ID: <mailman.59.1638807244.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/612e59b53f0c730ce1b881f7c08dc6d49f02c123
commit: 612e59b53f0c730ce1b881f7c08dc6d49f02c123
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-06T17:13:53+01:00
summary:

bpo-45950: Fix macOS framework builds of _bootstrap_python (GH-29936)

files:
M Makefile.pre.in
M Modules/getpath.c
M configure
M configure.ac

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 8e6e553554de1..94fc5c37209ef 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -954,9 +954,9 @@ BOOTSTRAP_HEADERS = \
 
 Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS)
 
-_bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local
+_bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath_bootstrap.o Modules/Setup.local
 	$(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \
-		Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS)
+		Programs/_bootstrap_python.o Modules/getpath_bootstrap.o $(LIBS) $(MODLIBS) $(SYSLIBS)
 
 ############################################################################
 # Deepfreeze targets
@@ -1205,6 +1205,18 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h M
 		-DPLATLIBDIR='"$(PLATLIBDIR)"' \
 		-o $@ $(srcdir)/Modules/getpath.c
 
+# like getpath.o with additional -DPY_BOOTSTRAP_PYTHON=1
+Modules/getpath_bootstrap.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h Makefile $(PYTHON_HEADERS)
+	$(CC) -c $(PY_CORE_CFLAGS) -DPYTHONPATH='"$(PYTHONPATH)"' \
+		-DPREFIX='"$(prefix)"' \
+		-DEXEC_PREFIX='"$(exec_prefix)"' \
+		-DVERSION='"$(VERSION)"' \
+		-DVPATH='"$(VPATH)"' \
+		-DPLATLIBDIR='"$(PLATLIBDIR)"' \
+		-DPY_BOOTSTRAP_PYTHON=1 \
+		-o $@ $(srcdir)/Modules/getpath.c
+
+
 Programs/python.o: $(srcdir)/Programs/python.c
 	$(MAINCC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Programs/python.c
 
diff --git a/Modules/getpath.c b/Modules/getpath.c
index f77b18eee95b6..8f90a7008625f 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -754,7 +754,8 @@ library_to_dict(PyObject *dict, const char *key)
     if (PyWin_DLLhModule) {
         return winmodule_to_dict(dict, key, PyWin_DLLhModule);
     }
-#elif defined(WITH_NEXT_FRAMEWORK)
+#elif defined(WITH_NEXT_FRAMEWORK) && !defined(PY_BOOTSTRAP_PYTHON)
+    // _bootstrap_python does not use framework and crashes
     static const char modPath[MAXPATHLEN + 1];
     static int modPathInitialized = -1;
     if (modPathInitialized < 0) {
diff --git a/configure b/configure
index 8582224dfd28f..fca9567c7102f 100755
--- a/configure
+++ b/configure
@@ -1736,7 +1736,7 @@ Optional Packages:
                           path to _freeze_module binary for cross compiling
   --with-build-python=python3.11
                           path to build python binary for cross compiling
-                          (default: python3.11)
+                          (default: _bootstrap_python or python3.11)
   --with-pkg-config=[yes|no|check]
                           use pkg-config to detect build options (default is
                           check)
@@ -3243,14 +3243,11 @@ if test "${with_build_python+set}" = set; then :
     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-build-python" >&5
 $as_echo_n "checking for --with-build-python... " >&6; }
 
-    if test "x$cross_compiling" = xno; then :
-  as_fn_error $? "--with-build-python only applies to cross compiling" "$LINENO" 5
-fi
     if test "x$with_build_python" = xyes; then :
   with_build_python=python$PACKAGE_VERSION
 fi
     if test "x$with_build_python" = xno; then :
-  as_fn_error $? "invalid --with-build-python option: expected path, not \"no\"" "$LINENO" 5
+  as_fn_error $? "invalid --with-build-python option: expected path or \"yes\", not \"no\"" "$LINENO" 5
 fi
 
     if ! $(command -v "$with_build_python" >/dev/null 2>&1); then
diff --git a/configure.ac b/configure.ac
index 39890fe38f0a1..c7c71255a3a4c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -125,16 +125,16 @@ AC_ARG_WITH(
 )
 AC_SUBST([FREEZE_MODULE])
 
+dnl build-python is used for cross compiling and macOS framework builds.
 AC_ARG_WITH(
   [build-python],
   [AS_HELP_STRING([--with-build-python=python]PYTHON_VERSION,
-                  [path to build python binary for cross compiling (default: python]PYTHON_VERSION[)])],
+                  [path to build python binary for cross compiling (default: _bootstrap_python or python]PYTHON_VERSION[)])],
   [
     AC_MSG_CHECKING([for --with-build-python])
 
-    AS_VAR_IF([cross_compiling], [no], AC_MSG_ERROR([--with-build-python only applies to cross compiling]))
     AS_VAR_IF([with_build_python], [yes], [with_build_python=python$PACKAGE_VERSION])
-    AS_VAR_IF([with_build_python], [no], [AC_MSG_ERROR([invalid --with-build-python option: expected path, not "no"])])
+    AS_VAR_IF([with_build_python], [no], [AC_MSG_ERROR([invalid --with-build-python option: expected path or "yes", not "no"])])
 
     if ! $(command -v "$with_build_python" >/dev/null 2>&1); then
       AC_MSG_ERROR([invalid or missing build python binary "$with_build_python"])


From webhook-mailer at python.org  Mon Dec  6 12:25:34 2021
From: webhook-mailer at python.org (zooba)
Date: Mon, 06 Dec 2021 17:25:34 -0000
Subject: [Python-checkins] bpo-45582: Fix getpath_isxfile() and test_embed
 on Windows (GH-29930)
Message-ID: <mailman.60.1638811535.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/af1db4eb555e02d2bff3476f99f7a653764203b0
commit: af1db4eb555e02d2bff3476f99f7a653764203b0
branch: main
author: neonene <53406459+neonene at users.noreply.github.com>
committer: zooba <steve.dower at microsoft.com>
date: 2021-12-06T17:25:19Z
summary:

bpo-45582: Fix getpath_isxfile() and test_embed on Windows (GH-29930)

files:
M Lib/test/test_embed.py
M Modules/getpath.c

diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 3620a7619601d..94161b651ff86 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -575,7 +575,7 @@ def _get_expected_config(self):
         return configs
 
     def get_expected_config(self, expected_preconfig, expected,
-                            env, api, modify_path_cb=None):
+                            env, api, modify_path_cb=None, cwd=None):
         configs = self._get_expected_config()
 
         pre_config = configs['pre_config']
@@ -618,6 +618,14 @@ def get_expected_config(self, expected_preconfig, expected,
             expected['base_executable'] = default_executable
         if expected['program_name'] is self.GET_DEFAULT_CONFIG:
             expected['program_name'] = './_testembed'
+            if MS_WINDOWS:
+                # follow the calculation in getpath.py
+                tmpname = expected['program_name'] + '.exe'
+                if cwd:
+                    tmpname = os.path.join(cwd, tmpname)
+                if os.path.isfile(tmpname):
+                    expected['program_name'] += '.exe'
+                del tmpname
 
         config = configs['config']
         for key, value in expected.items():
@@ -711,7 +719,7 @@ def check_all_configs(self, testname, expected_config=None,
         self.get_expected_config(expected_preconfig,
                                  expected_config,
                                  env,
-                                 api, modify_path_cb)
+                                 api, modify_path_cb, cwd)
 
         out, err = self.run_embedded_interpreter(testname,
                                                  env=env, cwd=cwd)
diff --git a/Modules/getpath.c b/Modules/getpath.c
index 8f90a7008625f..2e46226a08531 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -230,7 +230,7 @@ getpath_isxfile(PyObject *Py_UNUSED(self), PyObject *args)
         DWORD attr = GetFileAttributesW(path);
         r = (attr != INVALID_FILE_ATTRIBUTES) &&
             !(attr & FILE_ATTRIBUTE_DIRECTORY) &&
-            SUCCEEDED(PathCchFindExtension(path, cchPath, &ext)) &&
+            SUCCEEDED(PathCchFindExtension(path, cchPath + 1, &ext)) &&
             (CompareStringOrdinal(ext, -1, L".exe", -1, 1 /* ignore case */) == CSTR_EQUAL)
             ? Py_True : Py_False;
 #else


From webhook-mailer at python.org  Mon Dec  6 13:13:22 2021
From: webhook-mailer at python.org (tiran)
Date: Mon, 06 Dec 2021 18:13:22 -0000
Subject: [Python-checkins] bpo-45582: framework build: modPath must not be
 const (GH-29944)
Message-ID: <mailman.61.1638814403.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/f16f93e5279f957ca25dd8b91233a44833167a8a
commit: f16f93e5279f957ca25dd8b91233a44833167a8a
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-06T19:13:12+01:00
summary:

bpo-45582: framework build: modPath must not be const (GH-29944)

Co-authored-by: Ronald Oussoren <ronaldoussoren at mac.com>

files:
M Modules/getpath.c

diff --git a/Modules/getpath.c b/Modules/getpath.c
index 2e46226a08531..9ce7260f77826 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -756,7 +756,7 @@ library_to_dict(PyObject *dict, const char *key)
     }
 #elif defined(WITH_NEXT_FRAMEWORK) && !defined(PY_BOOTSTRAP_PYTHON)
     // _bootstrap_python does not use framework and crashes
-    static const char modPath[MAXPATHLEN + 1];
+    static char modPath[MAXPATHLEN + 1];
     static int modPathInitialized = -1;
     if (modPathInitialized < 0) {
         NSModule pythonModule;


From webhook-mailer at python.org  Mon Dec  6 15:43:54 2021
From: webhook-mailer at python.org (tiran)
Date: Mon, 06 Dec 2021 20:43:54 -0000
Subject: [Python-checkins] bpo-45847: Fix uuid detection on macOS (GH-29946)
Message-ID: <mailman.62.1638823435.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/fc012d801202a9ea139df143b934778060d51a60
commit: fc012d801202a9ea139df143b934778060d51a60
branch: main
author: Christian Heimes <christian at python.org>
committer: tiran <christian at python.org>
date: 2021-12-06T21:43:44+01:00
summary:

bpo-45847: Fix uuid detection on macOS (GH-29946)

files:
M configure
M configure.ac

diff --git a/configure b/configure
index fca9567c7102f..2237e6ed8ce2a 100755
--- a/configure
+++ b/configure
@@ -10623,7 +10623,6 @@ fi
 
       LIBUUID_LIBS="-luuid"
       LIBUUID_CFLAGS=
-      have_uuid=no
       for ac_header in uuid/uuid.h
 do :
   ac_fn_c_check_header_mongrel "$LINENO" "uuid/uuid.h" "ac_cv_header_uuid_uuid_h" "$ac_includes_default"
@@ -10754,7 +10753,6 @@ $as_echo "no" >&6; }
 
       LIBUUID_LIBS="-luuid"
       LIBUUID_CFLAGS=
-      have_uuid=no
       for ac_header in uuid/uuid.h
 do :
   ac_fn_c_check_header_mongrel "$LINENO" "uuid/uuid.h" "ac_cv_header_uuid_uuid_h" "$ac_includes_default"
@@ -10895,6 +10893,37 @@ fi
 
 fi
 
+if test "x$have_uuid" = xmissing; then :
+
+  for ac_header in uuid/uuid.h
+do :
+  ac_fn_c_check_header_mongrel "$LINENO" "uuid/uuid.h" "ac_cv_header_uuid_uuid_h" "$ac_includes_default"
+if test "x$ac_cv_header_uuid_uuid_h" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_UUID_UUID_H 1
+_ACEOF
+
+    ac_fn_c_check_func "$LINENO" "uuid_generate_time" "ac_cv_func_uuid_generate_time"
+if test "x$ac_cv_func_uuid_generate_time" = xyes; then :
+
+      have_uuid=yes
+      LIBUUID_CFLAGS=
+      LIBUUID_LIBS=
+
+fi
+
+
+fi
+
+done
+
+
+fi
+
+if test "x$have_uuid" = xmissing; then :
+  have_uuid=no
+fi
+
 # 'Real Time' functions on Solaris
 # posix4 on Solaris 2.6
 # pthread (first!) on Linux
diff --git a/configure.ac b/configure.ac
index c7c71255a3a4c..f1aac2db71f5d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2963,7 +2963,6 @@ AS_VAR_IF([have_uuid], [missing], [
     ], [
       LIBUUID_LIBS="-luuid"
       LIBUUID_CFLAGS=
-      have_uuid=no
       AC_CHECK_HEADERS([uuid/uuid.h], [
         WITH_SAVE_ENV(
           [AC_CHECK_LIB([uuid], [uuid_generate_time], [have_uuid=yes])
@@ -2979,6 +2978,19 @@ AS_VAR_IF([have_uuid], [missing], [
   )
 ])
 
+dnl macOS has uuid/uuid.h but uuid_generate_time is in libc
+AS_VAR_IF([have_uuid], [missing], [
+  AC_CHECK_HEADERS([uuid/uuid.h], [
+    AC_CHECK_FUNC([uuid_generate_time], [
+      have_uuid=yes
+      LIBUUID_CFLAGS=
+      LIBUUID_LIBS=
+    ])
+  ])
+])
+
+AS_VAR_IF([have_uuid], [missing], [have_uuid=no])
+
 # 'Real Time' functions on Solaris
 # posix4 on Solaris 2.6
 # pthread (first!) on Linux


From webhook-mailer at python.org  Mon Dec  6 15:46:49 2021
From: webhook-mailer at python.org (pablogsal)
Date: Mon, 06 Dec 2021 20:46:49 -0000
Subject: [Python-checkins] Python 3.10.1
Message-ID: <mailman.63.1638823610.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/2cd268a3a9340346dd86b66db2e9b428b3f878fc
commit: 2cd268a3a9340346dd86b66db2e9b428b3f878fc
branch: 3.10
author: Pablo Galindo <pablogsal at gmail.com>
committer: pablogsal <Pablogsal at gmail.com>
date: 2021-12-06T18:23:39Z
summary:

Python 3.10.1

files:
A Misc/NEWS.d/3.10.1.rst
D Misc/NEWS.d/next/Build/2021-09-09-16-45-26.bpo-45067.mFmY92.rst
D Misc/NEWS.d/next/Build/2021-09-16-18-00-43.bpo-45220.TgbkvW.rst
D Misc/NEWS.d/next/Build/2021-10-11-16-27-38.bpo-45405.iSfdW5.rst
D Misc/NEWS.d/next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst
D Misc/NEWS.d/next/Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst
D Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst
D Misc/NEWS.d/next/Build/2021-10-21-14-38-30.bpo-45561.PVqhZE.rst
D Misc/NEWS.d/next/Build/2021-10-22-15-28-29.bpo-45571.yY8NsJ.rst
D Misc/NEWS.d/next/Build/2021-11-01-12-51-46.bpo-43158.fghS6w.rst
D Misc/NEWS.d/next/Build/2021-11-24-17-14-06.bpo-45881.GTXXLk.rst
D Misc/NEWS.d/next/Build/2021-11-25-09-15-04.bpo-41498.qAk5eo.rst
D Misc/NEWS.d/next/Build/2021-11-25-13-53-36.bpo-45866.ZH1W8N.rst
D Misc/NEWS.d/next/Build/2021-11-25-20-26-06.bpo-33393.24YNtM.rst
D Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
D Misc/NEWS.d/next/C API/2021-07-27-17-29-12.bpo-44751.4qmbDG.rst
D Misc/NEWS.d/next/C API/2021-09-19-17-18-25.bpo-44687.3fqDRC.rst
D Misc/NEWS.d/next/C API/2021-09-28-12-00-55.bpo-45307.3ETFfX.rst
D Misc/NEWS.d/next/C API/2021-11-09-15-42-11.bpo-39026.sUnYWn.rst
D Misc/NEWS.d/next/Core and Builtins/2021-09-01-16-55-43.bpo-45056.7AK2d9.rst
D Misc/NEWS.d/next/Core and Builtins/2021-09-01-23-55-49.bpo-45083.cLi9G3.rst
D Misc/NEWS.d/next/Core and Builtins/2021-09-07-17-10-16.bpo-45121.iG-Hsf.rst
D Misc/NEWS.d/next/Core and Builtins/2021-09-08-00-30-09.bpo-44050.mFI15u.rst
D Misc/NEWS.d/next/Core and Builtins/2021-09-08-08-29-41.bpo-44959.OSwwPf.rst
D Misc/NEWS.d/next/Core and Builtins/2021-09-09-10-32-33.bpo-44219.WiYyjz.rst
D Misc/NEWS.d/next/Core and Builtins/2021-09-14-09-23-59.bpo-45167.CPSSoV.rst
D Misc/NEWS.d/next/Core and Builtins/2021-10-06-21-20-11.bpo-45385.CTUT8s.rst
D Misc/NEWS.d/next/Core and Builtins/2021-10-07-21-26-44.bpo-45408.qUqzcd.rst
D Misc/NEWS.d/next/Core and Builtins/2021-10-16-17-27-48.bpo-45494.vMt1g4.rst
D Misc/NEWS.d/next/Core and Builtins/2021-10-18-22-40-33.bpo-45521.GdMiuW.rst
D Misc/NEWS.d/next/Core and Builtins/2021-10-19-01-04-08.bpo-30570._G30Ms.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-02-09-27-46.bpo-45688.v5Der1.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-04-20-19-07.bpo-45716.5C0pA1.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-09-13-01-35.bpo-45773.POU8A4.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-14-00-14-45.bpo-45738.e0cgKd.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-15-12-08-27.bpo-42540.V2w107.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-16-19-00-27.bpo-45820.2X6Psr.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-16-19-41-04.bpo-45822.OT6ueS.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-17-08-05-27.bpo-45826.OERoTm.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-19-22-57-42.bpo-45848.HgVBJ5.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-23-12-06-41.bpo-45614.fIekgI.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-24-18-24-49.bpo-45727._xVbbo.rst
D Misc/NEWS.d/next/Core and Builtins/2021-11-26-22-31-22.bpo-42268.3wl-09.rst
D Misc/NEWS.d/next/Documentation/2021-05-24-05-00-12.bpo-43905.tBIndE.rst
D Misc/NEWS.d/next/Documentation/2021-06-21-17-51-51.bpo-25381.7Kn-_H.rst
D Misc/NEWS.d/next/Documentation/2021-09-08-17-20-19.bpo-45024.dkNPNi.rst
D Misc/NEWS.d/next/Documentation/2021-09-18-13-45-19.bpo-45216.o56nyt.rst
D Misc/NEWS.d/next/Documentation/2021-10-13-00-42-54.bpo-20692.K5rGtP.rst
D Misc/NEWS.d/next/Documentation/2021-10-18-20-12-18.bpo-45516.EJh4K8.rst
D Misc/NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst
D Misc/NEWS.d/next/Documentation/2021-10-20-16-26-53.bpo-45464.mOISBs.rst
D Misc/NEWS.d/next/Documentation/2021-10-22-12-09-18.bpo-45250.Iit5-Y.rst
D Misc/NEWS.d/next/Documentation/2021-10-26-10-00-45.bpo-45604.Dm-YhV.rst
D Misc/NEWS.d/next/Documentation/2021-10-28-19-22-55.bpo-45655.aPYGaS.rst
D Misc/NEWS.d/next/Documentation/2021-11-03-14-51-03.bpo-45680.9_NTFU.rst
D Misc/NEWS.d/next/Documentation/2021-11-05-12-15-24.bpo-45726.GwRr7e.rst
D Misc/NEWS.d/next/Documentation/2021-11-06-10-54-17.bpo-45392.JZnVOz.rst
D Misc/NEWS.d/next/Documentation/2021-11-09-13-10-55.bpo-45772.EdrM3t.rst
D Misc/NEWS.d/next/Documentation/2021-11-18-00-07-40.bpo-45788.qibUoB.rst
D Misc/NEWS.d/next/Documentation/2021-11-18-16-44-12.bpo-45640.lSpc2A.rst
D Misc/NEWS.d/next/IDLE/2021-09-15-03-20-06.bpo-45193.G61_GV.rst
D Misc/NEWS.d/next/IDLE/2021-09-27-01-21-59.bpo-45296.9H8rdY.rst
D Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst
D Misc/NEWS.d/next/Library/2021-04-20-14-14-16.bpo-43498.L_Hq-8.rst
D Misc/NEWS.d/next/Library/2021-06-02-16-39-42.bpo-44295.erg01m.rst
D Misc/NEWS.d/next/Library/2021-07-12-10-32-48.bpo-44594.eEa5zi.rst
D Misc/NEWS.d/next/Library/2021-08-18-10-36-14.bpo-39039.A63LYh.rst
D Misc/NEWS.d/next/Library/2021-08-28-13-00-12.bpo-45021.rReeaj.rst
D Misc/NEWS.d/next/Library/2021-08-30-00-19-23.bpo-24444.Ki4bgz.rst
D Misc/NEWS.d/next/Library/2021-09-08-01-19-31.bpo-20499.tSxx8Y.rst
D Misc/NEWS.d/next/Library/2021-09-10-21-35-53.bpo-45166.UHipXF.rst
D Misc/NEWS.d/next/Library/2021-09-11-10-45-12.bpo-35474.tEY3SD.rst
D Misc/NEWS.d/next/Library/2021-09-11-14-47-05.bpo-45160.VzMXbW.rst
D Misc/NEWS.d/next/Library/2021-09-13-19-32-58.bpo-42135.1ZAHqR.rst
D Misc/NEWS.d/next/Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst
D Misc/NEWS.d/next/Library/2021-09-17-09-59-33.bpo-45228.WV1dcT.rst
D Misc/NEWS.d/next/Library/2021-09-17-11-20-55.bpo-45234.qUcTVt.rst
D Misc/NEWS.d/next/Library/2021-09-17-15-58-53.bpo-45183.Vv_vch.rst
D Misc/NEWS.d/next/Library/2021-09-17-16-55-37.bpo-45235.sXnmPA.rst
D Misc/NEWS.d/next/Library/2021-09-18-13-14-57.bpo-36674.a2k5Zb.rst
D Misc/NEWS.d/next/Library/2021-09-18-16-56-33.bpo-45238.Hng_9V.rst
D Misc/NEWS.d/next/Library/2021-09-23-22-17-26.bpo-45274.gPpa4E.rst
D Misc/NEWS.d/next/Library/2021-09-24-17-20-23.bpo-1596321.3nhPUk.rst
D Misc/NEWS.d/next/Library/2021-09-30-08-22-44.bpo-45328.8Z-Q0B.rst
D Misc/NEWS.d/next/Library/2021-09-30-23-00-18.bpo-41710.svuloZ.rst
D Misc/NEWS.d/next/Library/2021-10-01-13-09-53.bpo-45329.9iMYaO.rst
D Misc/NEWS.d/next/Library/2021-10-01-23-07-02.bpo-45343.ixmctD.rst
D Misc/NEWS.d/next/Library/2021-10-03-21-14-37.bpo-20028.zBA4RK.rst
D Misc/NEWS.d/next/Library/2021-10-05-11-03-48.bpo-45371.NOwcDJ.rst
D Misc/NEWS.d/next/Library/2021-10-07-00-05-05.bpo-45386.q9ORpA.rst
D Misc/NEWS.d/next/Library/2021-10-07-14-04-10.bpo-45262.HqF71Z.rst
D Misc/NEWS.d/next/Library/2021-10-08-19-24-48.bpo-45406.Qh_Mz4.rst
D Misc/NEWS.d/next/Library/2021-10-09-18-42-27.bpo-44904.RlW5h8.rst
D Misc/NEWS.d/next/Library/2021-10-09-20-53-13.bpo-45419.CauCgt.rst
D Misc/NEWS.d/next/Library/2021-10-10-09-42-34.bpo-45416.n35O0_.rst
D Misc/NEWS.d/next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst
D Misc/NEWS.d/next/Library/2021-10-13-17-52-48.bpo-45239.7li1_0.rst
D Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst
D Misc/NEWS.d/next/Library/2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst
D Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst
D Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst
D Misc/NEWS.d/next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst
D Misc/NEWS.d/next/Library/2021-10-21-16-18-51.bpo-45557.4MQt4r.rst
D Misc/NEWS.d/next/Library/2021-10-22-21-57-02.bpo-45581.rlH6ay.rst
D Misc/NEWS.d/next/Library/2021-10-22-23-06-33.bpo-45574.svqA84.rst
D Misc/NEWS.d/next/Library/2021-10-27-10-05-39.bpo-45438.Xz5lGU.rst
D Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst
D Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
D Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
D Misc/NEWS.d/next/Library/2021-10-30-21-11-37.bpo-45679.Dq8Cpu.rst
D Misc/NEWS.d/next/Library/2021-11-06-17-47-46.bpo-45644.ZMqHD_.rst
D Misc/NEWS.d/next/Library/2021-11-08-23-22-14.bpo-45757.MHZHt3.rst
D Misc/NEWS.d/next/Library/2021-11-09-09-04-19.bpo-45765.JVobxK.rst
D Misc/NEWS.d/next/Library/2021-11-11-13-03-17.bpo-45235.8ZbkHa.rst
D Misc/NEWS.d/next/Library/2021-11-16-18-13-49.bpo-41735.D72UY1.rst
D Misc/NEWS.d/next/Library/2021-11-17-19-25-37.bpo-45831.9-TojK.rst
D Misc/NEWS.d/next/Library/2021-11-20-17-04-25.bpo-45803.wSgFOy.rst
D Misc/NEWS.d/next/Library/2021-11-21-20-50-42.bpo-44649.E8M936.rst
D Misc/NEWS.d/next/Library/2021-11-28-15-30-34.bpo-37658.8Hno7d.rst
D Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
D Misc/NEWS.d/next/Tests/2021-08-27-22-37-19.bpo-25130.ig4oJe.rst
D Misc/NEWS.d/next/Tests/2021-09-08-13-01-37.bpo-44860.qXd0kx.rst
D Misc/NEWS.d/next/Tests/2021-09-11-22-08-18.bpo-45125.FVSzs2.rst
D Misc/NEWS.d/next/Tests/2021-09-13-00-28-17.bpo-45156.8oomV3.rst
D Misc/NEWS.d/next/Tests/2021-09-14-13-16-18.bpo-45195.EyQR1G.rst
D Misc/NEWS.d/next/Tests/2021-09-15-23-32-39.bpo-45209.55ntL5.rst
D Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst
D Misc/NEWS.d/next/Tests/2021-09-24-10-41-49.bpo-45269.8jKEr8.rst
D Misc/NEWS.d/next/Tests/2021-09-25-11-05-31.bpo-45280.3MA6lC.rst
D Misc/NEWS.d/next/Tests/2021-09-30-16-54-39.bpo-40173.J_slCw.rst
D Misc/NEWS.d/next/Tests/2021-10-07-13-11-45.bpo-45400.h3iT7V.rst
D Misc/NEWS.d/next/Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst
D Misc/NEWS.d/next/Tests/2021-10-21-17-22-26.bpo-43592.kHRsra.rst
D Misc/NEWS.d/next/Tests/2021-10-22-12-05-21.bpo-45566.2gQ3ZB.rst
D Misc/NEWS.d/next/Tests/2021-10-22-19-44-13.bpo-45577.dSaNvK.rst
D Misc/NEWS.d/next/Tests/2021-10-30-13-12-20.bpo-45678.bKrYeS.rst
D Misc/NEWS.d/next/Tests/2021-10-30-19-00-25.bpo-45578.bvu6X2.rst
D Misc/NEWS.d/next/Tests/2021-11-04-20-03-32.bpo-45678.1xNMjN.rst
D Misc/NEWS.d/next/Tests/2021-11-17-14-28-08.bpo-45835.Mgyhjx.rst
D Misc/NEWS.d/next/Tests/2021-11-28-15-25-02.bpo-19460.lr0aWs.rst
D Misc/NEWS.d/next/Tools-Demos/2021-09-14-11-44-26.bpo-44786.DU0LC0.rst
D Misc/NEWS.d/next/Windows/2021-09-30-23-17-27.bpo-45337.qg7U_h.rst
D Misc/NEWS.d/next/Windows/2021-11-04-00-41-50.bpo-43652.RnqV7I.rst
D Misc/NEWS.d/next/Windows/2021-11-05-01-05-46.bpo-45720.47Nc5I.rst
D Misc/NEWS.d/next/Windows/2021-11-08-21-53-11.bpo-45732.idl5kx.rst
D Misc/NEWS.d/next/Windows/2021-11-23-11-44-42.bpo-45616.K52PLZ.rst
D Misc/NEWS.d/next/Windows/2021-11-26-18-17-41.bpo-45901.c5IBqM.rst
D Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst
D Misc/NEWS.d/next/macOS/2021-10-25-02-02-21.bpo-44828.XBdXlJ.rst
D Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
M Include/patchlevel.h
M Lib/pydoc_data/topics.py
M README.rst

diff --git a/Include/patchlevel.h b/Include/patchlevel.h
index 56869c5fa74ab..9e8a56099cc5a 100644
--- a/Include/patchlevel.h
+++ b/Include/patchlevel.h
@@ -18,12 +18,12 @@
 /*--start constants--*/
 #define PY_MAJOR_VERSION        3
 #define PY_MINOR_VERSION        10
-#define PY_MICRO_VERSION        0
+#define PY_MICRO_VERSION        1
 #define PY_RELEASE_LEVEL        PY_RELEASE_LEVEL_FINAL
 #define PY_RELEASE_SERIAL       0
 
 /* Version as a string */
-#define PY_VERSION              "3.10.0+"
+#define PY_VERSION              "3.10.1"
 /*--end constants--*/
 
 /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.
diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py
index 7720694bd592e..00c98ad51072c 100644
--- a/Lib/pydoc_data/topics.py
+++ b/Lib/pydoc_data/topics.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Autogenerated by Sphinx on Mon Oct  4 18:28:12 2021
+# Autogenerated by Sphinx on Mon Dec  6 17:57:38 2021
 topics = {'assert': 'The "assert" statement\n'
            '**********************\n'
            '\n'
@@ -3339,9 +3339,9 @@
              '\n'
              'The same keyword should not be repeated in class patterns.\n'
              '\n'
-             'The following is the logical flow for matching a mapping '
-             'pattern\n'
-             'against a subject value:\n'
+             'The following is the logical flow for matching a class pattern '
+             'against\n'
+             'a subject value:\n'
              '\n'
              '1. If "name_or_attr" is not an instance of the builtin "type" , '
              'raise\n'
@@ -5488,20 +5488,32 @@
               'binding\n'
               'operations.\n'
               '\n'
-              'The following constructs bind names: formal parameters to '
-              'functions,\n'
-              '"import" statements, class and function definitions (these bind '
-              'the\n'
-              'class or function name in the defining block), and targets that '
-              'are\n'
-              'identifiers if occurring in an assignment, "for" loop header, '
-              'or after\n'
-              '"as" in a "with" statement or "except" clause. The "import" '
-              'statement\n'
-              'of the form "from ... import *" binds all names defined in the\n'
-              'imported module, except those beginning with an underscore.  '
-              'This form\n'
-              'may only be used at the module level.\n'
+              'The following constructs bind names:\n'
+              '\n'
+              '* formal parameters to functions,\n'
+              '\n'
+              '* class definitions,\n'
+              '\n'
+              '* function definitions,\n'
+              '\n'
+              '* assignment expressions,\n'
+              '\n'
+              '* targets that are identifiers if occurring in an assignment:\n'
+              '\n'
+              '  * "for" loop header,\n'
+              '\n'
+              '  * after "as" in a "with" statement, "except" clause or in the '
+              'as-\n'
+              '    pattern in structural pattern matching,\n'
+              '\n'
+              '  * in a capture pattern in structural pattern matching\n'
+              '\n'
+              '* "import" statements.\n'
+              '\n'
+              'The "import" statement of the form "from ... import *" binds '
+              'all names\n'
+              'defined in the imported module, except those beginning with an\n'
+              'underscore. This form may only be used at the module level.\n'
               '\n'
               'A target occurring in a "del" statement is also considered '
               'bound for\n'
@@ -5574,9 +5586,9 @@
               'operations.\n'
               '\n'
               'If the "global" statement occurs within a block, all uses of '
-              'the name\n'
-              'specified in the statement refer to the binding of that name in '
-              'the\n'
+              'the names\n'
+              'specified in the statement refer to the bindings of those names '
+              'in the\n'
               'top-level namespace.  Names are resolved in the top-level '
               'namespace by\n'
               'searching the global namespace, i.e. the namespace of the '
@@ -5585,9 +5597,10 @@
               'namespace\n'
               'of the module "builtins".  The global namespace is searched '
               'first.  If\n'
-              'the name is not found there, the builtins namespace is '
-              'searched.  The\n'
-              '"global" statement must precede all uses of the name.\n'
+              'the names are not found there, the builtins namespace is '
+              'searched.\n'
+              'The "global" statement must precede all uses of the listed '
+              'names.\n'
               '\n'
               'The "global" statement has the same scope as a name binding '
               'operation\n'
@@ -6943,22 +6956,31 @@
                'trailing underscore characters:\n'
                '\n'
                '"_*"\n'
-               '   Not imported by "from module import *".  The special '
-               'identifier "_"\n'
-               '   is used in the interactive interpreter to store the result '
-               'of the\n'
-               '   last evaluation; it is stored in the "builtins" module.  '
-               'When not\n'
-               '   in interactive mode, "_" has no special meaning and is not '
-               'defined.\n'
-               '   See section The import statement.\n'
+               '   Not imported by "from module import *".\n'
+               '\n'
+               '"_"\n'
+               '   In a "case" pattern within a "match" statement, "_" is a '
+               'soft\n'
+               '   keyword that denotes a wildcard.\n'
+               '\n'
+               '   Separately, the interactive interpreter makes the result of '
+               'the\n'
+               '   last evaluation available in the variable "_". (It is '
+               'stored in the\n'
+               '   "builtins" module, alongside built-in functions like '
+               '"print".)\n'
+               '\n'
+               '   Elsewhere, "_" is a regular identifier. It is often used to '
+               'name\n'
+               '   ?special? items, but it is not special to Python itself.\n'
                '\n'
                '   Note:\n'
                '\n'
                '     The name "_" is often used in conjunction with\n'
                '     internationalization; refer to the documentation for the\n'
                '     "gettext" module for more information on this '
-               'convention.\n'
+               'convention.It is\n'
+               '     also commonly used for unused variables.\n'
                '\n'
                '"__*__"\n'
                '   System-defined names, informally known as ?dunder? names. '
@@ -7113,15 +7135,23 @@
                 'trailing underscore characters:\n'
                 '\n'
                 '"_*"\n'
-                '   Not imported by "from module import *".  The special '
-                'identifier "_"\n'
-                '   is used in the interactive interpreter to store the result '
+                '   Not imported by "from module import *".\n'
+                '\n'
+                '"_"\n'
+                '   In a "case" pattern within a "match" statement, "_" is a '
+                'soft\n'
+                '   keyword that denotes a wildcard.\n'
+                '\n'
+                '   Separately, the interactive interpreter makes the result '
                 'of the\n'
-                '   last evaluation; it is stored in the "builtins" module.  '
-                'When not\n'
-                '   in interactive mode, "_" has no special meaning and is not '
-                'defined.\n'
-                '   See section The import statement.\n'
+                '   last evaluation available in the variable "_". (It is '
+                'stored in the\n'
+                '   "builtins" module, alongside built-in functions like '
+                '"print".)\n'
+                '\n'
+                '   Elsewhere, "_" is a regular identifier. It is often used '
+                'to name\n'
+                '   ?special? items, but it is not special to Python itself.\n'
                 '\n'
                 '   Note:\n'
                 '\n'
@@ -7129,7 +7159,8 @@
                 '     internationalization; refer to the documentation for '
                 'the\n'
                 '     "gettext" module for more information on this '
-                'convention.\n'
+                'convention.It is\n'
+                '     also commonly used for unused variables.\n'
                 '\n'
                 '"__*__"\n'
                 '   System-defined names, informally known as ?dunder? names. '
@@ -7590,20 +7621,32 @@
            '*Names* refer to objects.  Names are introduced by name binding\n'
            'operations.\n'
            '\n'
-           'The following constructs bind names: formal parameters to '
-           'functions,\n'
-           '"import" statements, class and function definitions (these bind '
-           'the\n'
-           'class or function name in the defining block), and targets that '
-           'are\n'
-           'identifiers if occurring in an assignment, "for" loop header, or '
-           'after\n'
-           '"as" in a "with" statement or "except" clause. The "import" '
-           'statement\n'
-           'of the form "from ... import *" binds all names defined in the\n'
-           'imported module, except those beginning with an underscore.  This '
-           'form\n'
-           'may only be used at the module level.\n'
+           'The following constructs bind names:\n'
+           '\n'
+           '* formal parameters to functions,\n'
+           '\n'
+           '* class definitions,\n'
+           '\n'
+           '* function definitions,\n'
+           '\n'
+           '* assignment expressions,\n'
+           '\n'
+           '* targets that are identifiers if occurring in an assignment:\n'
+           '\n'
+           '  * "for" loop header,\n'
+           '\n'
+           '  * after "as" in a "with" statement, "except" clause or in the '
+           'as-\n'
+           '    pattern in structural pattern matching,\n'
+           '\n'
+           '  * in a capture pattern in structural pattern matching\n'
+           '\n'
+           '* "import" statements.\n'
+           '\n'
+           'The "import" statement of the form "from ... import *" binds all '
+           'names\n'
+           'defined in the imported module, except those beginning with an\n'
+           'underscore. This form may only be used at the module level.\n'
            '\n'
            'A target occurring in a "del" statement is also considered bound '
            'for\n'
@@ -7673,8 +7716,8 @@
            'operations.\n'
            '\n'
            'If the "global" statement occurs within a block, all uses of the '
-           'name\n'
-           'specified in the statement refer to the binding of that name in '
+           'names\n'
+           'specified in the statement refer to the bindings of those names in '
            'the\n'
            'top-level namespace.  Names are resolved in the top-level '
            'namespace by\n'
@@ -7683,9 +7726,9 @@
            'namespace\n'
            'of the module "builtins".  The global namespace is searched '
            'first.  If\n'
-           'the name is not found there, the builtins namespace is searched.  '
-           'The\n'
-           '"global" statement must precede all uses of the name.\n'
+           'the names are not found there, the builtins namespace is '
+           'searched.\n'
+           'The "global" statement must precede all uses of the listed names.\n'
            '\n'
            'The "global" statement has the same scope as a name binding '
            'operation\n'
@@ -8020,9 +8063,9 @@
                   '   of the object truncated to an "Integral" (typically an '
                   '"int").\n'
                   '\n'
-                  '   If "__int__()" is not defined then the built-in function '
-                  '"int()"\n'
-                  '   falls back to "__trunc__()".\n',
+                  '   The built-in function "int()" falls back to '
+                  '"__trunc__()" if\n'
+                  '   neither "__int__()" nor "__index__()" is defined.\n',
  'objects': 'Objects, values and types\n'
             '*************************\n'
             '\n'
@@ -8644,22 +8687,24 @@
                    'object.__getitem__(self, key)\n'
                    '\n'
                    '   Called to implement evaluation of "self[key]". For '
-                   'sequence types,\n'
-                   '   the accepted keys should be integers and slice '
-                   'objects.  Note that\n'
-                   '   the special interpretation of negative indexes (if the '
-                   'class wishes\n'
-                   '   to emulate a sequence type) is up to the '
-                   '"__getitem__()" method. If\n'
-                   '   *key* is of an inappropriate type, "TypeError" may be '
-                   'raised; if of\n'
-                   '   a value outside the set of indexes for the sequence '
-                   '(after any\n'
-                   '   special interpretation of negative values), '
-                   '"IndexError" should be\n'
-                   '   raised. For mapping types, if *key* is missing (not in '
+                   '*sequence*\n'
+                   '   types, the accepted keys should be integers and slice '
+                   'objects.\n'
+                   '   Note that the special interpretation of negative '
+                   'indexes (if the\n'
+                   '   class wishes to emulate a *sequence* type) is up to '
                    'the\n'
-                   '   container), "KeyError" should be raised.\n'
+                   '   "__getitem__()" method. If *key* is of an inappropriate '
+                   'type,\n'
+                   '   "TypeError" may be raised; if of a value outside the '
+                   'set of indexes\n'
+                   '   for the sequence (after any special interpretation of '
+                   'negative\n'
+                   '   values), "IndexError" should be raised. For *mapping* '
+                   'types, if\n'
+                   '   *key* is missing (not in the container), "KeyError" '
+                   'should be\n'
+                   '   raised.\n'
                    '\n'
                    '   Note:\n'
                    '\n'
@@ -8669,6 +8714,15 @@
                    'of the\n'
                    '     sequence.\n'
                    '\n'
+                   '   Note:\n'
+                   '\n'
+                   '     When subscripting a *class*, the special class '
+                   'method\n'
+                   '     "__class_getitem__()" may be called instead of '
+                   '"__getitem__()".\n'
+                   '     See __class_getitem__ versus __getitem__ for more '
+                   'details.\n'
+                   '\n'
                    'object.__setitem__(self, key, value)\n'
                    '\n'
                    '   Called to implement assignment to "self[key]".  Same '
@@ -8704,19 +8758,13 @@
                    '\n'
                    'object.__iter__(self)\n'
                    '\n'
-                   '   This method is called when an iterator is required for '
-                   'a container.\n'
-                   '   This method should return a new iterator object that '
-                   'can iterate\n'
-                   '   over all the objects in the container.  For mappings, '
-                   'it should\n'
-                   '   iterate over the keys of the container.\n'
-                   '\n'
-                   '   Iterator objects also need to implement this method; '
-                   'they are\n'
-                   '   required to return themselves.  For more information on '
-                   'iterator\n'
-                   '   objects, see Iterator Types.\n'
+                   '   This method is called when an *iterator* is required '
+                   'for a\n'
+                   '   container. This method should return a new iterator '
+                   'object that can\n'
+                   '   iterate over all the objects in the container.  For '
+                   'mappings, it\n'
+                   '   should iterate over the keys of the container.\n'
                    '\n'
                    'object.__reversed__(self)\n'
                    '\n'
@@ -10294,9 +10342,33 @@
                  'Emulating generic types\n'
                  '=======================\n'
                  '\n'
-                 'One can implement the generic class syntax as specified by '
-                 '**PEP 484**\n'
-                 '(for example "List[int]") by defining a special method:\n'
+                 'When using *type annotations*, it is often useful to '
+                 '*parameterize* a\n'
+                 '*generic type* using Python?s square-brackets notation. For '
+                 'example,\n'
+                 'the annotation "list[int]" might be used to signify a "list" '
+                 'in which\n'
+                 'all the elements are of type "int".\n'
+                 '\n'
+                 'See also:\n'
+                 '\n'
+                 '  **PEP 484** - Type Hints\n'
+                 '     Introducing Python?s framework for type annotations\n'
+                 '\n'
+                 '  Generic Alias Types\n'
+                 '     Documentation for objects representing parameterized '
+                 'generic\n'
+                 '     classes\n'
+                 '\n'
+                 '  Generics, user-defined generics and "typing.Generic"\n'
+                 '     Documentation on how to implement generic classes that '
+                 'can be\n'
+                 '     parameterized at runtime and understood by static '
+                 'type-checkers.\n'
+                 '\n'
+                 'A class can *generally* only be parameterized if it defines '
+                 'the\n'
+                 'special class method "__class_getitem__()".\n'
                  '\n'
                  'classmethod object.__class_getitem__(cls, key)\n'
                  '\n'
@@ -10304,18 +10376,144 @@
                  'generic class\n'
                  '   by type arguments found in *key*.\n'
                  '\n'
-                 'This method is looked up on the class object itself, and '
-                 'when defined\n'
-                 'in the class body, this method is implicitly a class '
-                 'method.  Note,\n'
-                 'this mechanism is primarily reserved for use with static '
-                 'type hints,\n'
-                 'other usage is discouraged.\n'
+                 '   When defined on a class, "__class_getitem__()" is '
+                 'automatically a\n'
+                 '   class method. As such, there is no need for it to be '
+                 'decorated with\n'
+                 '   "@classmethod" when it is defined.\n'
+                 '\n'
+                 '\n'
+                 'The purpose of *__class_getitem__*\n'
+                 '----------------------------------\n'
+                 '\n'
+                 'The purpose of "__class_getitem__()" is to allow runtime\n'
+                 'parameterization of standard-library generic classes in '
+                 'order to more\n'
+                 'easily apply *type hints* to these classes.\n'
+                 '\n'
+                 'To implement custom generic classes that can be '
+                 'parameterized at\n'
+                 'runtime and understood by static type-checkers, users should '
+                 'either\n'
+                 'inherit from a standard library class that already '
+                 'implements\n'
+                 '"__class_getitem__()", or inherit from "typing.Generic", '
+                 'which has its\n'
+                 'own implementation of "__class_getitem__()".\n'
+                 '\n'
+                 'Custom implementations of "__class_getitem__()" on classes '
+                 'defined\n'
+                 'outside of the standard library may not be understood by '
+                 'third-party\n'
+                 'type-checkers such as mypy. Using "__class_getitem__()" on '
+                 'any class\n'
+                 'for purposes other than type hinting is discouraged.\n'
+                 '\n'
+                 '\n'
+                 '*__class_getitem__* versus *__getitem__*\n'
+                 '----------------------------------------\n'
+                 '\n'
+                 'Usually, the subscription of an object using square brackets '
+                 'will call\n'
+                 'the "__getitem__()" instance method defined on the object?s '
+                 'class.\n'
+                 'However, if the object being subscribed is itself a class, '
+                 'the class\n'
+                 'method "__class_getitem__()" may be called instead.\n'
+                 '"__class_getitem__()" should return a GenericAlias object if '
+                 'it is\n'
+                 'properly defined.\n'
+                 '\n'
+                 'Presented with the *expression* "obj[x]", the Python '
+                 'interpreter\n'
+                 'follows something like the following process to decide '
+                 'whether\n'
+                 '"__getitem__()" or "__class_getitem__()" should be called:\n'
+                 '\n'
+                 '   from inspect import isclass\n'
+                 '\n'
+                 '   def subscribe(obj, x):\n'
+                 '       """Return the result of the expression `obj[x]`"""\n'
+                 '\n'
+                 '       class_of_obj = type(obj)\n'
+                 '\n'
+                 '       # If the class of obj defines __getitem__,\n'
+                 '       # call class_of_obj.__getitem__(obj, x)\n'
+                 "       if hasattr(class_of_obj, '__getitem__'):\n"
+                 '           return class_of_obj.__getitem__(obj, x)\n'
+                 '\n'
+                 '       # Else, if obj is a class and defines '
+                 '__class_getitem__,\n'
+                 '       # call obj.__class_getitem__(x)\n'
+                 '       elif isclass(obj) and hasattr(obj, '
+                 "'__class_getitem__'):\n"
+                 '           return obj.__class_getitem__(x)\n'
+                 '\n'
+                 '       # Else, raise an exception\n'
+                 '       else:\n'
+                 '           raise TypeError(\n'
+                 '               f"\'{class_of_obj.__name__}\' object is not '
+                 'subscriptable"\n'
+                 '           )\n'
+                 '\n'
+                 'In Python, all classes are themselves instances of other '
+                 'classes. The\n'
+                 'class of a class is known as that class?s *metaclass*, and '
+                 'most\n'
+                 'classes have the "type" class as their metaclass. "type" '
+                 'does not\n'
+                 'define "__getitem__()", meaning that expressions such as '
+                 '"list[int]",\n'
+                 '"dict[str, float]" and "tuple[str, bytes]" all result in\n'
+                 '"__class_getitem__()" being called:\n'
+                 '\n'
+                 '   >>> # list has class "type" as its metaclass, like most '
+                 'classes:\n'
+                 '   >>> type(list)\n'
+                 "   <class 'type'>\n"
+                 '   >>> type(dict) == type(list) == type(tuple) == type(str) '
+                 '== type(bytes)\n'
+                 '   True\n'
+                 '   >>> # "list[int]" calls "list.__class_getitem__(int)"\n'
+                 '   >>> list[int]\n'
+                 '   list[int]\n'
+                 '   >>> # list.__class_getitem__ returns a GenericAlias '
+                 'object:\n'
+                 '   >>> type(list[int])\n'
+                 "   <class 'types.GenericAlias'>\n"
+                 '\n'
+                 'However, if a class has a custom metaclass that defines\n'
+                 '"__getitem__()", subscribing the class may result in '
+                 'different\n'
+                 'behaviour. An example of this can be found in the "enum" '
+                 'module:\n'
+                 '\n'
+                 '   >>> from enum import Enum\n'
+                 '   >>> class Menu(Enum):\n'
+                 '   ...     """A breakfast menu"""\n'
+                 "   ...     SPAM = 'spam'\n"
+                 "   ...     BACON = 'bacon'\n"
+                 '   ...\n'
+                 '   >>> # Enum classes have a custom metaclass:\n'
+                 '   >>> type(Menu)\n'
+                 "   <class 'enum.EnumMeta'>\n"
+                 '   >>> # EnumMeta defines __getitem__,\n'
+                 '   >>> # so __class_getitem__ is not called,\n'
+                 '   >>> # and the result is not a GenericAlias object:\n'
+                 "   >>> Menu['SPAM']\n"
+                 "   <Menu.SPAM: 'spam'>\n"
+                 "   >>> type(Menu['SPAM'])\n"
+                 "   <enum 'Menu'>\n"
                  '\n'
                  'See also:\n'
                  '\n'
-                 '  **PEP 560** - Core support for typing module and generic '
+                 '  **PEP 560** - Core Support for typing module and generic '
                  'types\n'
+                 '     Introducing "__class_getitem__()", and outlining when '
+                 'a\n'
+                 '     subscription results in "__class_getitem__()" being '
+                 'called\n'
+                 '     instead of "__getitem__()"\n'
                  '\n'
                  '\n'
                  'Emulating callable objects\n'
@@ -10445,22 +10643,23 @@
                  'object.__getitem__(self, key)\n'
                  '\n'
                  '   Called to implement evaluation of "self[key]". For '
-                 'sequence types,\n'
-                 '   the accepted keys should be integers and slice objects.  '
-                 'Note that\n'
-                 '   the special interpretation of negative indexes (if the '
-                 'class wishes\n'
-                 '   to emulate a sequence type) is up to the "__getitem__()" '
-                 'method. If\n'
-                 '   *key* is of an inappropriate type, "TypeError" may be '
-                 'raised; if of\n'
-                 '   a value outside the set of indexes for the sequence '
-                 '(after any\n'
-                 '   special interpretation of negative values), "IndexError" '
+                 '*sequence*\n'
+                 '   types, the accepted keys should be integers and slice '
+                 'objects.\n'
+                 '   Note that the special interpretation of negative indexes '
+                 '(if the\n'
+                 '   class wishes to emulate a *sequence* type) is up to the\n'
+                 '   "__getitem__()" method. If *key* is of an inappropriate '
+                 'type,\n'
+                 '   "TypeError" may be raised; if of a value outside the set '
+                 'of indexes\n'
+                 '   for the sequence (after any special interpretation of '
+                 'negative\n'
+                 '   values), "IndexError" should be raised. For *mapping* '
+                 'types, if\n'
+                 '   *key* is missing (not in the container), "KeyError" '
                  'should be\n'
-                 '   raised. For mapping types, if *key* is missing (not in '
-                 'the\n'
-                 '   container), "KeyError" should be raised.\n'
+                 '   raised.\n'
                  '\n'
                  '   Note:\n'
                  '\n'
@@ -10470,6 +10669,14 @@
                  'the\n'
                  '     sequence.\n'
                  '\n'
+                 '   Note:\n'
+                 '\n'
+                 '     When subscripting a *class*, the special class method\n'
+                 '     "__class_getitem__()" may be called instead of '
+                 '"__getitem__()".\n'
+                 '     See __class_getitem__ versus __getitem__ for more '
+                 'details.\n'
+                 '\n'
                  'object.__setitem__(self, key, value)\n'
                  '\n'
                  '   Called to implement assignment to "self[key]".  Same note '
@@ -10505,19 +10712,13 @@
                  '\n'
                  'object.__iter__(self)\n'
                  '\n'
-                 '   This method is called when an iterator is required for a '
-                 'container.\n'
-                 '   This method should return a new iterator object that can '
-                 'iterate\n'
-                 '   over all the objects in the container.  For mappings, it '
-                 'should\n'
-                 '   iterate over the keys of the container.\n'
-                 '\n'
-                 '   Iterator objects also need to implement this method; they '
-                 'are\n'
-                 '   required to return themselves.  For more information on '
-                 'iterator\n'
-                 '   objects, see Iterator Types.\n'
+                 '   This method is called when an *iterator* is required for '
+                 'a\n'
+                 '   container. This method should return a new iterator '
+                 'object that can\n'
+                 '   iterate over all the objects in the container.  For '
+                 'mappings, it\n'
+                 '   should iterate over the keys of the container.\n'
                  '\n'
                  'object.__reversed__(self)\n'
                  '\n'
@@ -10760,9 +10961,9 @@
                  '   of the object truncated to an "Integral" (typically an '
                  '"int").\n'
                  '\n'
-                 '   If "__int__()" is not defined then the built-in function '
-                 '"int()"\n'
-                 '   falls back to "__trunc__()".\n'
+                 '   The built-in function "int()" falls back to "__trunc__()" '
+                 'if\n'
+                 '   neither "__int__()" nor "__index__()" is defined.\n'
                  '\n'
                  '\n'
                  'With Statement Context Managers\n'
@@ -12971,20 +13172,18 @@
           '      A function or method which uses the "yield" statement (see\n'
           '      section The yield statement) is called a *generator '
           'function*.\n'
-          '      Such a function, when called, always returns an iterator '
-          'object\n'
-          '      which can be used to execute the body of the function:  '
-          'calling\n'
-          '      the iterator?s "iterator.__next__()" method will cause the\n'
-          '      function to execute until it provides a value using the '
-          '"yield"\n'
-          '      statement.  When the function executes a "return" statement '
-          'or\n'
-          '      falls off the end, a "StopIteration" exception is raised and '
-          'the\n'
-          '      iterator will have reached the end of the set of values to '
-          'be\n'
-          '      returned.\n'
+          '      Such a function, when called, always returns an *iterator*\n'
+          '      object which can be used to execute the body of the '
+          'function:\n'
+          '      calling the iterator?s "iterator.__next__()" method will '
+          'cause\n'
+          '      the function to execute until it provides a value using the\n'
+          '      "yield" statement.  When the function executes a "return"\n'
+          '      statement or falls off the end, a "StopIteration" exception '
+          'is\n'
+          '      raised and the iterator will have reached the end of the set '
+          'of\n'
+          '      values to be returned.\n'
           '\n'
           '   Coroutine functions\n'
           '      A function or method which is defined using "async def" is\n'
@@ -13000,9 +13199,9 @@
           '      which uses the "yield" statement is called a *asynchronous\n'
           '      generator function*.  Such a function, when called, returns '
           'an\n'
-          '      asynchronous iterator object which can be used in an "async '
-          'for"\n'
-          '      statement to execute the body of the function.\n'
+          '      *asynchronous iterator* object which can be used in an '
+          '"async\n'
+          '      for" statement to execute the body of the function.\n'
           '\n'
           '      Calling the asynchronous iterator?s "aiterator.__anext__()"\n'
           '      method will return an *awaitable* which when awaited will\n'
diff --git a/Misc/NEWS.d/3.10.1.rst b/Misc/NEWS.d/3.10.1.rst
new file mode 100644
index 0000000000000..0f8938ab6154d
--- /dev/null
+++ b/Misc/NEWS.d/3.10.1.rst
@@ -0,0 +1,1530 @@
+.. bpo: 42268
+.. date: 2021-11-26-22-31-22
+.. nonce: 3wl-09
+.. release date: 2021-12-06
+.. section: Core and Builtins
+
+Fail the configure step if the selected compiler doesn't support memory
+sanitizer. Patch by Pablo Galindo
+
+..
+
+.. bpo: 45727
+.. date: 2021-11-24-18-24-49
+.. nonce: _xVbbo
+.. section: Core and Builtins
+
+Refine the custom syntax error that suggests that a comma may be missing to
+trigger only when the expressions are detected between parentheses or
+brackets. Patch by Pablo Galindo
+
+..
+
+.. bpo: 45614
+.. date: 2021-11-23-12-06-41
+.. nonce: fIekgI
+.. section: Core and Builtins
+
+Fix :mod:`traceback` display for exceptions with invalid module name.
+
+..
+
+.. bpo: 45848
+.. date: 2021-11-19-22-57-42
+.. nonce: HgVBJ5
+.. section: Core and Builtins
+
+Allow the parser to obtain error lines directly from encoded files. Patch by
+Pablo Galindo
+
+..
+
+.. bpo: 45826
+.. date: 2021-11-17-08-05-27
+.. nonce: OERoTm
+.. section: Core and Builtins
+
+Fixed a crash when calling ``.with_traceback(None)`` on ``NameError``. This
+occurs internally in ``unittest.TestCase.assertRaises()``.
+
+..
+
+.. bpo: 45822
+.. date: 2021-11-16-19-41-04
+.. nonce: OT6ueS
+.. section: Core and Builtins
+
+Fixed a bug in the parser that was causing it to not respect :pep:`263`
+coding cookies when no flags are provided. Patch by Pablo Galindo
+
+..
+
+.. bpo: 45820
+.. date: 2021-11-16-19-00-27
+.. nonce: 2X6Psr
+.. section: Core and Builtins
+
+Fix a segfault when the parser fails without reading any input. Patch by
+Pablo Galindo
+
+..
+
+.. bpo: 42540
+.. date: 2021-11-15-12-08-27
+.. nonce: V2w107
+.. section: Core and Builtins
+
+Fix crash when :func:`os.fork` is called with an active non-default memory
+allocator.
+
+..
+
+.. bpo: 45738
+.. date: 2021-11-14-00-14-45
+.. nonce: e0cgKd
+.. section: Core and Builtins
+
+Fix computation of error location for invalid continuation characters in the
+parser. Patch by Pablo Galindo.
+
+..
+
+.. bpo: 45773
+.. date: 2021-11-09-13-01-35
+.. nonce: POU8A4
+.. section: Core and Builtins
+
+Fix a compiler hang when attempting to optimize certain jump patterns.
+
+..
+
+.. bpo: 45716
+.. date: 2021-11-04-20-19-07
+.. nonce: 5C0pA1
+.. section: Core and Builtins
+
+Improve the :exc:`SyntaxError` message when using ``True``, ``None`` or
+``False`` as keywords in a function call. Patch by Pablo Galindo.
+
+..
+
+.. bpo: 45688
+.. date: 2021-11-02-09-27-46
+.. nonce: v5Der1
+.. section: Core and Builtins
+
+:data:`sys.stdlib_module_names` now contains the macOS-specific module
+:mod:`_scproxy`.
+
+..
+
+.. bpo: 30570
+.. date: 2021-10-19-01-04-08
+.. nonce: _G30Ms
+.. section: Core and Builtins
+
+Fixed a crash in ``issubclass()`` from infinite recursion when searching
+pathological ``__bases__`` tuples.
+
+..
+
+.. bpo: 45521
+.. date: 2021-10-18-22-40-33
+.. nonce: GdMiuW
+.. section: Core and Builtins
+
+Fix a bug in the obmalloc radix tree code.  On 64-bit machines, the bug
+causes the tree to hold 46-bits of virtual addresses, rather than the
+intended 48-bits.
+
+..
+
+.. bpo: 45494
+.. date: 2021-10-16-17-27-48
+.. nonce: vMt1g4
+.. section: Core and Builtins
+
+Fix parser crash when reporting errors involving invalid continuation
+characters. Patch by Pablo Galindo.
+
+..
+
+.. bpo: 45408
+.. date: 2021-10-07-21-26-44
+.. nonce: qUqzcd
+.. section: Core and Builtins
+
+Fix a crash in the parser when reporting tokenizer errors that occur at the
+same time unclosed parentheses are detected. Patch by Pablo Galindo.
+
+..
+
+.. bpo: 45385
+.. date: 2021-10-06-21-20-11
+.. nonce: CTUT8s
+.. section: Core and Builtins
+
+Fix reference leak from descr_check. Patch by Dong-hee Na.
+
+..
+
+.. bpo: 45167
+.. date: 2021-09-14-09-23-59
+.. nonce: CPSSoV
+.. section: Core and Builtins
+
+Fix deepcopying of :class:`types.GenericAlias` objects.
+
+..
+
+.. bpo: 44219
+.. date: 2021-09-09-10-32-33
+.. nonce: WiYyjz
+.. section: Core and Builtins
+
+Release the GIL while performing ``isatty`` system calls on arbitrary file
+descriptors. In particular, this affects :func:`os.isatty`,
+:func:`os.device_encoding` and :class:`io.TextIOWrapper`. By extension,
+:func:`io.open` in text mode is also affected. This change solves a deadlock
+in :func:`os.isatty`. Patch by Vincent Michel in :issue:`44219`.
+
+..
+
+.. bpo: 44959
+.. date: 2021-09-08-08-29-41
+.. nonce: OSwwPf
+.. section: Core and Builtins
+
+Added fallback to extension modules with '.sl' suffix on HP-UX
+
+..
+
+.. bpo: 44050
+.. date: 2021-09-08-00-30-09
+.. nonce: mFI15u
+.. section: Core and Builtins
+
+Extensions that indicate they use global state (by setting ``m_size`` to -1)
+can again be used in multiple interpreters. This reverts to behavior of
+Python 3.8.
+
+..
+
+.. bpo: 45121
+.. date: 2021-09-07-17-10-16
+.. nonce: iG-Hsf
+.. section: Core and Builtins
+
+Fix issue where ``Protocol.__init__`` raises ``RecursionError`` when it's
+called directly or via ``super()``. Patch provided by Yurii Karabas.
+
+..
+
+.. bpo: 45083
+.. date: 2021-09-01-23-55-49
+.. nonce: cLi9G3
+.. section: Core and Builtins
+
+When the interpreter renders an exception, its name now has a complete
+qualname. Previously only the class name was concatenated to the module
+name, which sometimes resulted in an incorrect full name being displayed.
+
+(This issue impacted only the C code exception rendering, the
+:mod:`traceback` module was using qualname already).
+
+..
+
+.. bpo: 45056
+.. date: 2021-09-01-16-55-43
+.. nonce: 7AK2d9
+.. section: Core and Builtins
+
+Compiler now removes trailing unused constants from co_consts.
+
+..
+
+.. bpo: 27946
+.. date: 2021-12-04-20-08-42
+.. nonce: -Vuarf
+.. section: Library
+
+Fix possible crash when getting an attribute of
+class:`xml.etree.ElementTree.Element` simultaneously with replacing the
+``attrib`` dict.
+
+..
+
+.. bpo: 37658
+.. date: 2021-11-28-15-30-34
+.. nonce: 8Hno7d
+.. section: Library
+
+Fix issue when on certain conditions ``asyncio.wait_for()`` may allow a
+coroutine to complete successfully, but fail to return the result,
+potentially causing memory leaks or other issues.
+
+..
+
+.. bpo: 44649
+.. date: 2021-11-21-20-50-42
+.. nonce: E8M936
+.. section: Library
+
+Handle dataclass(slots=True) with a field that has default a default value,
+but for which init=False.
+
+..
+
+.. bpo: 45803
+.. date: 2021-11-20-17-04-25
+.. nonce: wSgFOy
+.. section: Library
+
+Added missing kw_only parameter to dataclasses.make_dataclass().
+
+..
+
+.. bpo: 45831
+.. date: 2021-11-17-19-25-37
+.. nonce: 9-TojK
+.. section: Library
+
+:mod:`faulthandler` can now write ASCII-only strings (like filenames and
+function names) with a single write() syscall when dumping a traceback. It
+reduces the risk of getting an unreadable dump when two threads or two
+processes dump a traceback to the same file (like stderr) at the same time.
+Patch by Victor Stinner.
+
+..
+
+.. bpo: 41735
+.. date: 2021-11-16-18-13-49
+.. nonce: D72UY1
+.. section: Library
+
+Fix thread lock in ``zlib.Decompress.flush()`` method before
+``PyObject_GetBuffer``.
+
+..
+
+.. bpo: 45235
+.. date: 2021-11-11-13-03-17
+.. nonce: 8ZbkHa
+.. section: Library
+
+Reverted an argparse bugfix that caused regression in the handling of
+default arguments for subparsers.  This prevented leaf level arguments from
+taking precedence over root level arguments.
+
+..
+
+.. bpo: 45765
+.. date: 2021-11-09-09-04-19
+.. nonce: JVobxK
+.. section: Library
+
+In importlib.metadata, fix distribution discovery for an empty path.
+
+..
+
+.. bpo: 45757
+.. date: 2021-11-08-23-22-14
+.. nonce: MHZHt3
+.. section: Library
+
+Fix bug where :mod:`dis` produced an incorrect oparg when
+:opcode:`EXTENDED_ARG` is followed by an opcode that does not use its
+argument.
+
+..
+
+.. bpo: 45644
+.. date: 2021-11-06-17-47-46
+.. nonce: ZMqHD_
+.. section: Library
+
+In-place JSON file formatting using ``python3 -m json.tool infile infile``
+now works correctly, previously it left the file empty.  Patch by Chris
+Wesseling.
+
+..
+
+.. bpo: 45679
+.. date: 2021-10-30-21-11-37
+.. nonce: Dq8Cpu
+.. section: Library
+
+Fix caching of multi-value :data:`typing.Literal`. ``Literal[True, 2]`` is
+no longer equal to ``Literal[1, 2]``.
+
+..
+
+.. bpo: 45664
+.. date: 2021-10-28-23-40-54
+.. nonce: 7dqtxQ
+.. section: Library
+
+Fix :func:`types.resolve_bases` and :func:`types.new_class` for
+:class:`types.GenericAlias` instance as a base.
+
+..
+
+.. bpo: 45663
+.. date: 2021-10-28-23-11-59
+.. nonce: J90N5R
+.. section: Library
+
+Fix :func:`dataclasses.is_dataclass` for dataclasses which are subclasses of
+:class:`types.GenericAlias`.
+
+..
+
+.. bpo: 45662
+.. date: 2021-10-28-22-58-14
+.. nonce: sJd7Ir
+.. section: Library
+
+Fix the repr of :data:`dataclasses.InitVar` with a type alias to the
+built-in class, e.g. ``InitVar[list[int]]``.
+
+..
+
+.. bpo: 45438
+.. date: 2021-10-27-10-05-39
+.. nonce: Xz5lGU
+.. section: Library
+
+Fix typing.Signature string representation for generic builtin types.
+
+..
+
+.. bpo: 45574
+.. date: 2021-10-22-23-06-33
+.. nonce: svqA84
+.. section: Library
+
+Fix warning about ``print_escape`` being unused.
+
+..
+
+.. bpo: 45581
+.. date: 2021-10-22-21-57-02
+.. nonce: rlH6ay
+.. section: Library
+
+:meth:`sqlite3.connect` now correctly raises :exc:`MemoryError` if the
+underlying SQLite API signals memory error. Patch by Erlend E. Aasland.
+
+..
+
+.. bpo: 45557
+.. date: 2021-10-21-16-18-51
+.. nonce: 4MQt4r
+.. section: Library
+
+pprint.pprint() now handles underscore_numbers correctly. Previously it was
+always setting it to False.
+
+..
+
+.. bpo: 45515
+.. date: 2021-10-18-14-52-48
+.. nonce: aXdvm_
+.. section: Library
+
+Add references to :mod:`zoneinfo` in the :mod:`datetime` documentation,
+mostly replacing outdated references to ``dateutil.tz``. Change by Paul
+Ganssle.
+
+..
+
+.. bpo: 45475
+.. date: 2021-10-18-10-46-47
+.. nonce: sb9KDF
+.. section: Library
+
+Reverted optimization of iterating :class:`gzip.GzipFile`,
+:class:`bz2.BZ2File`, and :class:`lzma.LZMAFile` (see bpo-43787) because it
+caused regression when user iterate them without having reference of them.
+Patch by Inada Naoki.
+
+..
+
+.. bpo: 45428
+.. date: 2021-10-14-18-04-17
+.. nonce: mM2War
+.. section: Library
+
+Fix a regression in py_compile when reading filenames from standard input.
+
+..
+
+.. bpo: 45467
+.. date: 2021-10-14-13-31-19
+.. nonce: Q7Ma6A
+.. section: Library
+
+Fix incremental decoder and stream reader in the "raw-unicode-escape" codec.
+Previously they failed if the escape sequence was split.
+
+..
+
+.. bpo: 45461
+.. date: 2021-10-14-00-19-02
+.. nonce: 4LB_tJ
+.. section: Library
+
+Fix incremental decoder and stream reader in the "unicode-escape" codec.
+Previously they failed if the escape sequence was split.
+
+..
+
+.. bpo: 45239
+.. date: 2021-10-13-17-52-48
+.. nonce: 7li1_0
+.. section: Library
+
+Fixed :func:`email.utils.parsedate_tz` crashing with
+:exc:`UnboundLocalError` on certain invalid input instead of returning
+``None``. Patch by Ben Hoyt.
+
+..
+
+.. bpo: 45249
+.. date: 2021-10-10-16-14-33
+.. nonce: xqLliz
+.. section: Library
+
+Fix the behaviour of :func:`traceback.print_exc` when displaying the caret
+when the ``end_offset`` in the exception is set to 0. Patch by Pablo Galindo
+
+..
+
+.. bpo: 45416
+.. date: 2021-10-10-09-42-34
+.. nonce: n35O0_
+.. section: Library
+
+Fix use of :class:`asyncio.Condition` with explicit :class:`asyncio.Lock`
+objects, which was a regression due to removal of explicit loop arguments.
+Patch by Joongi Kim.
+
+..
+
+.. bpo: 45419
+.. date: 2021-10-09-20-53-13
+.. nonce: CauCgt
+.. section: Library
+
+Correct interfaces on DegenerateFiles.Path.
+
+..
+
+.. bpo: 44904
+.. date: 2021-10-09-18-42-27
+.. nonce: RlW5h8
+.. section: Library
+
+Fix bug in the :mod:`doctest` module that caused it to fail if a docstring
+included an example with a ``classmethod`` ``property``. Patch by Alex
+Waygood.
+
+..
+
+.. bpo: 45406
+.. date: 2021-10-08-19-24-48
+.. nonce: Qh_Mz4
+.. section: Library
+
+Make :func:`inspect.getmodule` catch ``FileNotFoundError`` raised by
+:'func:`inspect.getabsfile`, and return ``None`` to indicate that the module
+could not be determined.
+
+..
+
+.. bpo: 45262
+.. date: 2021-10-07-14-04-10
+.. nonce: HqF71Z
+.. section: Library
+
+Prevent use-after-free in asyncio. Make sure the cached running loop holder
+gets cleared on dealloc to prevent use-after-free in get_running_loop
+
+..
+
+.. bpo: 45386
+.. date: 2021-10-07-00-05-05
+.. nonce: q9ORpA
+.. section: Library
+
+Make :mod:`xmlrpc.client` more robust to C runtimes where the underlying C
+``strftime`` function results in a ``ValueError`` when testing for year
+formatting options.
+
+..
+
+.. bpo: 45371
+.. date: 2021-10-05-11-03-48
+.. nonce: NOwcDJ
+.. section: Library
+
+Fix clang rpath issue in :mod:`distutils`. The UnixCCompiler now uses
+correct clang option to add a runtime library directory (rpath) to a shared
+library.
+
+..
+
+.. bpo: 20028
+.. date: 2021-10-03-21-14-37
+.. nonce: zBA4RK
+.. section: Library
+
+Improve error message of :class:`csv.Dialect` when initializing. Patch by
+Vajrasky Kok and Dong-hee Na.
+
+..
+
+.. bpo: 45343
+.. date: 2021-10-01-23-07-02
+.. nonce: ixmctD
+.. section: Library
+
+Update bundled pip to 21.2.4 and setuptools to 58.1.0
+
+..
+
+.. bpo: 45329
+.. date: 2021-10-01-13-09-53
+.. nonce: 9iMYaO
+.. section: Library
+
+Fix freed memory access in :class:`pyexpat.xmlparser` when building it with
+an installed expat library <= 2.2.0.
+
+..
+
+.. bpo: 41710
+.. date: 2021-09-30-23-00-18
+.. nonce: svuloZ
+.. section: Library
+
+On Unix, if the ``sem_clockwait()`` function is available in the C library
+(glibc 2.30 and newer), the :meth:`threading.Lock.acquire` method now uses
+the monotonic clock (:data:`time.CLOCK_MONOTONIC`) for the timeout, rather
+than using the system clock (:data:`time.CLOCK_REALTIME`), to not be
+affected by system clock changes. Patch by Victor Stinner.
+
+..
+
+.. bpo: 45328
+.. date: 2021-09-30-08-22-44
+.. nonce: 8Z-Q0B
+.. section: Library
+
+Fixed :class:`http.client.HTTPConnection` to work properly in OSs that don't
+support the ``TCP_NODELAY`` socket option.
+
+..
+
+.. bpo: 1596321
+.. date: 2021-09-24-17-20-23
+.. nonce: 3nhPUk
+.. section: Library
+
+Fix the :func:`threading._shutdown` function when the :mod:`threading`
+module was imported first from a thread different than the main thread: no
+longer log an error at Python exit.
+
+..
+
+.. bpo: 45274
+.. date: 2021-09-23-22-17-26
+.. nonce: gPpa4E
+.. section: Library
+
+Fix a race condition in the :meth:`Thread.join() <threading.Thread.join>`
+method of the :mod:`threading` module. If the function is interrupted by a
+signal and the signal handler raises an exception, make sure that the thread
+remains in a consistent state to prevent a deadlock. Patch by Victor
+Stinner.
+
+..
+
+.. bpo: 45238
+.. date: 2021-09-18-16-56-33
+.. nonce: Hng_9V
+.. section: Library
+
+Fix :meth:`unittest.IsolatedAsyncioTestCase.debug`: it runs now asynchronous
+methods and callbacks.
+
+..
+
+.. bpo: 36674
+.. date: 2021-09-18-13-14-57
+.. nonce: a2k5Zb
+.. section: Library
+
+:meth:`unittest.TestCase.debug` raises now a :class:`unittest.SkipTest` if
+the class or the test method are decorated with the skipping decorator.
+
+..
+
+.. bpo: 45235
+.. date: 2021-09-17-16-55-37
+.. nonce: sXnmPA
+.. section: Library
+
+Fix an issue where argparse would not preserve values in a provided
+namespace when using a subparser with defaults.
+
+..
+
+.. bpo: 45183
+.. date: 2021-09-17-15-58-53
+.. nonce: Vv_vch
+.. section: Library
+
+Have zipimport.zipimporter.find_spec() not raise an exception when the
+underlying zip file has been deleted and the internal cache has been reset
+via invalidate_cache().
+
+..
+
+.. bpo: 45234
+.. date: 2021-09-17-11-20-55
+.. nonce: qUcTVt
+.. section: Library
+
+Fixed a regression in :func:`~shutil.copyfile`, :func:`~shutil.copy`,
+:func:`~shutil.copy2` raising :exc:`FileNotFoundError` when source is a
+directory, which should raise :exc:`IsADirectoryError`
+
+..
+
+.. bpo: 45228
+.. date: 2021-09-17-09-59-33
+.. nonce: WV1dcT
+.. section: Library
+
+Fix stack buffer overflow in parsing J1939 network address.
+
+..
+
+.. bpo: 45192
+.. date: 2021-09-14-15-52-47
+.. nonce: DjA-BI
+.. section: Library
+
+Fix the ``tempfile._infer_return_type`` function so that the ``dir``
+argument of the :mod:`tempfile` functions accepts an object implementing the
+``os.PathLike`` protocol.
+
+Patch by Kyungmin Lee.
+
+..
+
+.. bpo: 42135
+.. date: 2021-09-13-19-32-58
+.. nonce: 1ZAHqR
+.. section: Library
+
+Fix typo: ``importlib.find_loader`` is really slated for removal in Python
+3.12 not 3.10, like the others in GH-25169.
+
+Patch by Hugo van Kemenade.
+
+..
+
+.. bpo: 45160
+.. date: 2021-09-11-14-47-05
+.. nonce: VzMXbW
+.. section: Library
+
+When tracing a tkinter variable used by a ttk OptionMenu, callbacks are no
+longer made twice.
+
+..
+
+.. bpo: 35474
+.. date: 2021-09-11-10-45-12
+.. nonce: tEY3SD
+.. section: Library
+
+Calling :func:`mimetypes.guess_all_extensions` with ``strict=False`` no
+longer affects the result of the following call with ``strict=True``. Also,
+mutating the returned list no longer affects the global state.
+
+..
+
+.. bpo: 45166
+.. date: 2021-09-10-21-35-53
+.. nonce: UHipXF
+.. section: Library
+
+:func:`typing.get_type_hints` now works with :data:`~typing.Final` wrapped
+in :class:`~typing.ForwardRef`.
+
+..
+
+.. bpo: 20499
+.. date: 2021-09-08-01-19-31
+.. nonce: tSxx8Y
+.. section: Library
+
+Improve the speed and accuracy of statistics.pvariance().
+
+..
+
+.. bpo: 24444
+.. date: 2021-08-30-00-19-23
+.. nonce: Ki4bgz
+.. section: Library
+
+Fixed an error raised in :mod:`argparse` help display when help for an
+option is set to 1+ blank spaces or when *choices* arg is an empty
+container.
+
+..
+
+.. bpo: 45021
+.. date: 2021-08-28-13-00-12
+.. nonce: rReeaj
+.. section: Library
+
+Fix a potential deadlock at shutdown of forked children when using
+:mod:`concurrent.futures` module
+
+..
+
+.. bpo: 39039
+.. date: 2021-08-18-10-36-14
+.. nonce: A63LYh
+.. section: Library
+
+tarfile.open raises :exc:`~tarfile.ReadError` when a zlib error occurs
+during file extraction.
+
+..
+
+.. bpo: 44594
+.. date: 2021-07-12-10-32-48
+.. nonce: eEa5zi
+.. section: Library
+
+Fix an edge case of :class:`ExitStack` and :class:`AsyncExitStack` exception
+chaining.  They will now match ``with`` block behavior when ``__context__``
+is explicitly set to ``None`` when the exception is in flight.
+
+..
+
+.. bpo: 44295
+.. date: 2021-06-02-16-39-42
+.. nonce: erg01m
+.. section: Library
+
+Ensure deprecation warning from :func:`assertDictContainsSubset` points at
+calling code - by Anthony Sottile.
+
+..
+
+.. bpo: 43498
+.. date: 2021-04-20-14-14-16
+.. nonce: L_Hq-8
+.. section: Library
+
+Avoid a possible *"RuntimeError: dictionary changed size during iteration"*
+when adjusting the process count of :class:`ProcessPoolExecutor`.
+
+..
+
+.. bpo: 45640
+.. date: 2021-11-18-16-44-12
+.. nonce: lSpc2A
+.. section: Documentation
+
+Properly marked-up grammar tokens in the documentation are now clickable and
+take you to the definition of a given piece of grammar.  Patch by Arthur
+Milchior.
+
+..
+
+.. bpo: 45788
+.. date: 2021-11-18-00-07-40
+.. nonce: qibUoB
+.. section: Documentation
+
+Link doc for sys.prefix to sysconfig doc on installation paths.
+
+..
+
+.. bpo: 45772
+.. date: 2021-11-09-13-10-55
+.. nonce: EdrM3t
+.. section: Documentation
+
+``socket.socket`` documentation is corrected to a class from a function.
+
+..
+
+.. bpo: 45392
+.. date: 2021-11-06-10-54-17
+.. nonce: JZnVOz
+.. section: Documentation
+
+Update the docstring of the :class:`type` built-in to remove a redundant
+line and to mention keyword arguments for the constructor.
+
+..
+
+.. bpo: 45726
+.. date: 2021-11-05-12-15-24
+.. nonce: GwRr7e
+.. section: Documentation
+
+Improve documentation for :func:`functools.singledispatch` and
+:class:`functools.singledispatchmethod`.
+
+..
+
+.. bpo: 45680
+.. date: 2021-11-03-14-51-03
+.. nonce: 9_NTFU
+.. section: Documentation
+
+Amend the docs on ``GenericAlias`` objects to clarify that non-container
+classes can also implement ``__class_getitem__``. Patch contributed by Alex
+Waygood.
+
+..
+
+.. bpo: 45655
+.. date: 2021-10-28-19-22-55
+.. nonce: aPYGaS
+.. section: Documentation
+
+Add a new "relevant PEPs" section to the top of the documentation for the
+``typing`` module. Patch by Alex Waygood.
+
+..
+
+.. bpo: 45604
+.. date: 2021-10-26-10-00-45
+.. nonce: Dm-YhV
+.. section: Documentation
+
+Add ``level`` argument to ``multiprocessing.log_to_stderr`` function docs.
+
+..
+
+.. bpo: 45250
+.. date: 2021-10-22-12-09-18
+.. nonce: Iit5-Y
+.. section: Documentation
+
+Update the documentation to note that CPython does not consistently require
+iterators to define ``__iter__``.
+
+..
+
+.. bpo: 45464
+.. date: 2021-10-20-16-26-53
+.. nonce: mOISBs
+.. section: Documentation
+
+Mention in the documentation of :ref:`Built-in Exceptions
+<bltin-exceptions>` that inheriting from multiple exception types in a
+single subclass is not recommended due to possible memory layout
+incompatibility.
+
+..
+
+.. bpo: 45449
+.. date: 2021-10-19-01-41-40
+.. nonce: fjHZJc
+.. section: Documentation
+
+Add note about :pep:`585` in :mod:`collections.abc`.
+
+..
+
+.. bpo: 45516
+.. date: 2021-10-18-20-12-18
+.. nonce: EJh4K8
+.. section: Documentation
+
+Add protocol description to the :class:`importlib.abc.Traversable`
+documentation.
+
+..
+
+.. bpo: 20692
+.. date: 2021-10-13-00-42-54
+.. nonce: K5rGtP
+.. section: Documentation
+
+Add Programming FAQ entry explaining that int literal attribute access
+requires either a space after or parentheses around the literal.
+
+..
+
+.. bpo: 45216
+.. date: 2021-09-18-13-45-19
+.. nonce: o56nyt
+.. section: Documentation
+
+Remove extra documentation listing methods in ``difflib``. It was rendering
+twice in pydoc and was outdated in some places.
+
+..
+
+.. bpo: 45024
+.. date: 2021-09-08-17-20-19
+.. nonce: dkNPNi
+.. section: Documentation
+
+:mod:`collections.abc` documentation has been expanded to explicitly cover
+how instance and subclass checks work, with additional doctest examples and
+an exhaustive list of ABCs which test membership purely by presence of the
+right :term:`special method`\s. Patch by Raymond Hettinger.
+
+..
+
+.. bpo: 25381
+.. date: 2021-06-21-17-51-51
+.. nonce: 7Kn-_H
+.. section: Documentation
+
+In the extending chapter of the extending doc, update a paragraph about the
+global variables containing exception information.
+
+..
+
+.. bpo: 43905
+.. date: 2021-05-24-05-00-12
+.. nonce: tBIndE
+.. section: Documentation
+
+Expanded :func:`~dataclasses.astuple` and :func:`~dataclasses.asdict` docs,
+warning about deepcopy being applied and providing a workaround.
+
+..
+
+.. bpo: 19460
+.. date: 2021-11-28-15-25-02
+.. nonce: lr0aWs
+.. section: Tests
+
+Add new Test for :class:`email.mime.nonmultipart.MIMENonMultipart`.
+
+..
+
+.. bpo: 45835
+.. date: 2021-11-17-14-28-08
+.. nonce: Mgyhjx
+.. section: Tests
+
+Fix race condition in test_queue tests with multiple "feeder" threads.
+
+..
+
+.. bpo: 45678
+.. date: 2021-11-04-20-03-32
+.. nonce: 1xNMjN
+.. section: Tests
+
+Add tests for scenarios in which :class:`functools.singledispatchmethod` is
+stacked on top of a method that has already been wrapped by two other
+decorators. Patch by Alex Waygood.
+
+..
+
+.. bpo: 45578
+.. date: 2021-10-30-19-00-25
+.. nonce: bvu6X2
+.. section: Tests
+
+Add tests for :func:`dis.distb`
+
+..
+
+.. bpo: 45678
+.. date: 2021-10-30-13-12-20
+.. nonce: bKrYeS
+.. section: Tests
+
+Add tests to ensure that ``functools.singledispatchmethod`` correctly wraps
+the attributes of the target function.
+
+..
+
+.. bpo: 45577
+.. date: 2021-10-22-19-44-13
+.. nonce: dSaNvK
+.. section: Tests
+
+Add subtests for all ``pickle`` protocols in ``test_zoneinfo``.
+
+..
+
+.. bpo: 45566
+.. date: 2021-10-22-12-05-21
+.. nonce: 2gQ3ZB
+.. section: Tests
+
+Fix ``test_frozen_pickle`` in ``test_dataclasses`` to check all ``pickle``
+versions.
+
+..
+
+.. bpo: 43592
+.. date: 2021-10-21-17-22-26
+.. nonce: kHRsra
+.. section: Tests
+
+:mod:`test.libregrtest` now raises the soft resource limit for the maximum
+number of file descriptors when the default is too low for our test suite as
+was often the case on macOS.
+
+..
+
+.. bpo: 39679
+.. date: 2021-10-18-16-18-41
+.. nonce: F18qcE
+.. section: Tests
+
+Add more test cases for `@functools.singledispatchmethod` when combined with
+`@classmethod` or `@staticmethod`.
+
+..
+
+.. bpo: 45400
+.. date: 2021-10-07-13-11-45
+.. nonce: h3iT7V
+.. section: Tests
+
+Fix test_name_error_suggestions_do_not_trigger_for_too_many_locals() of
+test_exceptions if a directory name contains "a1" (like "Python-3.11.0a1"):
+use a stricter regular expression. Patch by Victor Stinner.
+
+..
+
+.. bpo: 40173
+.. date: 2021-09-30-16-54-39
+.. nonce: J_slCw
+.. section: Tests
+
+Fix :func:`test.support.import_helper.import_fresh_module`.
+
+..
+
+.. bpo: 45280
+.. date: 2021-09-25-11-05-31
+.. nonce: 3MA6lC
+.. section: Tests
+
+Add a test case for empty :class:`typing.NamedTuple`.
+
+..
+
+.. bpo: 45269
+.. date: 2021-09-24-10-41-49
+.. nonce: 8jKEr8
+.. section: Tests
+
+Cover case when invalid ``markers`` type is supplied to ``c_make_encoder``.
+
+..
+
+.. bpo: 45128
+.. date: 2021-09-16-17-22-35
+.. nonce: Jz6fl2
+.. section: Tests
+
+Fix ``test_multiprocessing_fork`` failure due to ``test_logging`` and
+``sys.modules`` manipulation.
+
+..
+
+.. bpo: 45209
+.. date: 2021-09-15-23-32-39
+.. nonce: 55ntL5
+.. section: Tests
+
+Fix ``UserWarning: resource_tracker`` warning in
+``_test_multiprocessing._TestSharedMemory.test_shared_memory_cleaned_after_process_termination``
+
+..
+
+.. bpo: 45195
+.. date: 2021-09-14-13-16-18
+.. nonce: EyQR1G
+.. section: Tests
+
+Fix test_readline.test_nonascii(): sometimes, the newline character is not
+written at the end, so don't expect it in the output. Patch by Victor
+Stinner.
+
+..
+
+.. bpo: 45156
+.. date: 2021-09-13-00-28-17
+.. nonce: 8oomV3
+.. section: Tests
+
+Fixes infinite loop on :func:`unittest.mock.seal` of mocks created by
+:func:`~unittest.create_autospec`.
+
+..
+
+.. bpo: 45125
+.. date: 2021-09-11-22-08-18
+.. nonce: FVSzs2
+.. section: Tests
+
+Improves pickling tests and docs of ``SharedMemory`` and ``SharableList``
+objects.
+
+..
+
+.. bpo: 44860
+.. date: 2021-09-08-13-01-37
+.. nonce: qXd0kx
+.. section: Tests
+
+Update ``test_sysconfig.test_user_similar()`` for the posix_user scheme:
+``platlib`` doesn't use :data:`sys.platlibdir`. Patch by Victor Stinner.
+
+..
+
+.. bpo: 25130
+.. date: 2021-08-27-22-37-19
+.. nonce: ig4oJe
+.. section: Tests
+
+Add calls of :func:`gc.collect` in tests to support PyPy.
+
+..
+
+.. bpo: 44035
+.. date: 2021-12-06-09-31-27
+.. nonce: BiO4XC
+.. section: Build
+
+CI now verifies that autoconf files have been regenerated with a current and
+unpatched autoconf package.
+
+..
+
+.. bpo: 33393
+.. date: 2021-11-25-20-26-06
+.. nonce: 24YNtM
+.. section: Build
+
+Update ``config.guess`` to 2021-06-03 and ``config.sub`` to 2021-08-14.
+``Makefile`` now has an ``update-config`` target to make updating more
+convenient.
+
+..
+
+.. bpo: 45866
+.. date: 2021-11-25-13-53-36
+.. nonce: ZH1W8N
+.. section: Build
+
+``make regen-all`` now produces the same output when run from a directory
+other than the source tree: when building Python out of the source tree.
+pegen now strips directory of the "generated by pygen from <FILENAME>"
+header Patch by Victor Stinner.
+
+..
+
+.. bpo: 41498
+.. date: 2021-11-25-09-15-04
+.. nonce: qAk5eo
+.. section: Build
+
+Python now compiles on platforms without ``sigset_t``. Several functions in
+:mod:`signal` are not available when ``sigset_t`` is missing.
+
+Based on patch by Roman Yurchak for pyodide.
+
+..
+
+.. bpo: 45881
+.. date: 2021-11-24-17-14-06
+.. nonce: GTXXLk
+.. section: Build
+
+``setup.py`` now uses ``CC`` from environment first to discover multiarch
+and cross compile paths.
+
+..
+
+.. bpo: 43158
+.. date: 2021-11-01-12-51-46
+.. nonce: fghS6w
+.. section: Build
+
+``setup.py`` now uses values from configure script to build the ``_uuid``
+extension module. Configure now detects util-linux's ``libuuid``, too.
+
+..
+
+.. bpo: 45571
+.. date: 2021-10-22-15-28-29
+.. nonce: yY8NsJ
+.. section: Build
+
+``Modules/Setup`` now use ``PY_CFLAGS_NODIST`` instead of ``PY_CFLAGS`` to
+compile shared modules.
+
+..
+
+.. bpo: 45561
+.. date: 2021-10-21-14-38-30
+.. nonce: PVqhZE
+.. section: Build
+
+Run smelly.py tool from $(srcdir).
+
+..
+
+.. bpo: 45532
+.. date: 2021-10-20-16-07-39
+.. nonce: kyhvis
+.. section: Build
+
+Update :data:`sys.version` to use ``main`` as fallback information. Patch by
+Jeong YunWon.
+
+..
+
+.. bpo: 45536
+.. date: 2021-10-20-12-42-39
+.. nonce: oQNYHB
+.. section: Build
+
+The ``configure`` script now checks whether OpenSSL headers and libraries
+provide required APIs. Most common APIs are verified. The check detects
+outdated or missing OpenSSL. Failures do not stop configure.
+
+..
+
+.. bpo: 45221
+.. date: 2021-10-18-10-25-56
+.. nonce: rnulhf
+.. section: Build
+
+Fixed regression in handling of ``LDFLAGS`` and ``CPPFLAGS`` options where
+:meth:`argparse.parse_known_args` could interpret an option as one of the
+built-in command line argument, for example ``-h`` for help.
+
+..
+
+.. bpo: 45405
+.. date: 2021-10-11-16-27-38
+.. nonce: iSfdW5
+.. section: Build
+
+Prevent ``internal configure error`` when running ``configure`` with recent
+versions of non-Apple clang.  Patch by David Bohman.
+
+..
+
+.. bpo: 45220
+.. date: 2021-09-16-18-00-43
+.. nonce: TgbkvW
+.. section: Build
+
+Avoid building with the Windows 11 SDK previews automatically. This may be
+overridden by setting the ``DefaultWindowsSDKVersion`` environment variable
+before building.
+
+..
+
+.. bpo: 45067
+.. date: 2021-09-09-16-45-26
+.. nonce: mFmY92
+.. section: Build
+
+The ncurses function extended_color_content was introduced in 2017
+
+(https://invisible-island.net/ncurses/NEWS.html#index-t20170401).  The
+
+ncurses-devel package in CentOS 7 had a older version ncurses resulted in
+compilation error.  For compiling ncurses with extended color support, we
+verify the version of the ncurses library >= 20170401.
+
+..
+
+.. bpo: 45901
+.. date: 2021-11-26-18-17-41
+.. nonce: c5IBqM
+.. section: Windows
+
+When installed through the Microsoft Store and set as the default app for
+:file:`*.py` files, command line arguments will now be passed to Python when
+invoking a script without explicitly launching Python (that is, ``script.py
+args`` rather than ``python script.py args``).
+
+..
+
+.. bpo: 45616
+.. date: 2021-11-23-11-44-42
+.. nonce: K52PLZ
+.. section: Windows
+
+Fix Python Launcher's ability to distinguish between versions 3.1 and 3.10
+when either one is explicitly requested.  Previously, 3.1 would be used if
+3.10 was requested but not installed, and 3.10 would be used if 3.1 was
+requested but 3.10 was installed.
+
+..
+
+.. bpo: 45732
+.. date: 2021-11-08-21-53-11
+.. nonce: idl5kx
+.. section: Windows
+
+Updates bundled Tcl/Tk to 8.6.12.
+
+..
+
+.. bpo: 45720
+.. date: 2021-11-05-01-05-46
+.. nonce: 47Nc5I
+.. section: Windows
+
+Internal reference to :file:`shlwapi.dll` was dropped to help improve
+startup time. This DLL will no longer be loaded at the start of every Python
+process.
+
+..
+
+.. bpo: 43652
+.. date: 2021-11-04-00-41-50
+.. nonce: RnqV7I
+.. section: Windows
+
+Update Tcl/Tk to 8.6.11, actually this time. The previous update incorrectly
+included 8.6.10.
+
+..
+
+.. bpo: 45337
+.. date: 2021-09-30-23-17-27
+.. nonce: qg7U_h
+.. section: Windows
+
+venv now warns when the created environment may need to be accessed at a
+different path, due to redirections, links or junctions. It also now
+correctly installs or upgrades components when the alternate path is
+required.
+
+..
+
+.. bpo: 45732
+.. date: 2021-12-05-23-52-03
+.. nonce: -BWrnh
+.. section: macOS
+
+Update python.org macOS installer to use Tcl/Tk 8.6.12.
+
+..
+
+.. bpo: 44828
+.. date: 2021-10-25-02-02-21
+.. nonce: XBdXlJ
+.. section: macOS
+
+Avoid tkinter file dialog failure on macOS 12 Monterey when using the Tk
+8.6.11 provided by python.org macOS installers. Patch by Marc Culler of the
+Tk project.
+
+..
+
+.. bpo: 34602
+.. date: 2021-08-27-16-55-10
+.. nonce: ZjHsYJ
+.. section: macOS
+
+When building CPython on macOS with ``./configure
+--with-undefined-behavior-sanitizer --with-pydebug``, the stack size is now
+quadrupled to allow for the entire test suite to pass.
+
+..
+
+.. bpo: 45495
+.. date: 2021-10-16-17-20-32
+.. nonce: ST8RFt
+.. section: IDLE
+
+Add context keywords 'case' and 'match' to completions list.
+
+..
+
+.. bpo: 45296
+.. date: 2021-09-27-01-21-59
+.. nonce: 9H8rdY
+.. section: IDLE
+
+On Windows, change exit/quit message to suggest Ctrl-D, which works, instead
+of <Ctrl-Z Return>, which does not work in IDLE.
+
+..
+
+.. bpo: 45193
+.. date: 2021-09-15-03-20-06
+.. nonce: G61_GV
+.. section: IDLE
+
+Make completion boxes appear on Ubuntu again.
+
+..
+
+.. bpo: 44786
+.. date: 2021-09-14-11-44-26
+.. nonce: DU0LC0
+.. section: Tools/Demos
+
+Fix a warning in regular expression in the c-analyzer script.
+
+..
+
+.. bpo: 39026
+.. date: 2021-11-09-15-42-11
+.. nonce: sUnYWn
+.. section: C API
+
+Fix Python.h to build C extensions with Xcode: remove a relative include
+from ``Include/cpython/pystate.h``.
+
+..
+
+.. bpo: 45307
+.. date: 2021-09-28-12-00-55
+.. nonce: 3ETFfX
+.. section: C API
+
+Restore the private C API function :func:`_PyImport_FindExtensionObject`. It
+will be removed in Python 3.11.
+
+..
+
+.. bpo: 44687
+.. date: 2021-09-19-17-18-25
+.. nonce: 3fqDRC
+.. section: C API
+
+:meth:`BufferedReader.peek` no longer raises :exc:`ValueError` when the
+entire file has already been buffered.
+
+..
+
+.. bpo: 44751
+.. date: 2021-07-27-17-29-12
+.. nonce: 4qmbDG
+.. section: C API
+
+Remove ``crypt.h`` include from the public ``Python.h`` header.
diff --git a/Misc/NEWS.d/next/Build/2021-09-09-16-45-26.bpo-45067.mFmY92.rst b/Misc/NEWS.d/next/Build/2021-09-09-16-45-26.bpo-45067.mFmY92.rst
deleted file mode 100644
index a89736eb33e82..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-09-09-16-45-26.bpo-45067.mFmY92.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-The ncurses function extended_color_content was introduced in 2017
-
-(https://invisible-island.net/ncurses/NEWS.html#index-t20170401).  The
-
-ncurses-devel package in CentOS 7 had a older version ncurses resulted in
-compilation error.  For compiling ncurses with extended color support, we
-verify the version of the ncurses library >= 20170401.
diff --git a/Misc/NEWS.d/next/Build/2021-09-16-18-00-43.bpo-45220.TgbkvW.rst b/Misc/NEWS.d/next/Build/2021-09-16-18-00-43.bpo-45220.TgbkvW.rst
deleted file mode 100644
index 8bbd634fa61a3..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-09-16-18-00-43.bpo-45220.TgbkvW.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Avoid building with the Windows 11 SDK previews automatically. This may be
-overridden by setting the ``DefaultWindowsSDKVersion`` environment variable
-before building.
diff --git a/Misc/NEWS.d/next/Build/2021-10-11-16-27-38.bpo-45405.iSfdW5.rst b/Misc/NEWS.d/next/Build/2021-10-11-16-27-38.bpo-45405.iSfdW5.rst
deleted file mode 100644
index a2dc5bcc32217..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-10-11-16-27-38.bpo-45405.iSfdW5.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Prevent ``internal configure error`` when running ``configure``
-with recent versions of non-Apple clang.  Patch by David Bohman.
diff --git a/Misc/NEWS.d/next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst b/Misc/NEWS.d/next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst
deleted file mode 100644
index cb981d96f3047..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-10-18-10-25-56.bpo-45221.rnulhf.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fixed regression in handling of ``LDFLAGS`` and ``CPPFLAGS`` options
-where :meth:`argparse.parse_known_args` could interpret an option as
-one of the built-in command line argument, for example ``-h`` for help.
diff --git a/Misc/NEWS.d/next/Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst b/Misc/NEWS.d/next/Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst
deleted file mode 100644
index e560b71ede2d2..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-10-20-12-42-39.bpo-45536.oQNYHB.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-The ``configure`` script now checks whether OpenSSL headers and libraries
-provide required APIs. Most common APIs are verified. The check detects
-outdated or missing OpenSSL. Failures do not stop configure.
diff --git a/Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst b/Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst
deleted file mode 100644
index 575e2fb9ae93b..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-10-20-16-07-39.bpo-45532.kyhvis.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Update :data:`sys.version` to use ``main`` as fallback information.
-Patch by Jeong YunWon.
diff --git a/Misc/NEWS.d/next/Build/2021-10-21-14-38-30.bpo-45561.PVqhZE.rst b/Misc/NEWS.d/next/Build/2021-10-21-14-38-30.bpo-45561.PVqhZE.rst
deleted file mode 100644
index cf5d8686b9840..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-10-21-14-38-30.bpo-45561.PVqhZE.rst
+++ /dev/null
@@ -1 +0,0 @@
-Run smelly.py tool from $(srcdir).
diff --git a/Misc/NEWS.d/next/Build/2021-10-22-15-28-29.bpo-45571.yY8NsJ.rst b/Misc/NEWS.d/next/Build/2021-10-22-15-28-29.bpo-45571.yY8NsJ.rst
deleted file mode 100644
index f2042d11e24b7..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-10-22-15-28-29.bpo-45571.yY8NsJ.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-``Modules/Setup`` now use ``PY_CFLAGS_NODIST`` instead of ``PY_CFLAGS`` to
-compile shared modules.
diff --git a/Misc/NEWS.d/next/Build/2021-11-01-12-51-46.bpo-43158.fghS6w.rst b/Misc/NEWS.d/next/Build/2021-11-01-12-51-46.bpo-43158.fghS6w.rst
deleted file mode 100644
index 3b1de478e7fd4..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-11-01-12-51-46.bpo-43158.fghS6w.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-``setup.py`` now uses values from configure script to build the ``_uuid``
-extension module. Configure now detects util-linux's ``libuuid``, too.
diff --git a/Misc/NEWS.d/next/Build/2021-11-24-17-14-06.bpo-45881.GTXXLk.rst b/Misc/NEWS.d/next/Build/2021-11-24-17-14-06.bpo-45881.GTXXLk.rst
deleted file mode 100644
index b697658cf3aaa..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-11-24-17-14-06.bpo-45881.GTXXLk.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-``setup.py`` now uses ``CC`` from environment first to discover multiarch
-and cross compile paths.
diff --git a/Misc/NEWS.d/next/Build/2021-11-25-09-15-04.bpo-41498.qAk5eo.rst b/Misc/NEWS.d/next/Build/2021-11-25-09-15-04.bpo-41498.qAk5eo.rst
deleted file mode 100644
index 18dc290dafd02..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-11-25-09-15-04.bpo-41498.qAk5eo.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-Python now compiles on platforms without ``sigset_t``. Several functions
-in :mod:`signal` are not available when ``sigset_t`` is missing.
-
-Based on patch by Roman Yurchak for pyodide.
diff --git a/Misc/NEWS.d/next/Build/2021-11-25-13-53-36.bpo-45866.ZH1W8N.rst b/Misc/NEWS.d/next/Build/2021-11-25-13-53-36.bpo-45866.ZH1W8N.rst
deleted file mode 100644
index e87b93932ffa1..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-11-25-13-53-36.bpo-45866.ZH1W8N.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-``make regen-all`` now produces the same output when run from a directory
-other than the source tree: when building Python out of the source tree.
-pegen now strips directory of the "generated by pygen from <FILENAME>" header
-Patch by Victor Stinner.
diff --git a/Misc/NEWS.d/next/Build/2021-11-25-20-26-06.bpo-33393.24YNtM.rst b/Misc/NEWS.d/next/Build/2021-11-25-20-26-06.bpo-33393.24YNtM.rst
deleted file mode 100644
index c27869c9b6ded..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-11-25-20-26-06.bpo-33393.24YNtM.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Update ``config.guess`` to 2021-06-03 and ``config.sub`` to 2021-08-14.
-``Makefile`` now has an ``update-config`` target to make updating more
-convenient.
diff --git a/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst b/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
deleted file mode 100644
index 7530587b73d14..0000000000000
--- a/Misc/NEWS.d/next/Build/2021-12-06-09-31-27.bpo-44035.BiO4XC.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-CI now verifies that autoconf files have been regenerated with a current and
-unpatched autoconf package.
diff --git a/Misc/NEWS.d/next/C API/2021-07-27-17-29-12.bpo-44751.4qmbDG.rst b/Misc/NEWS.d/next/C API/2021-07-27-17-29-12.bpo-44751.4qmbDG.rst
deleted file mode 100644
index d7b9f09819669..0000000000000
--- a/Misc/NEWS.d/next/C API/2021-07-27-17-29-12.bpo-44751.4qmbDG.rst	
+++ /dev/null
@@ -1 +0,0 @@
-Remove ``crypt.h`` include from the public ``Python.h`` header.
diff --git a/Misc/NEWS.d/next/C API/2021-09-19-17-18-25.bpo-44687.3fqDRC.rst b/Misc/NEWS.d/next/C API/2021-09-19-17-18-25.bpo-44687.3fqDRC.rst
deleted file mode 100644
index d38fa6057f6f9..0000000000000
--- a/Misc/NEWS.d/next/C API/2021-09-19-17-18-25.bpo-44687.3fqDRC.rst	
+++ /dev/null
@@ -1 +0,0 @@
-:meth:`BufferedReader.peek` no longer raises :exc:`ValueError` when the entire file has already been buffered.
diff --git a/Misc/NEWS.d/next/C API/2021-09-28-12-00-55.bpo-45307.3ETFfX.rst b/Misc/NEWS.d/next/C API/2021-09-28-12-00-55.bpo-45307.3ETFfX.rst
deleted file mode 100644
index aa2bd7af54d26..0000000000000
--- a/Misc/NEWS.d/next/C API/2021-09-28-12-00-55.bpo-45307.3ETFfX.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Restore the private C API function :func:`_PyImport_FindExtensionObject`. It
-will be removed in Python 3.11.
diff --git a/Misc/NEWS.d/next/C API/2021-11-09-15-42-11.bpo-39026.sUnYWn.rst b/Misc/NEWS.d/next/C API/2021-11-09-15-42-11.bpo-39026.sUnYWn.rst
deleted file mode 100644
index 77a0119792152..0000000000000
--- a/Misc/NEWS.d/next/C API/2021-11-09-15-42-11.bpo-39026.sUnYWn.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix Python.h to build C extensions with Xcode: remove a relative include
-from ``Include/cpython/pystate.h``.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-01-16-55-43.bpo-45056.7AK2d9.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-01-16-55-43.bpo-45056.7AK2d9.rst
deleted file mode 100644
index 6c790f5c50c48..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-09-01-16-55-43.bpo-45056.7AK2d9.rst	
+++ /dev/null
@@ -1 +0,0 @@
-Compiler now removes trailing unused constants from co_consts.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-01-23-55-49.bpo-45083.cLi9G3.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-01-23-55-49.bpo-45083.cLi9G3.rst
deleted file mode 100644
index 7bfd87b942059..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-09-01-23-55-49.bpo-45083.cLi9G3.rst	
+++ /dev/null
@@ -1,3 +0,0 @@
-When the interpreter renders an exception, its name now has a complete qualname. Previously only the class name was concatenated to the module name, which sometimes resulted in an incorrect full name being displayed.
-
-(This issue impacted only the C code exception rendering, the :mod:`traceback` module was using qualname already).
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-07-17-10-16.bpo-45121.iG-Hsf.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-07-17-10-16.bpo-45121.iG-Hsf.rst
deleted file mode 100644
index 19eb331412516..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-09-07-17-10-16.bpo-45121.iG-Hsf.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix issue where ``Protocol.__init__`` raises ``RecursionError`` when it's
-called directly or via ``super()``. Patch provided by Yurii Karabas.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-08-00-30-09.bpo-44050.mFI15u.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-08-00-30-09.bpo-44050.mFI15u.rst
deleted file mode 100644
index d6eed9f1bcfe9..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-09-08-00-30-09.bpo-44050.mFI15u.rst	
+++ /dev/null
@@ -1,3 +0,0 @@
-Extensions that indicate they use global state (by setting ``m_size`` to -1)
-can again be used in multiple interpreters. This reverts to behavior of
-Python 3.8.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-08-08-29-41.bpo-44959.OSwwPf.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-08-08-29-41.bpo-44959.OSwwPf.rst
deleted file mode 100644
index 02e11ae94e430..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-09-08-08-29-41.bpo-44959.OSwwPf.rst	
+++ /dev/null
@@ -1 +0,0 @@
-Added fallback to extension modules with '.sl' suffix on HP-UX
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-09-10-32-33.bpo-44219.WiYyjz.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-09-10-32-33.bpo-44219.WiYyjz.rst
deleted file mode 100644
index 2abd81673663b..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-09-09-10-32-33.bpo-44219.WiYyjz.rst	
+++ /dev/null
@@ -1,5 +0,0 @@
-Release the GIL while performing ``isatty`` system calls on arbitrary file
-descriptors. In particular, this affects :func:`os.isatty`,
-:func:`os.device_encoding` and :class:`io.TextIOWrapper`. By extension,
-:func:`io.open` in text mode is also affected. This change solves
-a deadlock in :func:`os.isatty`. Patch by Vincent Michel in :issue:`44219`.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-09-14-09-23-59.bpo-45167.CPSSoV.rst b/Misc/NEWS.d/next/Core and Builtins/2021-09-14-09-23-59.bpo-45167.CPSSoV.rst
deleted file mode 100644
index 47755ae59be2b..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-09-14-09-23-59.bpo-45167.CPSSoV.rst	
+++ /dev/null
@@ -1 +0,0 @@
-Fix deepcopying of :class:`types.GenericAlias` objects.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-06-21-20-11.bpo-45385.CTUT8s.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-06-21-20-11.bpo-45385.CTUT8s.rst
deleted file mode 100644
index 8047c102634f0..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-10-06-21-20-11.bpo-45385.CTUT8s.rst	
+++ /dev/null
@@ -1 +0,0 @@
-Fix reference leak from descr_check. Patch by Dong-hee Na.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-07-21-26-44.bpo-45408.qUqzcd.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-07-21-26-44.bpo-45408.qUqzcd.rst
deleted file mode 100644
index e4d4db9cb9536..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-10-07-21-26-44.bpo-45408.qUqzcd.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix a crash in the parser when reporting tokenizer errors that occur at the
-same time unclosed parentheses are detected. Patch by Pablo Galindo.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-16-17-27-48.bpo-45494.vMt1g4.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-16-17-27-48.bpo-45494.vMt1g4.rst
deleted file mode 100644
index 97e29813ab266..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-10-16-17-27-48.bpo-45494.vMt1g4.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix parser crash when reporting errors involving invalid continuation
-characters. Patch by Pablo Galindo.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-18-22-40-33.bpo-45521.GdMiuW.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-18-22-40-33.bpo-45521.GdMiuW.rst
deleted file mode 100644
index 3a082a4ffdbcb..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-10-18-22-40-33.bpo-45521.GdMiuW.rst	
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix a bug in the obmalloc radix tree code.  On 64-bit machines, the bug
-causes the tree to hold 46-bits of virtual addresses, rather than the
-intended 48-bits.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-19-01-04-08.bpo-30570._G30Ms.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-19-01-04-08.bpo-30570._G30Ms.rst
deleted file mode 100644
index d9ab60cd08f56..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-10-19-01-04-08.bpo-30570._G30Ms.rst	
+++ /dev/null
@@ -1 +0,0 @@
-Fixed a crash in ``issubclass()`` from infinite recursion when searching pathological ``__bases__`` tuples.
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-02-09-27-46.bpo-45688.v5Der1.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-02-09-27-46.bpo-45688.v5Der1.rst
deleted file mode 100644
index afd73a1266279..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-02-09-27-46.bpo-45688.v5Der1.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-:data:`sys.stdlib_module_names` now contains the macOS-specific module
-:mod:`_scproxy`.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-04-20-19-07.bpo-45716.5C0pA1.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-04-20-19-07.bpo-45716.5C0pA1.rst
deleted file mode 100644
index 682900c295416..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-04-20-19-07.bpo-45716.5C0pA1.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Improve the :exc:`SyntaxError` message when using ``True``, ``None`` or
-``False`` as keywords in a function call. Patch by Pablo Galindo.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-09-13-01-35.bpo-45773.POU8A4.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-09-13-01-35.bpo-45773.POU8A4.rst
deleted file mode 100644
index 2b9ba81be233f..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-09-13-01-35.bpo-45773.POU8A4.rst	
+++ /dev/null
@@ -1 +0,0 @@
-Fix a compiler hang when attempting to optimize certain jump patterns.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-14-00-14-45.bpo-45738.e0cgKd.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-14-00-14-45.bpo-45738.e0cgKd.rst
deleted file mode 100644
index b238034323c77..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-14-00-14-45.bpo-45738.e0cgKd.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix computation of error location for invalid continuation characters in the
-parser. Patch by Pablo Galindo.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-15-12-08-27.bpo-42540.V2w107.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-15-12-08-27.bpo-42540.V2w107.rst
deleted file mode 100644
index 91160598bd3f4..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-15-12-08-27.bpo-42540.V2w107.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix crash when :func:`os.fork` is called with an active non-default
-memory allocator.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-16-19-00-27.bpo-45820.2X6Psr.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-16-19-00-27.bpo-45820.2X6Psr.rst
deleted file mode 100644
index c2ec3d690cd4b..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-16-19-00-27.bpo-45820.2X6Psr.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix a segfault when the parser fails without reading any input. Patch by
-Pablo Galindo
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-16-19-41-04.bpo-45822.OT6ueS.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-16-19-41-04.bpo-45822.OT6ueS.rst
deleted file mode 100644
index 1ac7a8becee40..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-16-19-41-04.bpo-45822.OT6ueS.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Fixed a bug in the parser that was causing it to not respect :pep:`263`
-coding cookies when no flags are provided. Patch by Pablo Galindo
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-17-08-05-27.bpo-45826.OERoTm.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-17-08-05-27.bpo-45826.OERoTm.rst
deleted file mode 100644
index f04373bf2fc59..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-17-08-05-27.bpo-45826.OERoTm.rst	
+++ /dev/null
@@ -1 +0,0 @@
-Fixed a crash when calling ``.with_traceback(None)`` on ``NameError``. This occurs internally in ``unittest.TestCase.assertRaises()``.
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-19-22-57-42.bpo-45848.HgVBJ5.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-19-22-57-42.bpo-45848.HgVBJ5.rst
deleted file mode 100644
index d9394c9c1f08b..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-19-22-57-42.bpo-45848.HgVBJ5.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Allow the parser to obtain error lines directly from encoded files. Patch by
-Pablo Galindo
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-23-12-06-41.bpo-45614.fIekgI.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-23-12-06-41.bpo-45614.fIekgI.rst
deleted file mode 100644
index 4255e1885ad67..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-23-12-06-41.bpo-45614.fIekgI.rst	
+++ /dev/null
@@ -1 +0,0 @@
-Fix :mod:`traceback` display for exceptions with invalid module name.
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-24-18-24-49.bpo-45727._xVbbo.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-24-18-24-49.bpo-45727._xVbbo.rst
deleted file mode 100644
index d4b149ddccf5d..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-24-18-24-49.bpo-45727._xVbbo.rst	
+++ /dev/null
@@ -1,3 +0,0 @@
-Refine the custom syntax error that suggests that a comma may be missing to
-trigger only when the expressions are detected between parentheses or
-brackets. Patch by Pablo Galindo
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-26-22-31-22.bpo-42268.3wl-09.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-26-22-31-22.bpo-42268.3wl-09.rst
deleted file mode 100644
index 615bbb22ae2eb..0000000000000
--- a/Misc/NEWS.d/next/Core and Builtins/2021-11-26-22-31-22.bpo-42268.3wl-09.rst	
+++ /dev/null
@@ -1,2 +0,0 @@
-Fail the configure step if the selected compiler doesn't support memory
-sanitizer. Patch by Pablo Galindo
diff --git a/Misc/NEWS.d/next/Documentation/2021-05-24-05-00-12.bpo-43905.tBIndE.rst b/Misc/NEWS.d/next/Documentation/2021-05-24-05-00-12.bpo-43905.tBIndE.rst
deleted file mode 100644
index 760e1eea0deb7..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-05-24-05-00-12.bpo-43905.tBIndE.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Expanded :func:`~dataclasses.astuple` and :func:`~dataclasses.asdict` docs,
-warning about deepcopy being applied and providing a workaround.
diff --git a/Misc/NEWS.d/next/Documentation/2021-06-21-17-51-51.bpo-25381.7Kn-_H.rst b/Misc/NEWS.d/next/Documentation/2021-06-21-17-51-51.bpo-25381.7Kn-_H.rst
deleted file mode 100644
index f009f880e917d..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-06-21-17-51-51.bpo-25381.7Kn-_H.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-In the extending chapter of the extending doc, update a paragraph about the
-global variables containing exception information.
diff --git a/Misc/NEWS.d/next/Documentation/2021-09-08-17-20-19.bpo-45024.dkNPNi.rst b/Misc/NEWS.d/next/Documentation/2021-09-08-17-20-19.bpo-45024.dkNPNi.rst
deleted file mode 100644
index e73d52b8cc514..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-09-08-17-20-19.bpo-45024.dkNPNi.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-:mod:`collections.abc` documentation has been expanded to explicitly cover
-how instance and subclass checks work, with additional doctest examples and
-an exhaustive list of ABCs which test membership purely by presence of the
-right :term:`special method`\s. Patch by Raymond Hettinger.
diff --git a/Misc/NEWS.d/next/Documentation/2021-09-18-13-45-19.bpo-45216.o56nyt.rst b/Misc/NEWS.d/next/Documentation/2021-09-18-13-45-19.bpo-45216.o56nyt.rst
deleted file mode 100644
index d10b18ecdb8fd..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-09-18-13-45-19.bpo-45216.o56nyt.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Remove extra documentation listing methods in ``difflib``. It was rendering
-twice in pydoc and was outdated in some places.
diff --git a/Misc/NEWS.d/next/Documentation/2021-10-13-00-42-54.bpo-20692.K5rGtP.rst b/Misc/NEWS.d/next/Documentation/2021-10-13-00-42-54.bpo-20692.K5rGtP.rst
deleted file mode 100644
index 44ae468d1bccf..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-10-13-00-42-54.bpo-20692.K5rGtP.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Add Programming FAQ entry explaining that int literal attribute access
-requires either a space after or parentheses around the literal.
diff --git a/Misc/NEWS.d/next/Documentation/2021-10-18-20-12-18.bpo-45516.EJh4K8.rst b/Misc/NEWS.d/next/Documentation/2021-10-18-20-12-18.bpo-45516.EJh4K8.rst
deleted file mode 100644
index 98f5d3432db05..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-10-18-20-12-18.bpo-45516.EJh4K8.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Add protocol description to the :class:`importlib.abc.Traversable`
-documentation.
diff --git a/Misc/NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst b/Misc/NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst
deleted file mode 100644
index fb817757a1487..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-10-19-01-41-40.bpo-45449.fjHZJc.rst
+++ /dev/null
@@ -1 +0,0 @@
-Add note about :pep:`585` in :mod:`collections.abc`.
diff --git a/Misc/NEWS.d/next/Documentation/2021-10-20-16-26-53.bpo-45464.mOISBs.rst b/Misc/NEWS.d/next/Documentation/2021-10-20-16-26-53.bpo-45464.mOISBs.rst
deleted file mode 100644
index 1721aa2c2dfc4..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-10-20-16-26-53.bpo-45464.mOISBs.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-Mention in the documentation of :ref:`Built-in Exceptions
-<bltin-exceptions>` that inheriting from multiple exception types in a
-single subclass is not recommended due to possible memory layout
-incompatibility.
diff --git a/Misc/NEWS.d/next/Documentation/2021-10-22-12-09-18.bpo-45250.Iit5-Y.rst b/Misc/NEWS.d/next/Documentation/2021-10-22-12-09-18.bpo-45250.Iit5-Y.rst
deleted file mode 100644
index 0c2bf18b20010..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-10-22-12-09-18.bpo-45250.Iit5-Y.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Update the documentation to note that CPython does not consistently
-require iterators to define ``__iter__``.
diff --git a/Misc/NEWS.d/next/Documentation/2021-10-26-10-00-45.bpo-45604.Dm-YhV.rst b/Misc/NEWS.d/next/Documentation/2021-10-26-10-00-45.bpo-45604.Dm-YhV.rst
deleted file mode 100644
index 9da9cca7bf1ea..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-10-26-10-00-45.bpo-45604.Dm-YhV.rst
+++ /dev/null
@@ -1 +0,0 @@
-Add ``level`` argument to ``multiprocessing.log_to_stderr`` function docs.
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Documentation/2021-10-28-19-22-55.bpo-45655.aPYGaS.rst b/Misc/NEWS.d/next/Documentation/2021-10-28-19-22-55.bpo-45655.aPYGaS.rst
deleted file mode 100644
index fc5b3d0788817..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-10-28-19-22-55.bpo-45655.aPYGaS.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Add a new "relevant PEPs" section to the top of the documentation for the
-``typing`` module. Patch by Alex Waygood.
diff --git a/Misc/NEWS.d/next/Documentation/2021-11-03-14-51-03.bpo-45680.9_NTFU.rst b/Misc/NEWS.d/next/Documentation/2021-11-03-14-51-03.bpo-45680.9_NTFU.rst
deleted file mode 100644
index 79ea6e308e54a..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-11-03-14-51-03.bpo-45680.9_NTFU.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Amend the docs on ``GenericAlias`` objects to clarify that non-container
-classes can also implement ``__class_getitem__``. Patch contributed by Alex
-Waygood.
diff --git a/Misc/NEWS.d/next/Documentation/2021-11-05-12-15-24.bpo-45726.GwRr7e.rst b/Misc/NEWS.d/next/Documentation/2021-11-05-12-15-24.bpo-45726.GwRr7e.rst
deleted file mode 100644
index 7ff6283a2a34d..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-11-05-12-15-24.bpo-45726.GwRr7e.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Improve documentation for :func:`functools.singledispatch` and
-:class:`functools.singledispatchmethod`.
diff --git a/Misc/NEWS.d/next/Documentation/2021-11-06-10-54-17.bpo-45392.JZnVOz.rst b/Misc/NEWS.d/next/Documentation/2021-11-06-10-54-17.bpo-45392.JZnVOz.rst
deleted file mode 100644
index 09c16578ff478..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-11-06-10-54-17.bpo-45392.JZnVOz.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Update the docstring of the :class:`type` built-in to remove a redundant
-line and to mention keyword arguments for the constructor.
diff --git a/Misc/NEWS.d/next/Documentation/2021-11-09-13-10-55.bpo-45772.EdrM3t.rst b/Misc/NEWS.d/next/Documentation/2021-11-09-13-10-55.bpo-45772.EdrM3t.rst
deleted file mode 100644
index 47679521df30b..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-11-09-13-10-55.bpo-45772.EdrM3t.rst
+++ /dev/null
@@ -1 +0,0 @@
-``socket.socket`` documentation is corrected to a class from a function.
diff --git a/Misc/NEWS.d/next/Documentation/2021-11-18-00-07-40.bpo-45788.qibUoB.rst b/Misc/NEWS.d/next/Documentation/2021-11-18-00-07-40.bpo-45788.qibUoB.rst
deleted file mode 100644
index 8aa3293673e1f..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-11-18-00-07-40.bpo-45788.qibUoB.rst
+++ /dev/null
@@ -1 +0,0 @@
-Link doc for sys.prefix to sysconfig doc on installation paths.
diff --git a/Misc/NEWS.d/next/Documentation/2021-11-18-16-44-12.bpo-45640.lSpc2A.rst b/Misc/NEWS.d/next/Documentation/2021-11-18-16-44-12.bpo-45640.lSpc2A.rst
deleted file mode 100644
index 532288df7c76e..0000000000000
--- a/Misc/NEWS.d/next/Documentation/2021-11-18-16-44-12.bpo-45640.lSpc2A.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Properly marked-up grammar tokens in the documentation are now clickable and
-take you to the definition of a given piece of grammar.  Patch by Arthur
-Milchior.
diff --git a/Misc/NEWS.d/next/IDLE/2021-09-15-03-20-06.bpo-45193.G61_GV.rst b/Misc/NEWS.d/next/IDLE/2021-09-15-03-20-06.bpo-45193.G61_GV.rst
deleted file mode 100644
index 94729640c71eb..0000000000000
--- a/Misc/NEWS.d/next/IDLE/2021-09-15-03-20-06.bpo-45193.G61_GV.rst
+++ /dev/null
@@ -1 +0,0 @@
-Make completion boxes appear on Ubuntu again.
diff --git a/Misc/NEWS.d/next/IDLE/2021-09-27-01-21-59.bpo-45296.9H8rdY.rst b/Misc/NEWS.d/next/IDLE/2021-09-27-01-21-59.bpo-45296.9H8rdY.rst
deleted file mode 100644
index 52bade1e5327b..0000000000000
--- a/Misc/NEWS.d/next/IDLE/2021-09-27-01-21-59.bpo-45296.9H8rdY.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-On Windows, change exit/quit message to suggest Ctrl-D, which works, instead
-of <Ctrl-Z Return>, which does not work in IDLE.
diff --git a/Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst b/Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst
deleted file mode 100644
index 3868f8d136a04..0000000000000
--- a/Misc/NEWS.d/next/IDLE/2021-10-16-17-20-32.bpo-45495.ST8RFt.rst
+++ /dev/null
@@ -1 +0,0 @@
-Add context keywords 'case' and 'match' to completions list.
diff --git a/Misc/NEWS.d/next/Library/2021-04-20-14-14-16.bpo-43498.L_Hq-8.rst b/Misc/NEWS.d/next/Library/2021-04-20-14-14-16.bpo-43498.L_Hq-8.rst
deleted file mode 100644
index 4713d1427ccd3..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-04-20-14-14-16.bpo-43498.L_Hq-8.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Avoid a possible *"RuntimeError: dictionary changed size during iteration"*
-when adjusting the process count of :class:`ProcessPoolExecutor`.
diff --git a/Misc/NEWS.d/next/Library/2021-06-02-16-39-42.bpo-44295.erg01m.rst b/Misc/NEWS.d/next/Library/2021-06-02-16-39-42.bpo-44295.erg01m.rst
deleted file mode 100644
index 86501c15d86f4..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-06-02-16-39-42.bpo-44295.erg01m.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Ensure deprecation warning from :func:`assertDictContainsSubset` points at
-calling code - by Anthony Sottile.
diff --git a/Misc/NEWS.d/next/Library/2021-07-12-10-32-48.bpo-44594.eEa5zi.rst b/Misc/NEWS.d/next/Library/2021-07-12-10-32-48.bpo-44594.eEa5zi.rst
deleted file mode 100644
index a2bfd8ff5b51b..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-07-12-10-32-48.bpo-44594.eEa5zi.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix an edge case of :class:`ExitStack` and :class:`AsyncExitStack` exception
-chaining.  They will now match ``with`` block behavior when ``__context__`` is
-explicitly set to ``None`` when the exception is in flight.
diff --git a/Misc/NEWS.d/next/Library/2021-08-18-10-36-14.bpo-39039.A63LYh.rst b/Misc/NEWS.d/next/Library/2021-08-18-10-36-14.bpo-39039.A63LYh.rst
deleted file mode 100644
index 7250055c2a4a9..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-08-18-10-36-14.bpo-39039.A63LYh.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-tarfile.open raises :exc:`~tarfile.ReadError` when a zlib error occurs
-during file extraction.
diff --git a/Misc/NEWS.d/next/Library/2021-08-28-13-00-12.bpo-45021.rReeaj.rst b/Misc/NEWS.d/next/Library/2021-08-28-13-00-12.bpo-45021.rReeaj.rst
deleted file mode 100644
index 54fd9109a9ae5..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-08-28-13-00-12.bpo-45021.rReeaj.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix a potential deadlock at shutdown of forked children when using :mod:`concurrent.futures` module
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Library/2021-08-30-00-19-23.bpo-24444.Ki4bgz.rst b/Misc/NEWS.d/next/Library/2021-08-30-00-19-23.bpo-24444.Ki4bgz.rst
deleted file mode 100644
index efcacb8f0eb59..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-08-30-00-19-23.bpo-24444.Ki4bgz.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fixed an error raised in :mod:`argparse` help display when help for an
-option is set to 1+ blank spaces or when *choices* arg is an empty container.
diff --git a/Misc/NEWS.d/next/Library/2021-09-08-01-19-31.bpo-20499.tSxx8Y.rst b/Misc/NEWS.d/next/Library/2021-09-08-01-19-31.bpo-20499.tSxx8Y.rst
deleted file mode 100644
index cbbe61ac4a269..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-08-01-19-31.bpo-20499.tSxx8Y.rst
+++ /dev/null
@@ -1 +0,0 @@
-Improve the speed and accuracy of statistics.pvariance().
diff --git a/Misc/NEWS.d/next/Library/2021-09-10-21-35-53.bpo-45166.UHipXF.rst b/Misc/NEWS.d/next/Library/2021-09-10-21-35-53.bpo-45166.UHipXF.rst
deleted file mode 100644
index b7242d45ea9be..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-10-21-35-53.bpo-45166.UHipXF.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-:func:`typing.get_type_hints` now works with :data:`~typing.Final` wrapped in
-:class:`~typing.ForwardRef`.
diff --git a/Misc/NEWS.d/next/Library/2021-09-11-10-45-12.bpo-35474.tEY3SD.rst b/Misc/NEWS.d/next/Library/2021-09-11-10-45-12.bpo-35474.tEY3SD.rst
deleted file mode 100644
index f4dd3b947a493..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-11-10-45-12.bpo-35474.tEY3SD.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Calling :func:`mimetypes.guess_all_extensions` with ``strict=False`` no
-longer affects the result of the following call with ``strict=True``.
-Also, mutating the returned list no longer affects the global state.
diff --git a/Misc/NEWS.d/next/Library/2021-09-11-14-47-05.bpo-45160.VzMXbW.rst b/Misc/NEWS.d/next/Library/2021-09-11-14-47-05.bpo-45160.VzMXbW.rst
deleted file mode 100644
index 9d11ed0e55d24..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-11-14-47-05.bpo-45160.VzMXbW.rst
+++ /dev/null
@@ -1 +0,0 @@
-When tracing a tkinter variable used by a ttk OptionMenu, callbacks are no longer made twice.
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Library/2021-09-13-19-32-58.bpo-42135.1ZAHqR.rst b/Misc/NEWS.d/next/Library/2021-09-13-19-32-58.bpo-42135.1ZAHqR.rst
deleted file mode 100644
index 983b46e140e74..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-13-19-32-58.bpo-42135.1ZAHqR.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix typo: ``importlib.find_loader`` is really slated for removal in Python 3.12 not 3.10, like the others in GH-25169.
-
-Patch by Hugo van Kemenade.
diff --git a/Misc/NEWS.d/next/Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst b/Misc/NEWS.d/next/Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst
deleted file mode 100644
index 7dd9795aaa170..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-14-15-52-47.bpo-45192.DjA-BI.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-Fix the ``tempfile._infer_return_type`` function so that the ``dir``
-argument of the :mod:`tempfile` functions accepts an object implementing the
-``os.PathLike`` protocol.
-
-Patch by Kyungmin Lee.
diff --git a/Misc/NEWS.d/next/Library/2021-09-17-09-59-33.bpo-45228.WV1dcT.rst b/Misc/NEWS.d/next/Library/2021-09-17-09-59-33.bpo-45228.WV1dcT.rst
deleted file mode 100644
index 9336c0aed92bc..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-17-09-59-33.bpo-45228.WV1dcT.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix stack buffer overflow in parsing J1939 network address.
diff --git a/Misc/NEWS.d/next/Library/2021-09-17-11-20-55.bpo-45234.qUcTVt.rst b/Misc/NEWS.d/next/Library/2021-09-17-11-20-55.bpo-45234.qUcTVt.rst
deleted file mode 100644
index 3817b5de6449d..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-17-11-20-55.bpo-45234.qUcTVt.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fixed a regression in :func:`~shutil.copyfile`, :func:`~shutil.copy`,
-:func:`~shutil.copy2` raising :exc:`FileNotFoundError` when source is a
-directory, which should raise :exc:`IsADirectoryError`
diff --git a/Misc/NEWS.d/next/Library/2021-09-17-15-58-53.bpo-45183.Vv_vch.rst b/Misc/NEWS.d/next/Library/2021-09-17-15-58-53.bpo-45183.Vv_vch.rst
deleted file mode 100644
index f3194b34318f4..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-17-15-58-53.bpo-45183.Vv_vch.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Have zipimport.zipimporter.find_spec() not raise an exception when the underlying zip
-file has been deleted and the internal cache has been reset via
-invalidate_cache().
diff --git a/Misc/NEWS.d/next/Library/2021-09-17-16-55-37.bpo-45235.sXnmPA.rst b/Misc/NEWS.d/next/Library/2021-09-17-16-55-37.bpo-45235.sXnmPA.rst
deleted file mode 100644
index 871ec5281d334..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-17-16-55-37.bpo-45235.sXnmPA.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix an issue where argparse would not preserve values in a provided namespace
-when using a subparser with defaults.
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Library/2021-09-18-13-14-57.bpo-36674.a2k5Zb.rst b/Misc/NEWS.d/next/Library/2021-09-18-13-14-57.bpo-36674.a2k5Zb.rst
deleted file mode 100644
index bc8c9247b080d..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-18-13-14-57.bpo-36674.a2k5Zb.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-:meth:`unittest.TestCase.debug` raises now a :class:`unittest.SkipTest` if
-the class or the test method are decorated with the skipping decorator.
diff --git a/Misc/NEWS.d/next/Library/2021-09-18-16-56-33.bpo-45238.Hng_9V.rst b/Misc/NEWS.d/next/Library/2021-09-18-16-56-33.bpo-45238.Hng_9V.rst
deleted file mode 100644
index 857f315c520bb..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-18-16-56-33.bpo-45238.Hng_9V.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix :meth:`unittest.IsolatedAsyncioTestCase.debug`: it runs now asynchronous
-methods and callbacks.
diff --git a/Misc/NEWS.d/next/Library/2021-09-23-22-17-26.bpo-45274.gPpa4E.rst b/Misc/NEWS.d/next/Library/2021-09-23-22-17-26.bpo-45274.gPpa4E.rst
deleted file mode 100644
index 94d06cef89b7b..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-23-22-17-26.bpo-45274.gPpa4E.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-Fix a race condition in the :meth:`Thread.join() <threading.Thread.join>`
-method of the :mod:`threading` module. If the function is interrupted by a
-signal and the signal handler raises an exception, make sure that the thread
-remains in a consistent state to prevent a deadlock. Patch by Victor
-Stinner.
diff --git a/Misc/NEWS.d/next/Library/2021-09-24-17-20-23.bpo-1596321.3nhPUk.rst b/Misc/NEWS.d/next/Library/2021-09-24-17-20-23.bpo-1596321.3nhPUk.rst
deleted file mode 100644
index 61a3e5abf395e..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-24-17-20-23.bpo-1596321.3nhPUk.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix the :func:`threading._shutdown` function when the :mod:`threading` module
-was imported first from a thread different than the main thread: no longer log
-an error at Python exit.
diff --git a/Misc/NEWS.d/next/Library/2021-09-30-08-22-44.bpo-45328.8Z-Q0B.rst b/Misc/NEWS.d/next/Library/2021-09-30-08-22-44.bpo-45328.8Z-Q0B.rst
deleted file mode 100644
index eeb49310e8f66..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-30-08-22-44.bpo-45328.8Z-Q0B.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fixed :class:`http.client.HTTPConnection` to work properly in OSs that don't support the ``TCP_NODELAY`` socket option.
diff --git a/Misc/NEWS.d/next/Library/2021-09-30-23-00-18.bpo-41710.svuloZ.rst b/Misc/NEWS.d/next/Library/2021-09-30-23-00-18.bpo-41710.svuloZ.rst
deleted file mode 100644
index d8a4f9507c189..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-09-30-23-00-18.bpo-41710.svuloZ.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-On Unix, if the ``sem_clockwait()`` function is available in the C library
-(glibc 2.30 and newer), the :meth:`threading.Lock.acquire` method now uses the
-monotonic clock (:data:`time.CLOCK_MONOTONIC`) for the timeout, rather than
-using the system clock (:data:`time.CLOCK_REALTIME`), to not be affected by
-system clock changes. Patch by Victor Stinner.
diff --git a/Misc/NEWS.d/next/Library/2021-10-01-13-09-53.bpo-45329.9iMYaO.rst b/Misc/NEWS.d/next/Library/2021-10-01-13-09-53.bpo-45329.9iMYaO.rst
deleted file mode 100644
index b4bedbc278edf..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-01-13-09-53.bpo-45329.9iMYaO.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix freed memory access in :class:`pyexpat.xmlparser` when building it with an
-installed expat library <= 2.2.0.
diff --git a/Misc/NEWS.d/next/Library/2021-10-01-23-07-02.bpo-45343.ixmctD.rst b/Misc/NEWS.d/next/Library/2021-10-01-23-07-02.bpo-45343.ixmctD.rst
deleted file mode 100644
index 8dac4e6a25271..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-01-23-07-02.bpo-45343.ixmctD.rst
+++ /dev/null
@@ -1 +0,0 @@
-Update bundled pip to 21.2.4 and setuptools to 58.1.0
diff --git a/Misc/NEWS.d/next/Library/2021-10-03-21-14-37.bpo-20028.zBA4RK.rst b/Misc/NEWS.d/next/Library/2021-10-03-21-14-37.bpo-20028.zBA4RK.rst
deleted file mode 100644
index e75612116e942..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-03-21-14-37.bpo-20028.zBA4RK.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Improve error message of :class:`csv.Dialect` when initializing.
-Patch by Vajrasky Kok and Dong-hee Na.
diff --git a/Misc/NEWS.d/next/Library/2021-10-05-11-03-48.bpo-45371.NOwcDJ.rst b/Misc/NEWS.d/next/Library/2021-10-05-11-03-48.bpo-45371.NOwcDJ.rst
deleted file mode 100644
index 045489be81a19..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-05-11-03-48.bpo-45371.NOwcDJ.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix clang rpath issue in :mod:`distutils`. The UnixCCompiler now uses
-correct clang option to add a runtime library directory (rpath) to a shared
-library.
diff --git a/Misc/NEWS.d/next/Library/2021-10-07-00-05-05.bpo-45386.q9ORpA.rst b/Misc/NEWS.d/next/Library/2021-10-07-00-05-05.bpo-45386.q9ORpA.rst
deleted file mode 100644
index eec77ceccf933..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-07-00-05-05.bpo-45386.q9ORpA.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Make :mod:`xmlrpc.client` more robust to C runtimes where the underlying C
-``strftime`` function results in a ``ValueError`` when testing for year
-formatting options.
diff --git a/Misc/NEWS.d/next/Library/2021-10-07-14-04-10.bpo-45262.HqF71Z.rst b/Misc/NEWS.d/next/Library/2021-10-07-14-04-10.bpo-45262.HqF71Z.rst
deleted file mode 100644
index 4cd949fe1ed5d..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-07-14-04-10.bpo-45262.HqF71Z.rst
+++ /dev/null
@@ -1 +0,0 @@
-Prevent use-after-free in asyncio. Make sure the cached running loop holder gets cleared on dealloc to prevent use-after-free in get_running_loop
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Library/2021-10-08-19-24-48.bpo-45406.Qh_Mz4.rst b/Misc/NEWS.d/next/Library/2021-10-08-19-24-48.bpo-45406.Qh_Mz4.rst
deleted file mode 100644
index 2c3a8165aeb49..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-08-19-24-48.bpo-45406.Qh_Mz4.rst
+++ /dev/null
@@ -1 +0,0 @@
-Make :func:`inspect.getmodule` catch ``FileNotFoundError`` raised by  :'func:`inspect.getabsfile`, and return ``None`` to indicate that the module could not be determined.
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Library/2021-10-09-18-42-27.bpo-44904.RlW5h8.rst b/Misc/NEWS.d/next/Library/2021-10-09-18-42-27.bpo-44904.RlW5h8.rst
deleted file mode 100644
index b02d499d23500..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-09-18-42-27.bpo-44904.RlW5h8.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix bug in the :mod:`doctest` module that caused it to fail if a docstring
-included an example with a ``classmethod`` ``property``. Patch by Alex
-Waygood.
diff --git a/Misc/NEWS.d/next/Library/2021-10-09-20-53-13.bpo-45419.CauCgt.rst b/Misc/NEWS.d/next/Library/2021-10-09-20-53-13.bpo-45419.CauCgt.rst
deleted file mode 100644
index a901d7453819f..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-09-20-53-13.bpo-45419.CauCgt.rst
+++ /dev/null
@@ -1 +0,0 @@
-Correct interfaces on DegenerateFiles.Path.
diff --git a/Misc/NEWS.d/next/Library/2021-10-10-09-42-34.bpo-45416.n35O0_.rst b/Misc/NEWS.d/next/Library/2021-10-10-09-42-34.bpo-45416.n35O0_.rst
deleted file mode 100644
index cf335d1bcd2c9..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-10-09-42-34.bpo-45416.n35O0_.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix use of :class:`asyncio.Condition` with explicit :class:`asyncio.Lock` objects, which was a regression due to removal of explicit loop arguments.
-Patch by Joongi Kim.
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst b/Misc/NEWS.d/next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst
deleted file mode 100644
index 1d5a857e25435..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-10-16-14-33.bpo-45249.xqLliz.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix the behaviour of :func:`traceback.print_exc` when displaying the caret
-when the ``end_offset`` in the exception is set to 0. Patch by Pablo Galindo
diff --git a/Misc/NEWS.d/next/Library/2021-10-13-17-52-48.bpo-45239.7li1_0.rst b/Misc/NEWS.d/next/Library/2021-10-13-17-52-48.bpo-45239.7li1_0.rst
deleted file mode 100644
index 9e5ec561c362a..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-13-17-52-48.bpo-45239.7li1_0.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fixed :func:`email.utils.parsedate_tz` crashing with
-:exc:`UnboundLocalError` on certain invalid input instead of returning
-``None``. Patch by Ben Hoyt.
diff --git a/Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst b/Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst
deleted file mode 100644
index c1c4ed1ace248..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-14-00-19-02.bpo-45461.4LB_tJ.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix incremental decoder and stream reader in the "unicode-escape" codec.
-Previously they failed if the escape sequence was split.
diff --git a/Misc/NEWS.d/next/Library/2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst b/Misc/NEWS.d/next/Library/2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst
deleted file mode 100644
index f2c0ae4ae51fb..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-14-13-31-19.bpo-45467.Q7Ma6A.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix incremental decoder and stream reader in the "raw-unicode-escape" codec.
-Previously they failed if the escape sequence was split.
diff --git a/Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst b/Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst
deleted file mode 100644
index 556eca43ed3c7..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-14-18-04-17.bpo-45428.mM2War.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix a regression in py_compile when reading filenames from standard input.
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst b/Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst
deleted file mode 100644
index 6fce894e6e4d4..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-18-10-46-47.bpo-45475.sb9KDF.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-Reverted optimization of iterating :class:`gzip.GzipFile`,
-:class:`bz2.BZ2File`, and :class:`lzma.LZMAFile` (see bpo-43787) because it
-caused regression when user iterate them without having reference of them.
-Patch by Inada Naoki.
diff --git a/Misc/NEWS.d/next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst b/Misc/NEWS.d/next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst
deleted file mode 100644
index 382733ff91883..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-18-14-52-48.bpo-45515.aXdvm_.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Add references to :mod:`zoneinfo` in the :mod:`datetime` documentation,
-mostly replacing outdated references to ``dateutil.tz``. Change by Paul
-Ganssle.
diff --git a/Misc/NEWS.d/next/Library/2021-10-21-16-18-51.bpo-45557.4MQt4r.rst b/Misc/NEWS.d/next/Library/2021-10-21-16-18-51.bpo-45557.4MQt4r.rst
deleted file mode 100644
index 7472b08d1b08e..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-21-16-18-51.bpo-45557.4MQt4r.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-pprint.pprint() now handles underscore_numbers correctly. Previously it was
-always setting it to False.
diff --git a/Misc/NEWS.d/next/Library/2021-10-22-21-57-02.bpo-45581.rlH6ay.rst b/Misc/NEWS.d/next/Library/2021-10-22-21-57-02.bpo-45581.rlH6ay.rst
deleted file mode 100644
index 13a3b237434eb..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-22-21-57-02.bpo-45581.rlH6ay.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-:meth:`sqlite3.connect` now correctly raises :exc:`MemoryError` if the
-underlying SQLite API signals memory error. Patch by Erlend E. Aasland.
diff --git a/Misc/NEWS.d/next/Library/2021-10-22-23-06-33.bpo-45574.svqA84.rst b/Misc/NEWS.d/next/Library/2021-10-22-23-06-33.bpo-45574.svqA84.rst
deleted file mode 100644
index b404d24473960..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-22-23-06-33.bpo-45574.svqA84.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix warning about ``print_escape`` being unused.
diff --git a/Misc/NEWS.d/next/Library/2021-10-27-10-05-39.bpo-45438.Xz5lGU.rst b/Misc/NEWS.d/next/Library/2021-10-27-10-05-39.bpo-45438.Xz5lGU.rst
deleted file mode 100644
index cd6cfc1fcd407..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-27-10-05-39.bpo-45438.Xz5lGU.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix typing.Signature string representation for generic builtin types.
diff --git a/Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst b/Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst
deleted file mode 100644
index 050b443dd7cb2..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-28-22-58-14.bpo-45662.sJd7Ir.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix the repr of :data:`dataclasses.InitVar` with a type alias to the
-built-in class, e.g. ``InitVar[list[int]]``.
diff --git a/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst b/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
deleted file mode 100644
index f246f67cf80e5..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-28-23-11-59.bpo-45663.J90N5R.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix :func:`dataclasses.is_dataclass` for dataclasses which are subclasses of
-:class:`types.GenericAlias`.
diff --git a/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst b/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
deleted file mode 100644
index 573a569845a87..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix :func:`types.resolve_bases` and :func:`types.new_class` for
-:class:`types.GenericAlias` instance as a base.
diff --git a/Misc/NEWS.d/next/Library/2021-10-30-21-11-37.bpo-45679.Dq8Cpu.rst b/Misc/NEWS.d/next/Library/2021-10-30-21-11-37.bpo-45679.Dq8Cpu.rst
deleted file mode 100644
index a644492a12d17..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-10-30-21-11-37.bpo-45679.Dq8Cpu.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix caching of multi-value :data:`typing.Literal`. ``Literal[True, 2]`` is no
-longer equal to ``Literal[1, 2]``.
diff --git a/Misc/NEWS.d/next/Library/2021-11-06-17-47-46.bpo-45644.ZMqHD_.rst b/Misc/NEWS.d/next/Library/2021-11-06-17-47-46.bpo-45644.ZMqHD_.rst
deleted file mode 100644
index 2cf4eae02c449..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-11-06-17-47-46.bpo-45644.ZMqHD_.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-In-place JSON file formatting using ``python3 -m json.tool infile infile``
-now works correctly, previously it left the file empty.  Patch by Chris
-Wesseling.
diff --git a/Misc/NEWS.d/next/Library/2021-11-08-23-22-14.bpo-45757.MHZHt3.rst b/Misc/NEWS.d/next/Library/2021-11-08-23-22-14.bpo-45757.MHZHt3.rst
deleted file mode 100644
index f25638cc68f85..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-11-08-23-22-14.bpo-45757.MHZHt3.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where :mod:`dis` produced an incorrect oparg when :opcode:`EXTENDED_ARG` is followed by an opcode that does not use its argument.
\ No newline at end of file
diff --git a/Misc/NEWS.d/next/Library/2021-11-09-09-04-19.bpo-45765.JVobxK.rst b/Misc/NEWS.d/next/Library/2021-11-09-09-04-19.bpo-45765.JVobxK.rst
deleted file mode 100644
index a1f4a1f7aa91c..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-11-09-09-04-19.bpo-45765.JVobxK.rst
+++ /dev/null
@@ -1 +0,0 @@
-In importlib.metadata, fix distribution discovery for an empty path.
diff --git a/Misc/NEWS.d/next/Library/2021-11-11-13-03-17.bpo-45235.8ZbkHa.rst b/Misc/NEWS.d/next/Library/2021-11-11-13-03-17.bpo-45235.8ZbkHa.rst
deleted file mode 100644
index f73589ccc8872..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-11-11-13-03-17.bpo-45235.8ZbkHa.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Reverted an argparse bugfix that caused regression in the handling of
-default arguments for subparsers.  This prevented leaf level arguments from
-taking precedence over root level arguments.
diff --git a/Misc/NEWS.d/next/Library/2021-11-16-18-13-49.bpo-41735.D72UY1.rst b/Misc/NEWS.d/next/Library/2021-11-16-18-13-49.bpo-41735.D72UY1.rst
deleted file mode 100644
index 101da0e9ce648..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-11-16-18-13-49.bpo-41735.D72UY1.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix thread lock in ``zlib.Decompress.flush()`` method before ``PyObject_GetBuffer``.
diff --git a/Misc/NEWS.d/next/Library/2021-11-17-19-25-37.bpo-45831.9-TojK.rst b/Misc/NEWS.d/next/Library/2021-11-17-19-25-37.bpo-45831.9-TojK.rst
deleted file mode 100644
index 049449ff0a4a1..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-11-17-19-25-37.bpo-45831.9-TojK.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-:mod:`faulthandler` can now write ASCII-only strings (like filenames and
-function names) with a single write() syscall when dumping a traceback. It
-reduces the risk of getting an unreadable dump when two threads or two
-processes dump a traceback to the same file (like stderr) at the same time.
-Patch by Victor Stinner.
diff --git a/Misc/NEWS.d/next/Library/2021-11-20-17-04-25.bpo-45803.wSgFOy.rst b/Misc/NEWS.d/next/Library/2021-11-20-17-04-25.bpo-45803.wSgFOy.rst
deleted file mode 100644
index 77479d7db476b..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-11-20-17-04-25.bpo-45803.wSgFOy.rst
+++ /dev/null
@@ -1 +0,0 @@
-Added missing kw_only parameter to dataclasses.make_dataclass().
diff --git a/Misc/NEWS.d/next/Library/2021-11-21-20-50-42.bpo-44649.E8M936.rst b/Misc/NEWS.d/next/Library/2021-11-21-20-50-42.bpo-44649.E8M936.rst
deleted file mode 100644
index f6391a915a821..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-11-21-20-50-42.bpo-44649.E8M936.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Handle dataclass(slots=True) with a field that has default a default value,
-but for which init=False.
diff --git a/Misc/NEWS.d/next/Library/2021-11-28-15-30-34.bpo-37658.8Hno7d.rst b/Misc/NEWS.d/next/Library/2021-11-28-15-30-34.bpo-37658.8Hno7d.rst
deleted file mode 100644
index 97d1e961ac2be..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-11-28-15-30-34.bpo-37658.8Hno7d.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix issue when on certain conditions ``asyncio.wait_for()`` may allow a
-coroutine to complete successfully, but fail to return the result,
-potentially causing memory leaks or other issues.
diff --git a/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst b/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
deleted file mode 100644
index 0378efca746bb..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-12-04-20-08-42.bpo-27946.-Vuarf.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix possible crash when getting an attribute of
-class:`xml.etree.ElementTree.Element` simultaneously with
-replacing the ``attrib`` dict.
diff --git a/Misc/NEWS.d/next/Tests/2021-08-27-22-37-19.bpo-25130.ig4oJe.rst b/Misc/NEWS.d/next/Tests/2021-08-27-22-37-19.bpo-25130.ig4oJe.rst
deleted file mode 100644
index 43ce68bef4609..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-08-27-22-37-19.bpo-25130.ig4oJe.rst
+++ /dev/null
@@ -1 +0,0 @@
-Add calls of :func:`gc.collect` in tests to support PyPy.
diff --git a/Misc/NEWS.d/next/Tests/2021-09-08-13-01-37.bpo-44860.qXd0kx.rst b/Misc/NEWS.d/next/Tests/2021-09-08-13-01-37.bpo-44860.qXd0kx.rst
deleted file mode 100644
index 153a9c55733fb..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-09-08-13-01-37.bpo-44860.qXd0kx.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Update ``test_sysconfig.test_user_similar()`` for the posix_user scheme:
-``platlib`` doesn't use :data:`sys.platlibdir`. Patch by Victor Stinner.
diff --git a/Misc/NEWS.d/next/Tests/2021-09-11-22-08-18.bpo-45125.FVSzs2.rst b/Misc/NEWS.d/next/Tests/2021-09-11-22-08-18.bpo-45125.FVSzs2.rst
deleted file mode 100644
index 5dfbe0e5db463..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-09-11-22-08-18.bpo-45125.FVSzs2.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Improves pickling tests and docs of ``SharedMemory`` and ``SharableList``
-objects.
diff --git a/Misc/NEWS.d/next/Tests/2021-09-13-00-28-17.bpo-45156.8oomV3.rst b/Misc/NEWS.d/next/Tests/2021-09-13-00-28-17.bpo-45156.8oomV3.rst
deleted file mode 100644
index b2094b5765331..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-09-13-00-28-17.bpo-45156.8oomV3.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fixes infinite loop on :func:`unittest.mock.seal` of mocks created by
-:func:`~unittest.create_autospec`.
diff --git a/Misc/NEWS.d/next/Tests/2021-09-14-13-16-18.bpo-45195.EyQR1G.rst b/Misc/NEWS.d/next/Tests/2021-09-14-13-16-18.bpo-45195.EyQR1G.rst
deleted file mode 100644
index 16a1f4440483c..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-09-14-13-16-18.bpo-45195.EyQR1G.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix test_readline.test_nonascii(): sometimes, the newline character is not
-written at the end, so don't expect it in the output. Patch by Victor
-Stinner.
diff --git a/Misc/NEWS.d/next/Tests/2021-09-15-23-32-39.bpo-45209.55ntL5.rst b/Misc/NEWS.d/next/Tests/2021-09-15-23-32-39.bpo-45209.55ntL5.rst
deleted file mode 100644
index 4c3bed0983b89..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-09-15-23-32-39.bpo-45209.55ntL5.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix ``UserWarning: resource_tracker`` warning in
-``_test_multiprocessing._TestSharedMemory.test_shared_memory_cleaned_after_process_termination``
diff --git a/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst b/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst
deleted file mode 100644
index b50eb32b3faa8..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-09-16-17-22-35.bpo-45128.Jz6fl2.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix ``test_multiprocessing_fork`` failure due to ``test_logging`` and
-``sys.modules`` manipulation.
diff --git a/Misc/NEWS.d/next/Tests/2021-09-24-10-41-49.bpo-45269.8jKEr8.rst b/Misc/NEWS.d/next/Tests/2021-09-24-10-41-49.bpo-45269.8jKEr8.rst
deleted file mode 100644
index 72dd9471134ff..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-09-24-10-41-49.bpo-45269.8jKEr8.rst
+++ /dev/null
@@ -1 +0,0 @@
-Cover case when invalid ``markers`` type is supplied to ``c_make_encoder``.
diff --git a/Misc/NEWS.d/next/Tests/2021-09-25-11-05-31.bpo-45280.3MA6lC.rst b/Misc/NEWS.d/next/Tests/2021-09-25-11-05-31.bpo-45280.3MA6lC.rst
deleted file mode 100644
index 71691f5ba2b89..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-09-25-11-05-31.bpo-45280.3MA6lC.rst
+++ /dev/null
@@ -1 +0,0 @@
-Add a test case for empty :class:`typing.NamedTuple`.
diff --git a/Misc/NEWS.d/next/Tests/2021-09-30-16-54-39.bpo-40173.J_slCw.rst b/Misc/NEWS.d/next/Tests/2021-09-30-16-54-39.bpo-40173.J_slCw.rst
deleted file mode 100644
index 21671473c16cc..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-09-30-16-54-39.bpo-40173.J_slCw.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Fix :func:`test.support.import_helper.import_fresh_module`.
-
diff --git a/Misc/NEWS.d/next/Tests/2021-10-07-13-11-45.bpo-45400.h3iT7V.rst b/Misc/NEWS.d/next/Tests/2021-10-07-13-11-45.bpo-45400.h3iT7V.rst
deleted file mode 100644
index 61b6653320dfc..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-10-07-13-11-45.bpo-45400.h3iT7V.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Fix test_name_error_suggestions_do_not_trigger_for_too_many_locals() of
-test_exceptions if a directory name contains "a1" (like "Python-3.11.0a1"):
-use a stricter regular expression. Patch by Victor Stinner.
diff --git a/Misc/NEWS.d/next/Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst b/Misc/NEWS.d/next/Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst
deleted file mode 100644
index b0d1b686392ef..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-10-18-16-18-41.bpo-39679.F18qcE.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Add more test cases for `@functools.singledispatchmethod` when combined with
-`@classmethod` or `@staticmethod`.
diff --git a/Misc/NEWS.d/next/Tests/2021-10-21-17-22-26.bpo-43592.kHRsra.rst b/Misc/NEWS.d/next/Tests/2021-10-21-17-22-26.bpo-43592.kHRsra.rst
deleted file mode 100644
index 2528857caf9c0..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-10-21-17-22-26.bpo-43592.kHRsra.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-:mod:`test.libregrtest` now raises the soft resource limit for the maximum
-number of file descriptors when the default is too low for our test suite as
-was often the case on macOS.
diff --git a/Misc/NEWS.d/next/Tests/2021-10-22-12-05-21.bpo-45566.2gQ3ZB.rst b/Misc/NEWS.d/next/Tests/2021-10-22-12-05-21.bpo-45566.2gQ3ZB.rst
deleted file mode 100644
index a2ecf721800d9..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-10-22-12-05-21.bpo-45566.2gQ3ZB.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix ``test_frozen_pickle`` in ``test_dataclasses`` to check all ``pickle`` versions.
diff --git a/Misc/NEWS.d/next/Tests/2021-10-22-19-44-13.bpo-45577.dSaNvK.rst b/Misc/NEWS.d/next/Tests/2021-10-22-19-44-13.bpo-45577.dSaNvK.rst
deleted file mode 100644
index fc9783eb9ddaa..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-10-22-19-44-13.bpo-45577.dSaNvK.rst
+++ /dev/null
@@ -1 +0,0 @@
-Add subtests for all ``pickle`` protocols in ``test_zoneinfo``.
diff --git a/Misc/NEWS.d/next/Tests/2021-10-30-13-12-20.bpo-45678.bKrYeS.rst b/Misc/NEWS.d/next/Tests/2021-10-30-13-12-20.bpo-45678.bKrYeS.rst
deleted file mode 100644
index 885b2fa64ad0f..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-10-30-13-12-20.bpo-45678.bKrYeS.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Add tests to ensure that ``functools.singledispatchmethod`` correctly wraps
-the attributes of the target function.
diff --git a/Misc/NEWS.d/next/Tests/2021-10-30-19-00-25.bpo-45578.bvu6X2.rst b/Misc/NEWS.d/next/Tests/2021-10-30-19-00-25.bpo-45578.bvu6X2.rst
deleted file mode 100644
index 3d0e0ca3f04a1..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-10-30-19-00-25.bpo-45578.bvu6X2.rst
+++ /dev/null
@@ -1 +0,0 @@
-Add tests for :func:`dis.distb`
diff --git a/Misc/NEWS.d/next/Tests/2021-11-04-20-03-32.bpo-45678.1xNMjN.rst b/Misc/NEWS.d/next/Tests/2021-11-04-20-03-32.bpo-45678.1xNMjN.rst
deleted file mode 100644
index 736d5f65f961b..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-11-04-20-03-32.bpo-45678.1xNMjN.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Add tests for scenarios in which :class:`functools.singledispatchmethod` is
-stacked on top of a method that has already been wrapped by two other
-decorators. Patch by Alex Waygood.
diff --git a/Misc/NEWS.d/next/Tests/2021-11-17-14-28-08.bpo-45835.Mgyhjx.rst b/Misc/NEWS.d/next/Tests/2021-11-17-14-28-08.bpo-45835.Mgyhjx.rst
deleted file mode 100644
index 6a73b01959065..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-11-17-14-28-08.bpo-45835.Mgyhjx.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix race condition in test_queue tests with multiple "feeder" threads.
diff --git a/Misc/NEWS.d/next/Tests/2021-11-28-15-25-02.bpo-19460.lr0aWs.rst b/Misc/NEWS.d/next/Tests/2021-11-28-15-25-02.bpo-19460.lr0aWs.rst
deleted file mode 100644
index b082d6de20c07..0000000000000
--- a/Misc/NEWS.d/next/Tests/2021-11-28-15-25-02.bpo-19460.lr0aWs.rst
+++ /dev/null
@@ -1 +0,0 @@
-Add new Test for :class:`email.mime.nonmultipart.MIMENonMultipart`.
diff --git a/Misc/NEWS.d/next/Tools-Demos/2021-09-14-11-44-26.bpo-44786.DU0LC0.rst b/Misc/NEWS.d/next/Tools-Demos/2021-09-14-11-44-26.bpo-44786.DU0LC0.rst
deleted file mode 100644
index 96ebf2c33cc5a..0000000000000
--- a/Misc/NEWS.d/next/Tools-Demos/2021-09-14-11-44-26.bpo-44786.DU0LC0.rst
+++ /dev/null
@@ -1 +0,0 @@
-Fix a warning in regular expression in the c-analyzer script.
diff --git a/Misc/NEWS.d/next/Windows/2021-09-30-23-17-27.bpo-45337.qg7U_h.rst b/Misc/NEWS.d/next/Windows/2021-09-30-23-17-27.bpo-45337.qg7U_h.rst
deleted file mode 100644
index 007ee87195d6e..0000000000000
--- a/Misc/NEWS.d/next/Windows/2021-09-30-23-17-27.bpo-45337.qg7U_h.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-venv now warns when the created environment may need to be accessed at a
-different path, due to redirections, links or junctions. It also now
-correctly installs or upgrades components when the alternate path is
-required.
diff --git a/Misc/NEWS.d/next/Windows/2021-11-04-00-41-50.bpo-43652.RnqV7I.rst b/Misc/NEWS.d/next/Windows/2021-11-04-00-41-50.bpo-43652.RnqV7I.rst
deleted file mode 100644
index 7da8fc79a9dab..0000000000000
--- a/Misc/NEWS.d/next/Windows/2021-11-04-00-41-50.bpo-43652.RnqV7I.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Update Tcl/Tk to 8.6.11, actually this time. The previous update incorrectly
-included 8.6.10.
diff --git a/Misc/NEWS.d/next/Windows/2021-11-05-01-05-46.bpo-45720.47Nc5I.rst b/Misc/NEWS.d/next/Windows/2021-11-05-01-05-46.bpo-45720.47Nc5I.rst
deleted file mode 100644
index 315759b07e147..0000000000000
--- a/Misc/NEWS.d/next/Windows/2021-11-05-01-05-46.bpo-45720.47Nc5I.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Internal reference to :file:`shlwapi.dll` was dropped to help improve
-startup time. This DLL will no longer be loaded at the start of every Python
-process.
diff --git a/Misc/NEWS.d/next/Windows/2021-11-08-21-53-11.bpo-45732.idl5kx.rst b/Misc/NEWS.d/next/Windows/2021-11-08-21-53-11.bpo-45732.idl5kx.rst
deleted file mode 100644
index 563bcd33a401a..0000000000000
--- a/Misc/NEWS.d/next/Windows/2021-11-08-21-53-11.bpo-45732.idl5kx.rst
+++ /dev/null
@@ -1 +0,0 @@
-Updates bundled Tcl/Tk to 8.6.12.
diff --git a/Misc/NEWS.d/next/Windows/2021-11-23-11-44-42.bpo-45616.K52PLZ.rst b/Misc/NEWS.d/next/Windows/2021-11-23-11-44-42.bpo-45616.K52PLZ.rst
deleted file mode 100644
index 000f2fd3cc989..0000000000000
--- a/Misc/NEWS.d/next/Windows/2021-11-23-11-44-42.bpo-45616.K52PLZ.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-Fix Python Launcher's ability to distinguish between versions 3.1 and 3.10
-when either one is explicitly requested.  Previously, 3.1 would be used if
-3.10 was requested but not installed, and 3.10 would be used if 3.1 was
-requested but 3.10 was installed.
diff --git a/Misc/NEWS.d/next/Windows/2021-11-26-18-17-41.bpo-45901.c5IBqM.rst b/Misc/NEWS.d/next/Windows/2021-11-26-18-17-41.bpo-45901.c5IBqM.rst
deleted file mode 100644
index 2cb872bffe072..0000000000000
--- a/Misc/NEWS.d/next/Windows/2021-11-26-18-17-41.bpo-45901.c5IBqM.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-When installed through the Microsoft Store and set as the default app for
-:file:`*.py` files, command line arguments will now be passed to Python when
-invoking a script without explicitly launching Python (that is, ``script.py
-args`` rather than ``python script.py args``).
diff --git a/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst b/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst
deleted file mode 100644
index 29a6ff92554e1..0000000000000
--- a/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-When building CPython on macOS with ``./configure
---with-undefined-behavior-sanitizer --with-pydebug``, the stack size is now
-quadrupled to allow for the entire test suite to pass.
diff --git a/Misc/NEWS.d/next/macOS/2021-10-25-02-02-21.bpo-44828.XBdXlJ.rst b/Misc/NEWS.d/next/macOS/2021-10-25-02-02-21.bpo-44828.XBdXlJ.rst
deleted file mode 100644
index 021d7e4d73782..0000000000000
--- a/Misc/NEWS.d/next/macOS/2021-10-25-02-02-21.bpo-44828.XBdXlJ.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Avoid tkinter file dialog failure on macOS 12 Monterey when using the Tk
-8.6.11 provided by python.org macOS installers. Patch by Marc Culler of the
-Tk project.
diff --git a/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst b/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
deleted file mode 100644
index eb47985f86f95..0000000000000
--- a/Misc/NEWS.d/next/macOS/2021-12-05-23-52-03.bpo-45732.-BWrnh.rst
+++ /dev/null
@@ -1 +0,0 @@
-Update python.org macOS installer to use Tcl/Tk 8.6.12.
diff --git a/README.rst b/README.rst
index 74e7629eb4037..d2189e5c361e6 100644
--- a/README.rst
+++ b/README.rst
@@ -1,4 +1,4 @@
-This is Python version 3.10.0
+This is Python version 3.10.1
 =============================
 
 .. image:: https://travis-ci.com/python/cpython.svg?branch=master


From webhook-mailer at python.org  Mon Dec  6 18:12:40 2021
From: webhook-mailer at python.org (asvetlov)
Date: Mon, 06 Dec 2021 23:12:40 -0000
Subject: [Python-checkins] removal of duplicated text paragraph (#29666)
Message-ID: <mailman.64.1638832361.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/8518ee348c18da7e150f5e42b3424c86f7c0a3d8
commit: 8518ee348c18da7e150f5e42b3424c86f7c0a3d8
branch: main
author: Taras Sereda <taras.y.sereda at protonmail.com>
committer: asvetlov <andrew.svetlov at gmail.com>
date: 2021-12-07T01:12:29+02:00
summary:

removal of duplicated text paragraph (#29666)

files:
M Doc/library/asyncio-task.rst

diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 36e13f9295652..bfc983e304bcc 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -299,12 +299,6 @@ Sleeping
    tasks to run. This can be used by long-running functions to avoid
    blocking the event loop for the full duration of the function call.
 
-   .. deprecated-removed:: 3.8 3.10
-      The ``loop`` parameter.  This function has been implicitly getting the
-      current running loop since 3.7.  See
-      :ref:`What's New in 3.10's Removed section <whatsnew310-removed>`
-      for more information.
-
    .. _asyncio_example_sleep:
 
    Example of coroutine displaying the current date every second


From webhook-mailer at python.org  Mon Dec  6 18:40:41 2021
From: webhook-mailer at python.org (asvetlov)
Date: Mon, 06 Dec 2021 23:40:41 -0000
Subject: [Python-checkins] bpo-23819: asyncio: Replace AssertionError with
 TypeError where it makes sense (GH-29894)
Message-ID: <mailman.65.1638834042.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/265918bb1d782ab85c7dbc835eb62d6cfc2145b7
commit: 265918bb1d782ab85c7dbc835eb62d6cfc2145b7
branch: main
author: Kumar Aditya <rahuladitya303 at gmail.com>
committer: asvetlov <andrew.svetlov at gmail.com>
date: 2021-12-07T01:40:35+02:00
summary:

bpo-23819: asyncio: Replace AssertionError with TypeError where it makes sense (GH-29894)

files:
A Misc/NEWS.d/next/Library/2021-12-02-14-37-30.bpo-23819.An6vkT.rst
M Lib/asyncio/base_events.py
M Lib/asyncio/events.py
M Lib/test/test_asyncio/test_base_events.py
M Lib/test/test_asyncio/test_events.py

diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
index 054d7b45ec2d6..cfaf082587bb2 100644
--- a/Lib/asyncio/base_events.py
+++ b/Lib/asyncio/base_events.py
@@ -706,6 +706,8 @@ def call_later(self, delay, callback, *args, context=None):
         Any positional arguments after the callback will be passed to
         the callback when it is called.
         """
+        if delay is None:
+            raise TypeError('delay must not be None')
         timer = self.call_at(self.time() + delay, callback, *args,
                              context=context)
         if timer._source_traceback:
@@ -717,6 +719,8 @@ def call_at(self, when, callback, *args, context=None):
 
         Absolute time corresponds to the event loop's time() method.
         """
+        if when is None:
+            raise TypeError("when cannot be None")
         self._check_closed()
         if self._debug:
             self._check_thread()
diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py
index 7abaaca2d2b28..d91fe8db2b020 100644
--- a/Lib/asyncio/events.py
+++ b/Lib/asyncio/events.py
@@ -101,7 +101,6 @@ class TimerHandle(Handle):
     __slots__ = ['_scheduled', '_when']
 
     def __init__(self, when, callback, args, loop, context=None):
-        assert when is not None
         super().__init__(callback, args, loop, context)
         if self._source_traceback:
             del self._source_traceback[-1]
@@ -661,7 +660,8 @@ def get_event_loop(self):
     def set_event_loop(self, loop):
         """Set the event loop."""
         self._local._set_called = True
-        assert loop is None or isinstance(loop, AbstractEventLoop)
+        if loop is not None and not isinstance(loop, AbstractEventLoop):
+            raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'")
         self._local._loop = loop
 
     def new_event_loop(self):
@@ -745,7 +745,8 @@ def set_event_loop_policy(policy):
 
     If policy is None, the default policy is restored."""
     global _event_loop_policy
-    assert policy is None or isinstance(policy, AbstractEventLoopPolicy)
+    if policy is not None and not isinstance(policy, AbstractEventLoopPolicy):
+        raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'")
     _event_loop_policy = policy
 
 
diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
index b522fac23a23b..d812bc9edea57 100644
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -255,6 +255,8 @@ def cb():
         self.assertIsInstance(h, asyncio.TimerHandle)
         self.assertIn(h, self.loop._scheduled)
         self.assertNotIn(h, self.loop._ready)
+        with self.assertRaises(TypeError, msg="delay must not be None"):
+            self.loop.call_later(None, cb)
 
     def test_call_later_negative_delays(self):
         calls = []
@@ -286,6 +288,8 @@ def cb():
         # tolerate a difference of +800 ms because some Python buildbots
         # are really slow
         self.assertLessEqual(dt, 0.9, dt)
+        with self.assertRaises(TypeError, msg="when cannot be None"):
+            self.loop.call_at(None, cb)
 
     def check_thread(self, loop, debug):
         def cb():
diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
index e50a53d706784..fe791fa4232c0 100644
--- a/Lib/test/test_asyncio/test_events.py
+++ b/Lib/test/test_asyncio/test_events.py
@@ -2322,10 +2322,6 @@ def callback(*args):
         self.assertIsNone(h._callback)
         self.assertIsNone(h._args)
 
-        # when cannot be None
-        self.assertRaises(AssertionError,
-                          asyncio.TimerHandle, None, callback, args,
-                          self.loop)
 
     def test_timer_repr(self):
         self.loop.get_debug.return_value = False
@@ -2592,7 +2588,7 @@ def test_set_event_loop(self):
         policy = asyncio.DefaultEventLoopPolicy()
         old_loop = policy.get_event_loop()
 
-        self.assertRaises(AssertionError, policy.set_event_loop, object())
+        self.assertRaises(TypeError, policy.set_event_loop, object())
 
         loop = policy.new_event_loop()
         policy.set_event_loop(loop)
@@ -2608,7 +2604,7 @@ def test_get_event_loop_policy(self):
 
     def test_set_event_loop_policy(self):
         self.assertRaises(
-            AssertionError, asyncio.set_event_loop_policy, object())
+            TypeError, asyncio.set_event_loop_policy, object())
 
         old_policy = asyncio.get_event_loop_policy()
 
diff --git a/Misc/NEWS.d/next/Library/2021-12-02-14-37-30.bpo-23819.An6vkT.rst b/Misc/NEWS.d/next/Library/2021-12-02-14-37-30.bpo-23819.An6vkT.rst
new file mode 100644
index 0000000000000..d1ec505193d41
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-12-02-14-37-30.bpo-23819.An6vkT.rst
@@ -0,0 +1 @@
+Replaced asserts with exceptions in asyncio, patch by Kumar Aditya.
\ No newline at end of file


From webhook-mailer at python.org  Mon Dec  6 19:07:46 2021
From: webhook-mailer at python.org (zooba)
Date: Tue, 07 Dec 2021 00:07:46 -0000
Subject: [Python-checkins] bpo-45582: Ensure PYTHONHOME still overrides
 detected build prefixes (GH-29948)
Message-ID: <mailman.66.1638835667.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/b7ef27bc084665ce58d89fc69530c6f9d2d37754
commit: b7ef27bc084665ce58d89fc69530c6f9d2d37754
branch: main
author: Steve Dower <steve.dower at python.org>
committer: zooba <steve.dower at microsoft.com>
date: 2021-12-07T00:07:35Z
summary:

bpo-45582: Ensure PYTHONHOME still overrides detected build prefixes (GH-29948)

files:
M Lib/test/test_getpath.py
M Modules/getpath.py

diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py
index c18689c0590d4..9dd167b9975c0 100644
--- a/Lib/test/test_getpath.py
+++ b/Lib/test/test_getpath.py
@@ -208,6 +208,36 @@ def test_symlink_buildtree_win32(self):
         actual = getpath(ns, expected)
         self.assertEqual(expected, actual)
 
+    def test_buildtree_pythonhome_win32(self):
+        "Test an out-of-build-tree layout on Windows with PYTHONHOME override."
+        ns = MockNTNamespace(
+            argv0=r"C:\Out\python.exe",
+            real_executable=r"C:\Out\python.exe",
+            ENV_PYTHONHOME=r"C:\CPython",
+        )
+        ns.add_known_xfile(r"C:\Out\python.exe")
+        ns.add_known_file(r"C:\CPython\Lib\os.py")
+        ns.add_known_file(r"C:\Out\pybuilddir.txt", [""])
+        expected = dict(
+            executable=r"C:\Out\python.exe",
+            base_executable=r"C:\Out\python.exe",
+            prefix=r"C:\CPython",
+            exec_prefix=r"C:\CPython",
+            # This build_prefix is a miscalculation, because we have
+            # moved the output direction out of the prefix.
+            # Specify PYTHONHOME to get the correct prefix/exec_prefix
+            build_prefix="C:\\",
+            _is_python_build=1,
+            module_search_paths_set=1,
+            module_search_paths=[
+                r"C:\Out\python98.zip",
+                r"C:\CPython\Lib",
+                r"C:\Out",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
     def test_normal_posix(self):
         "Test a 'standard' install layout on *nix"
         ns = MockPosixNamespace(
diff --git a/Modules/getpath.py b/Modules/getpath.py
index 2eadfba1dfda2..4ef49a8847dd2 100644
--- a/Modules/getpath.py
+++ b/Modules/getpath.py
@@ -500,6 +500,8 @@ def search_up(prefix, *landmarks, test=isfile):
         prefix, had_delim, exec_prefix = home.partition(DELIM)
         if not had_delim:
             exec_prefix = prefix
+        # Reset the standard library directory if it was already set
+        stdlib_dir = None
 
 
     # First try to detect prefix by looking alongside our runtime library, if known


From webhook-mailer at python.org  Mon Dec  6 19:10:57 2021
From: webhook-mailer at python.org (miss-islington)
Date: Tue, 07 Dec 2021 00:10:57 -0000
Subject: [Python-checkins] bpo-28953: Use `raise from` when raising new
 IncompleteRead (GH-29861)
Message-ID: <mailman.67.1638835858.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/c5c365220ed2c867fe81078f70b827de22db2ee6
commit: c5c365220ed2c867fe81078f70b827de22db2ee6
branch: main
author: 180909 <734461790 at qq.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-06T16:10:49-08:00
summary:

bpo-28953: Use `raise from` when raising new IncompleteRead (GH-29861)



Automerge-Triggered-By: GH:asvetlov

files:
M Lib/http/client.py

diff --git a/Lib/http/client.py b/Lib/http/client.py
index a6ab135b2c387..f54172fd0deea 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -593,8 +593,8 @@ def _read_chunked(self, amt=None):
                     amt -= chunk_left
                 self.chunk_left = 0
             return b''.join(value)
-        except IncompleteRead:
-            raise IncompleteRead(b''.join(value))
+        except IncompleteRead as exc:
+            raise IncompleteRead(b''.join(value)) from exc
 
     def _readinto_chunked(self, b):
         assert self.chunked != _UNKNOWN


From webhook-mailer at python.org  Mon Dec  6 21:35:55 2021
From: webhook-mailer at python.org (ned-deily)
Date: Tue, 07 Dec 2021 02:35:55 -0000
Subject: [Python-checkins] bpo-45798: Let libmpdec decide which archs to
 build on macOS as done previously. (GH-29949)
Message-ID: <mailman.68.1638844557.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/ddbab69b6d44085564a9b5022b96b002a52b2f2b
commit: ddbab69b6d44085564a9b5022b96b002a52b2f2b
branch: main
author: Ned Deily <nad at python.org>
committer: ned-deily <nad at python.org>
date: 2021-12-06T21:35:50-05:00
summary:

bpo-45798: Let libmpdec decide which archs to build on macOS as done previously. (GH-29949)

files:
M configure
M configure.ac

diff --git a/configure b/configure
index 2237e6ed8ce2a..608055d91fac3 100755
--- a/configure
+++ b/configure
@@ -11815,21 +11815,9 @@ esac
 
 libmpdec_machine=unknown
 if test "$libmpdec_system" = Darwin; then
-    case $MACOSX_DEFAULT_ARCH in #(
-  i386) :
-    libmpdec_machine=ansi32 ;; #(
-  ppc) :
-    libmpdec_machine=ansi32 ;; #(
-  x86_64) :
-    libmpdec_machine=x64 ;; #(
-  ppc64) :
-    libmpdec_machine=ansi64 ;; #(
-  arm64) :
-    libmpdec_machine=ansi64 ;; #(
-  *) :
+    # universal here means: build libmpdec with the same arch options
+    # the python interpreter was built with
     libmpdec_machine=universal
-     ;;
-esac
 elif test $ac_cv_sizeof_size_t -eq 8; then
     if test "$ac_cv_gcc_asm_for_x64" = yes; then
         libmpdec_machine=x64
diff --git a/configure.ac b/configure.ac
index f1aac2db71f5d..7cc6c0c2f592a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3295,14 +3295,9 @@ AS_CASE([$ac_sys_system],
 
 libmpdec_machine=unknown
 if test "$libmpdec_system" = Darwin; then
-    AS_CASE([$MACOSX_DEFAULT_ARCH],
-      [i386],   [libmpdec_machine=ansi32],
-      [ppc],    [libmpdec_machine=ansi32],
-      [x86_64], [libmpdec_machine=x64],
-      [ppc64],  [libmpdec_machine=ansi64],
-      [arm64],  [libmpdec_machine=ansi64],
-      [libmpdec_machine=universal]
-    )
+    # universal here means: build libmpdec with the same arch options
+    # the python interpreter was built with
+    libmpdec_machine=universal
 elif test $ac_cv_sizeof_size_t -eq 8; then
     if test "$ac_cv_gcc_asm_for_x64" = yes; then
         libmpdec_machine=x64


From webhook-mailer at python.org  Tue Dec  7 04:58:57 2021
From: webhook-mailer at python.org (serhiy-storchaka)
Date: Tue, 07 Dec 2021 09:58:57 -0000
Subject: [Python-checkins] bpo-46001: Change OverflowError to RecursionError
 in JSON library docstrings (GH-29943)
Message-ID: <mailman.69.1638871138.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/8db06528cacc94e67eb1fb2e4c2acc061a515671
commit: 8db06528cacc94e67eb1fb2e4c2acc061a515671
branch: main
author: James Gerity <snoopjedi at gmail.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2021-12-07T11:58:40+02:00
summary:

bpo-46001: Change OverflowError to RecursionError in JSON library docstrings (GH-29943)

files:
M Doc/library/json.rst
M Lib/json/__init__.py
M Lib/json/encoder.py

diff --git a/Doc/library/json.rst b/Doc/library/json.rst
index 6fa89f578a2cf..1810e04cc8349 100644
--- a/Doc/library/json.rst
+++ b/Doc/library/json.rst
@@ -159,7 +159,7 @@ Basic Usage
 
    If *check_circular* is false (default: ``True``), then the circular
    reference check for container types will be skipped and a circular reference
-   will result in an :exc:`OverflowError` (or worse).
+   will result in an :exc:`RecursionError` (or worse).
 
    If *allow_nan* is false (default: ``True``), then it will be a
    :exc:`ValueError` to serialize out of range :class:`float` values (``nan``,
@@ -432,7 +432,7 @@ Encoders and Decoders
 
    If *check_circular* is true (the default), then lists, dicts, and custom
    encoded objects will be checked for circular references during encoding to
-   prevent an infinite recursion (which would cause an :exc:`OverflowError`).
+   prevent an infinite recursion (which would cause an :exc:`RecursionError`).
    Otherwise, no such check takes place.
 
    If *allow_nan* is true (the default), then ``NaN``, ``Infinity``, and
diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py
index 2c52bdeba6754..e4c21daaf3e47 100644
--- a/Lib/json/__init__.py
+++ b/Lib/json/__init__.py
@@ -133,7 +133,7 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True,
 
     If ``check_circular`` is false, then the circular reference check
     for container types will be skipped and a circular reference will
-    result in an ``OverflowError`` (or worse).
+    result in an ``RecursionError`` (or worse).
 
     If ``allow_nan`` is false, then it will be a ``ValueError`` to
     serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
@@ -195,7 +195,7 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True,
 
     If ``check_circular`` is false, then the circular reference check
     for container types will be skipped and a circular reference will
-    result in an ``OverflowError`` (or worse).
+    result in an ``RecursionError`` (or worse).
 
     If ``allow_nan`` is false, then it will be a ``ValueError`` to
     serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py
index c8c78b9c23765..21bff2c1a1fca 100644
--- a/Lib/json/encoder.py
+++ b/Lib/json/encoder.py
@@ -116,7 +116,7 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True,
 
         If check_circular is true, then lists, dicts, and custom encoded
         objects will be checked for circular references during encoding to
-        prevent an infinite recursion (which would cause an OverflowError).
+        prevent an infinite recursion (which would cause an RecursionError).
         Otherwise, no such check takes place.
 
         If allow_nan is true, then NaN, Infinity, and -Infinity will be


From webhook-mailer at python.org  Tue Dec  7 05:20:21 2021
From: webhook-mailer at python.org (miss-islington)
Date: Tue, 07 Dec 2021 10:20:21 -0000
Subject: [Python-checkins] bpo-46001: Change OverflowError to RecursionError
 in JSON library docstrings (GH-29943)
Message-ID: <mailman.70.1638872421.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/15da2a2723245710f1bd2c7cbd5b450532ae7728
commit: 15da2a2723245710f1bd2c7cbd5b450532ae7728
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-07T02:20:10-08:00
summary:

bpo-46001: Change OverflowError to RecursionError in JSON library docstrings (GH-29943)

(cherry picked from commit 8db06528cacc94e67eb1fb2e4c2acc061a515671)

Co-authored-by: James Gerity <snoopjedi at gmail.com>

files:
M Doc/library/json.rst
M Lib/json/__init__.py
M Lib/json/encoder.py

diff --git a/Doc/library/json.rst b/Doc/library/json.rst
index 6fa89f578a2cf..1810e04cc8349 100644
--- a/Doc/library/json.rst
+++ b/Doc/library/json.rst
@@ -159,7 +159,7 @@ Basic Usage
 
    If *check_circular* is false (default: ``True``), then the circular
    reference check for container types will be skipped and a circular reference
-   will result in an :exc:`OverflowError` (or worse).
+   will result in an :exc:`RecursionError` (or worse).
 
    If *allow_nan* is false (default: ``True``), then it will be a
    :exc:`ValueError` to serialize out of range :class:`float` values (``nan``,
@@ -432,7 +432,7 @@ Encoders and Decoders
 
    If *check_circular* is true (the default), then lists, dicts, and custom
    encoded objects will be checked for circular references during encoding to
-   prevent an infinite recursion (which would cause an :exc:`OverflowError`).
+   prevent an infinite recursion (which would cause an :exc:`RecursionError`).
    Otherwise, no such check takes place.
 
    If *allow_nan* is true (the default), then ``NaN``, ``Infinity``, and
diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py
index 2c52bdeba6754..e4c21daaf3e47 100644
--- a/Lib/json/__init__.py
+++ b/Lib/json/__init__.py
@@ -133,7 +133,7 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True,
 
     If ``check_circular`` is false, then the circular reference check
     for container types will be skipped and a circular reference will
-    result in an ``OverflowError`` (or worse).
+    result in an ``RecursionError`` (or worse).
 
     If ``allow_nan`` is false, then it will be a ``ValueError`` to
     serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
@@ -195,7 +195,7 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True,
 
     If ``check_circular`` is false, then the circular reference check
     for container types will be skipped and a circular reference will
-    result in an ``OverflowError`` (or worse).
+    result in an ``RecursionError`` (or worse).
 
     If ``allow_nan`` is false, then it will be a ``ValueError`` to
     serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py
index c8c78b9c23765..21bff2c1a1fca 100644
--- a/Lib/json/encoder.py
+++ b/Lib/json/encoder.py
@@ -116,7 +116,7 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True,
 
         If check_circular is true, then lists, dicts, and custom encoded
         objects will be checked for circular references during encoding to
-        prevent an infinite recursion (which would cause an OverflowError).
+        prevent an infinite recursion (which would cause an RecursionError).
         Otherwise, no such check takes place.
 
         If allow_nan is true, then NaN, Infinity, and -Infinity will be


From webhook-mailer at python.org  Tue Dec  7 05:25:07 2021
From: webhook-mailer at python.org (miss-islington)
Date: Tue, 07 Dec 2021 10:25:07 -0000
Subject: [Python-checkins] bpo-46001: Change OverflowError to RecursionError
 in JSON library docstrings (GH-29943)
Message-ID: <mailman.71.1638872709.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/2e360832d7ed2697d715e93cb9f859a52264d60b
commit: 2e360832d7ed2697d715e93cb9f859a52264d60b
branch: 3.9
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-07T02:25:02-08:00
summary:

bpo-46001: Change OverflowError to RecursionError in JSON library docstrings (GH-29943)

(cherry picked from commit 8db06528cacc94e67eb1fb2e4c2acc061a515671)

Co-authored-by: James Gerity <snoopjedi at gmail.com>

files:
M Doc/library/json.rst
M Lib/json/__init__.py
M Lib/json/encoder.py

diff --git a/Doc/library/json.rst b/Doc/library/json.rst
index 6fa89f578a2cf..1810e04cc8349 100644
--- a/Doc/library/json.rst
+++ b/Doc/library/json.rst
@@ -159,7 +159,7 @@ Basic Usage
 
    If *check_circular* is false (default: ``True``), then the circular
    reference check for container types will be skipped and a circular reference
-   will result in an :exc:`OverflowError` (or worse).
+   will result in an :exc:`RecursionError` (or worse).
 
    If *allow_nan* is false (default: ``True``), then it will be a
    :exc:`ValueError` to serialize out of range :class:`float` values (``nan``,
@@ -432,7 +432,7 @@ Encoders and Decoders
 
    If *check_circular* is true (the default), then lists, dicts, and custom
    encoded objects will be checked for circular references during encoding to
-   prevent an infinite recursion (which would cause an :exc:`OverflowError`).
+   prevent an infinite recursion (which would cause an :exc:`RecursionError`).
    Otherwise, no such check takes place.
 
    If *allow_nan* is true (the default), then ``NaN``, ``Infinity``, and
diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py
index 2c52bdeba6754..e4c21daaf3e47 100644
--- a/Lib/json/__init__.py
+++ b/Lib/json/__init__.py
@@ -133,7 +133,7 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True,
 
     If ``check_circular`` is false, then the circular reference check
     for container types will be skipped and a circular reference will
-    result in an ``OverflowError`` (or worse).
+    result in an ``RecursionError`` (or worse).
 
     If ``allow_nan`` is false, then it will be a ``ValueError`` to
     serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
@@ -195,7 +195,7 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True,
 
     If ``check_circular`` is false, then the circular reference check
     for container types will be skipped and a circular reference will
-    result in an ``OverflowError`` (or worse).
+    result in an ``RecursionError`` (or worse).
 
     If ``allow_nan`` is false, then it will be a ``ValueError`` to
     serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py
index c8c78b9c23765..21bff2c1a1fca 100644
--- a/Lib/json/encoder.py
+++ b/Lib/json/encoder.py
@@ -116,7 +116,7 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True,
 
         If check_circular is true, then lists, dicts, and custom encoded
         objects will be checked for circular references during encoding to
-        prevent an infinite recursion (which would cause an OverflowError).
+        prevent an infinite recursion (which would cause an RecursionError).
         Otherwise, no such check takes place.
 
         If allow_nan is true, then NaN, Infinity, and -Infinity will be


From webhook-mailer at python.org  Tue Dec  7 05:50:52 2021
From: webhook-mailer at python.org (markshannon)
Date: Tue, 07 Dec 2021 10:50:52 -0000
Subject: [Python-checkins] bpo-45890: Add tests for tracing
 try-except-finally blocks (GH-29746)
Message-ID: <mailman.72.1638874252.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/a310fd83a014484b8c680de83540c4908b344c6c
commit: a310fd83a014484b8c680de83540c4908b344c6c
branch: main
author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com>
committer: markshannon <mark at hotpy.org>
date: 2021-12-07T10:50:37Z
summary:

bpo-45890: Add tests for tracing try-except-finally blocks (GH-29746)

files:
M Lib/test/test_sys_settrace.py
M Lib/test/test_trace.py

diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py
index b565bef4c4423..15c33a28ff2ac 100644
--- a/Lib/test/test_sys_settrace.py
+++ b/Lib/test/test_sys_settrace.py
@@ -642,15 +642,18 @@ def func():
                 2
             except:
                 4
-            finally:
+            else:
                 6
+            finally:
+                8
 
         self.run_and_compare(func,
             [(0, 'call'),
              (1, 'line'),
              (2, 'line'),
              (6, 'line'),
-             (6, 'return')])
+             (8, 'line'),
+             (8, 'return')])
 
     def test_nested_loops(self):
 
@@ -1016,6 +1019,47 @@ def func():
              (3, 'line'),
              (3, 'return')])
 
+    def test_try_in_try_with_exception(self):
+
+        def func():
+            try:
+                try:
+                    raise TypeError
+                except ValueError as ex:
+                    5
+            except TypeError:
+                7
+
+        self.run_and_compare(func,
+            [(0, 'call'),
+             (1, 'line'),
+             (2, 'line'),
+             (3, 'line'),
+             (3, 'exception'),
+             (4, 'line'),
+             (6, 'line'),
+             (7, 'line'),
+             (7, 'return')])
+
+        def func():
+            try:
+                try:
+                    raise ValueError
+                except ValueError as ex:
+                    5
+            except TypeError:
+                7
+
+        self.run_and_compare(func,
+            [(0, 'call'),
+             (1, 'line'),
+             (2, 'line'),
+             (3, 'line'),
+             (3, 'exception'),
+             (4, 'line'),
+             (5, 'line'),
+             (5, 'return')])
+
     def test_if_in_if_in_if(self):
         def func(a=0, p=1, z=1):
             if p:
diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py
index dbfefca7ee5cd..d63c1778c9d08 100644
--- a/Lib/test/test_trace.py
+++ b/Lib/test/test_trace.py
@@ -11,6 +11,11 @@
 
 from test.tracedmodules import testmod
 
+##
+## See also test_sys_settrace.py, which contains tests that cover
+## tracing of many more code blocks.
+##
+
 #------------------------------- Utilities -----------------------------------#
 
 def fix_ext_py(filename):


From webhook-mailer at python.org  Tue Dec  7 06:15:54 2021
From: webhook-mailer at python.org (vsajip)
Date: Tue, 07 Dec 2021 11:15:54 -0000
Subject: [Python-checkins] bpo-35821: Add an example to Logger.propagate
 documentation. (GH-29841)
Message-ID: <mailman.73.1638875755.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/2bf551757e0a7e3cc6ce2ebed2178b82438ac6b5
commit: 2bf551757e0a7e3cc6ce2ebed2178b82438ac6b5
branch: main
author: Vinay Sajip <vinay_sajip at yahoo.co.uk>
committer: vsajip <vinay_sajip at yahoo.co.uk>
date: 2021-12-07T11:15:44Z
summary:

bpo-35821: Add an example to Logger.propagate documentation. (GH-29841)

files:
M Doc/library/logging.rst

diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
index 3f3a8534f54d0..e7c64e5ba4036 100644
--- a/Doc/library/logging.rst
+++ b/Doc/library/logging.rst
@@ -80,6 +80,15 @@ is the module's name in the Python package namespace.
       If this evaluates to false, logging messages are not passed to the handlers
       of ancestor loggers.
 
+      Spelling it out with an example: If the propagate attribute of the logger named
+      `A.B.C` evaluates to true, any event logged to `A.B.C` via a method call such as
+      `logging.getLogger('A.B.C').error(...)` will [subject to passing that logger's
+      level and filter settings] be passed in turn to any handlers attached to loggers
+      named `A.B`, `A` and the root logger, after first being passed to any handlers
+      attached to `A.B.C`. If any logger in the chain `A.B.C`, `A.B`, `A` has its
+      `propagate` attribute set to false, then that is the last logger whose handlers
+      are offered the event to handle, and propagation stops at that point.
+
       The constructor sets this attribute to ``True``.
 
       .. note:: If you attach a handler to a logger *and* one or more of its


From webhook-mailer at python.org  Tue Dec  7 06:31:11 2021
From: webhook-mailer at python.org (vstinner)
Date: Tue, 07 Dec 2021 11:31:11 -0000
Subject: [Python-checkins] Revert "bpo-28533: Remove asyncore, asynchat,
 smtpd modules (GH-29521)" (GH-29951)
Message-ID: <mailman.74.1638876673.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/cf7eaa4617295747ee5646c4e2b7e7a16d7c64ab
commit: cf7eaa4617295747ee5646c4e2b7e7a16d7c64ab
branch: main
author: Victor Stinner <vstinner at python.org>
committer: vstinner <vstinner at python.org>
date: 2021-12-07T12:31:04+01:00
summary:

Revert "bpo-28533: Remove asyncore, asynchat, smtpd modules (GH-29521)" (GH-29951)

This reverts commit 9bf2cbc4c498812e14f20d86acb61c53928a5a57.

files:
A Doc/library/asynchat.rst
A Doc/library/asyncore.rst
A Doc/library/smtpd.rst
A Lib/asynchat.py
A Lib/asyncore.py
A Lib/smtpd.py
A Lib/test/test_asynchat.py
A Lib/test/test_asyncore.py
A Lib/test/test_smtpd.py
D Lib/test/support/_asynchat.py
D Lib/test/support/_asyncore.py
D Lib/test/support/_smtpd.py
D Misc/NEWS.d/next/Library/2021-11-11-12-59-10.bpo-28533.68mMZa.rst
D Misc/NEWS.d/next/Library/2021-11-11-12-59-49.bpo-28533.LvIFCQ.rst
M .github/CODEOWNERS
M Doc/library/email.rst
M Doc/library/internet.rst
M Doc/library/ipc.rst
M Doc/library/socketserver.rst
M Doc/license.rst
M Doc/whatsnew/3.11.rst
M Lib/test/libregrtest/save_env.py
M Lib/test/mock_socket.py
M Lib/test/test_ftplib.py
M Lib/test/test_logging.py
M Lib/test/test_os.py
M Lib/test/test_poplib.py
M Lib/test/test_smtplib.py
M Lib/test/test_ssl.py
M PCbuild/lib.pyproj
M Python/stdlib_module_names.h

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 82f81e3452995..ce5121e7ac8f8 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -130,6 +130,8 @@ Lib/ast.py                    @isidentical
 
 **/*typing*                   @gvanrossum @Fidget-Spinner
 
+**/*asyncore                  @giampaolo
+**/*asynchat                  @giampaolo
 **/*ftplib                    @giampaolo
 **/*shutil                    @giampaolo
 
diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst
new file mode 100644
index 0000000000000..9e51416b83a57
--- /dev/null
+++ b/Doc/library/asynchat.rst
@@ -0,0 +1,213 @@
+:mod:`asynchat` --- Asynchronous socket command/response handler
+================================================================
+
+.. module:: asynchat
+   :synopsis: Support for asynchronous command/response protocols.
+
+.. moduleauthor:: Sam Rushing <rushing at nightmare.com>
+.. sectionauthor:: Steve Holden <sholden at holdenweb.com>
+
+**Source code:** :source:`Lib/asynchat.py`
+
+.. deprecated:: 3.6
+   Please use :mod:`asyncio` instead.
+
+--------------
+
+.. note::
+
+   This module exists for backwards compatibility only.  For new code we
+   recommend using :mod:`asyncio`.
+
+This module builds on the :mod:`asyncore` infrastructure, simplifying
+asynchronous clients and servers and making it easier to handle protocols
+whose elements are terminated by arbitrary strings, or are of variable length.
+:mod:`asynchat` defines the abstract class :class:`async_chat` that you
+subclass, providing implementations of the :meth:`collect_incoming_data` and
+:meth:`found_terminator` methods. It uses the same asynchronous loop as
+:mod:`asyncore`, and the two types of channel, :class:`asyncore.dispatcher`
+and :class:`asynchat.async_chat`, can freely be mixed in the channel map.
+Typically an :class:`asyncore.dispatcher` server channel generates new
+:class:`asynchat.async_chat` channel objects as it receives incoming
+connection requests.
+
+
+.. class:: async_chat()
+
+   This class is an abstract subclass of :class:`asyncore.dispatcher`. To make
+   practical use of the code you must subclass :class:`async_chat`, providing
+   meaningful :meth:`collect_incoming_data` and :meth:`found_terminator`
+   methods.
+   The :class:`asyncore.dispatcher` methods can be used, although not all make
+   sense in a message/response context.
+
+   Like :class:`asyncore.dispatcher`, :class:`async_chat` defines a set of
+   events that are generated by an analysis of socket conditions after a
+   :c:func:`select` call. Once the polling loop has been started the
+   :class:`async_chat` object's methods are called by the event-processing
+   framework with no action on the part of the programmer.
+
+   Two class attributes can be modified, to improve performance, or possibly
+   even to conserve memory.
+
+
+   .. data:: ac_in_buffer_size
+
+      The asynchronous input buffer size (default ``4096``).
+
+
+   .. data:: ac_out_buffer_size
+
+      The asynchronous output buffer size (default ``4096``).
+
+   Unlike :class:`asyncore.dispatcher`, :class:`async_chat` allows you to
+   define a :abbr:`FIFO (first-in, first-out)` queue of *producers*. A producer need
+   have only one method, :meth:`more`, which should return data to be
+   transmitted on the channel.
+   The producer indicates exhaustion (*i.e.* that it contains no more data) by
+   having its :meth:`more` method return the empty bytes object. At this point
+   the :class:`async_chat` object removes the producer from the queue and starts
+   using the next producer, if any. When the producer queue is empty the
+   :meth:`handle_write` method does nothing. You use the channel object's
+   :meth:`set_terminator` method to describe how to recognize the end of, or
+   an important breakpoint in, an incoming transmission from the remote
+   endpoint.
+
+   To build a functioning :class:`async_chat` subclass your  input methods
+   :meth:`collect_incoming_data` and :meth:`found_terminator` must handle the
+   data that the channel receives asynchronously. The methods are described
+   below.
+
+
+.. method:: async_chat.close_when_done()
+
+   Pushes a ``None`` on to the producer queue. When this producer is popped off
+   the queue it causes the channel to be closed.
+
+
+.. method:: async_chat.collect_incoming_data(data)
+
+   Called with *data* holding an arbitrary amount of received data.  The
+   default method, which must be overridden, raises a
+   :exc:`NotImplementedError` exception.
+
+
+.. method:: async_chat.discard_buffers()
+
+   In emergencies this method will discard any data held in the input and/or
+   output buffers and the producer queue.
+
+
+.. method:: async_chat.found_terminator()
+
+   Called when the incoming data stream  matches the termination condition set
+   by :meth:`set_terminator`. The default method, which must be overridden,
+   raises a :exc:`NotImplementedError` exception. The buffered input data
+   should be available via an instance attribute.
+
+
+.. method:: async_chat.get_terminator()
+
+   Returns the current terminator for the channel.
+
+
+.. method:: async_chat.push(data)
+
+   Pushes data on to the channel's queue to ensure its transmission.
+   This is all you need to do to have the channel write the data out to the
+   network, although it is possible to use your own producers in more complex
+   schemes to implement encryption and chunking, for example.
+
+
+.. method:: async_chat.push_with_producer(producer)
+
+   Takes a producer object and adds it to the producer queue associated with
+   the channel.  When all currently-pushed producers have been exhausted the
+   channel will consume this producer's data by calling its :meth:`more`
+   method and send the data to the remote endpoint.
+
+
+.. method:: async_chat.set_terminator(term)
+
+   Sets the terminating condition to be recognized on the channel.  ``term``
+   may be any of three types of value, corresponding to three different ways
+   to handle incoming protocol data.
+
+   +-----------+---------------------------------------------+
+   | term      | Description                                 |
+   +===========+=============================================+
+   | *string*  | Will call :meth:`found_terminator` when the |
+   |           | string is found in the input stream         |
+   +-----------+---------------------------------------------+
+   | *integer* | Will call :meth:`found_terminator` when the |
+   |           | indicated number of characters have been    |
+   |           | received                                    |
+   +-----------+---------------------------------------------+
+   | ``None``  | The channel continues to collect data       |
+   |           | forever                                     |
+   +-----------+---------------------------------------------+
+
+   Note that any data following the terminator will be available for reading
+   by the channel after :meth:`found_terminator` is called.
+
+
+.. _asynchat-example:
+
+asynchat Example
+----------------
+
+The following partial example shows how HTTP requests can be read with
+:class:`async_chat`.  A web server might create an
+:class:`http_request_handler` object for each incoming client connection.
+Notice that initially the channel terminator is set to match the blank line at
+the end of the HTTP headers, and a flag indicates that the headers are being
+read.
+
+Once the headers have been read, if the request is of type POST (indicating
+that further data are present in the input stream) then the
+``Content-Length:`` header is used to set a numeric terminator to read the
+right amount of data from the channel.
+
+The :meth:`handle_request` method is called once all relevant input has been
+marshalled, after setting the channel terminator to ``None`` to ensure that
+any extraneous data sent by the web client are ignored. ::
+
+
+   import asynchat
+
+   class http_request_handler(asynchat.async_chat):
+
+       def __init__(self, sock, addr, sessions, log):
+           asynchat.async_chat.__init__(self, sock=sock)
+           self.addr = addr
+           self.sessions = sessions
+           self.ibuffer = []
+           self.obuffer = b""
+           self.set_terminator(b"\r\n\r\n")
+           self.reading_headers = True
+           self.handling = False
+           self.cgi_data = None
+           self.log = log
+
+       def collect_incoming_data(self, data):
+           """Buffer the data"""
+           self.ibuffer.append(data)
+
+       def found_terminator(self):
+           if self.reading_headers:
+               self.reading_headers = False
+               self.parse_headers(b"".join(self.ibuffer))
+               self.ibuffer = []
+               if self.op.upper() == b"POST":
+                   clen = self.headers.getheader("content-length")
+                   self.set_terminator(int(clen))
+               else:
+                   self.handling = True
+                   self.set_terminator(None)
+                   self.handle_request()
+           elif not self.handling:
+               self.set_terminator(None)  # browsers sometimes over-send
+               self.cgi_data = parse(self.headers, b"".join(self.ibuffer))
+               self.handling = True
+               self.ibuffer = []
+               self.handle_request()
diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst
new file mode 100644
index 0000000000000..a86518ebff277
--- /dev/null
+++ b/Doc/library/asyncore.rst
@@ -0,0 +1,360 @@
+:mod:`asyncore` --- Asynchronous socket handler
+===============================================
+
+.. module:: asyncore
+   :synopsis: A base class for developing asynchronous socket handling
+              services.
+
+.. moduleauthor:: Sam Rushing <rushing at nightmare.com>
+.. sectionauthor:: Christopher Petrilli <petrilli at amber.org>
+.. sectionauthor:: Steve Holden <sholden at holdenweb.com>
+.. heavily adapted from original documentation by Sam Rushing
+
+**Source code:** :source:`Lib/asyncore.py`
+
+.. deprecated:: 3.6
+   Please use :mod:`asyncio` instead.
+
+--------------
+
+.. note::
+
+   This module exists for backwards compatibility only.  For new code we
+   recommend using :mod:`asyncio`.
+
+This module provides the basic infrastructure for writing asynchronous  socket
+service clients and servers.
+
+There are only two ways to have a program on a single processor do  "more than
+one thing at a time." Multi-threaded programming is the  simplest and most
+popular way to do it, but there is another very different technique, that lets
+you have nearly all the advantages of  multi-threading, without actually using
+multiple threads.  It's really  only practical if your program is largely I/O
+bound.  If your program is processor bound, then pre-emptive scheduled threads
+are probably what you really need.  Network servers are rarely processor
+bound, however.
+
+If your operating system supports the :c:func:`select` system call in its I/O
+library (and nearly all do), then you can use it to juggle multiple
+communication channels at once; doing other work while your I/O is taking
+place in the "background."  Although this strategy can seem strange and
+complex, especially at first, it is in many ways easier to understand and
+control than multi-threaded programming.  The :mod:`asyncore` module solves
+many of the difficult problems for you, making the task of building
+sophisticated high-performance network servers and clients a snap.  For
+"conversational" applications and protocols the companion :mod:`asynchat`
+module is invaluable.
+
+The basic idea behind both modules is to create one or more network
+*channels*, instances of class :class:`asyncore.dispatcher` and
+:class:`asynchat.async_chat`.  Creating the channels adds them to a global
+map, used by the :func:`loop` function if you do not provide it with your own
+*map*.
+
+Once the initial channel(s) is(are) created, calling the :func:`loop` function
+activates channel service, which continues until the last channel (including
+any that have been added to the map during asynchronous service) is closed.
+
+
+.. function:: loop([timeout[, use_poll[, map[,count]]]])
+
+   Enter a polling loop that terminates after count passes or all open
+   channels have been closed.  All arguments are optional.  The *count*
+   parameter defaults to ``None``, resulting in the loop terminating only when all
+   channels have been closed.  The *timeout* argument sets the timeout
+   parameter for the appropriate :func:`~select.select` or :func:`~select.poll`
+   call, measured in seconds; the default is 30 seconds.  The *use_poll*
+   parameter, if true, indicates that :func:`~select.poll` should be used in
+   preference to :func:`~select.select` (the default is ``False``).
+
+   The *map* parameter is a dictionary whose items are the channels to watch.
+   As channels are closed they are deleted from their map.  If *map* is
+   omitted, a global map is used. Channels (instances of
+   :class:`asyncore.dispatcher`, :class:`asynchat.async_chat` and subclasses
+   thereof) can freely be mixed in the map.
+
+
+.. class:: dispatcher()
+
+   The :class:`dispatcher` class is a thin wrapper around a low-level socket
+   object. To make it more useful, it has a few methods for event-handling
+   which are called from the asynchronous loop.   Otherwise, it can be treated
+   as a normal non-blocking socket object.
+
+   The firing of low-level events at certain times or in certain connection
+   states tells the asynchronous loop that certain higher-level events have
+   taken place.  For example, if we have asked for a socket to connect to
+   another host, we know that the connection has been made when the socket
+   becomes writable for the first time (at this point you know that you may
+   write to it with the expectation of success).  The implied higher-level
+   events are:
+
+   +----------------------+----------------------------------------+
+   | Event                | Description                            |
+   +======================+========================================+
+   | ``handle_connect()`` | Implied by the first read or write     |
+   |                      | event                                  |
+   +----------------------+----------------------------------------+
+   | ``handle_close()``   | Implied by a read event with no data   |
+   |                      | available                              |
+   +----------------------+----------------------------------------+
+   | ``handle_accepted()``| Implied by a read event on a listening |
+   |                      | socket                                 |
+   +----------------------+----------------------------------------+
+
+   During asynchronous processing, each mapped channel's :meth:`readable` and
+   :meth:`writable` methods are used to determine whether the channel's socket
+   should be added to the list of channels :c:func:`select`\ ed or
+   :c:func:`poll`\ ed for read and write events.
+
+   Thus, the set of channel events is larger than the basic socket events.  The
+   full set of methods that can be overridden in your subclass follows:
+
+
+   .. method:: handle_read()
+
+      Called when the asynchronous loop detects that a :meth:`read` call on the
+      channel's socket will succeed.
+
+
+   .. method:: handle_write()
+
+      Called when the asynchronous loop detects that a writable socket can be
+      written.  Often this method will implement the necessary buffering for
+      performance.  For example::
+
+         def handle_write(self):
+             sent = self.send(self.buffer)
+             self.buffer = self.buffer[sent:]
+
+
+   .. method:: handle_expt()
+
+      Called when there is out of band (OOB) data for a socket connection.  This
+      will almost never happen, as OOB is tenuously supported and rarely used.
+
+
+   .. method:: handle_connect()
+
+      Called when the active opener's socket actually makes a connection.  Might
+      send a "welcome" banner, or initiate a protocol negotiation with the
+      remote endpoint, for example.
+
+
+   .. method:: handle_close()
+
+      Called when the socket is closed.
+
+
+   .. method:: handle_error()
+
+      Called when an exception is raised and not otherwise handled.  The default
+      version prints a condensed traceback.
+
+
+   .. method:: handle_accept()
+
+      Called on listening channels (passive openers) when a connection can be
+      established with a new remote endpoint that has issued a :meth:`connect`
+      call for the local endpoint. Deprecated in version 3.2; use
+      :meth:`handle_accepted` instead.
+
+      .. deprecated:: 3.2
+
+
+   .. method:: handle_accepted(sock, addr)
+
+      Called on listening channels (passive openers) when a connection has been
+      established with a new remote endpoint that has issued a :meth:`connect`
+      call for the local endpoint.  *sock* is a *new* socket object usable to
+      send and receive data on the connection, and *addr* is the address
+      bound to the socket on the other end of the connection.
+
+      .. versionadded:: 3.2
+
+
+   .. method:: readable()
+
+      Called each time around the asynchronous loop to determine whether a
+      channel's socket should be added to the list on which read events can
+      occur.  The default method simply returns ``True``, indicating that by
+      default, all channels will be interested in read events.
+
+
+   .. method:: writable()
+
+      Called each time around the asynchronous loop to determine whether a
+      channel's socket should be added to the list on which write events can
+      occur.  The default method simply returns ``True``, indicating that by
+      default, all channels will be interested in write events.
+
+
+   In addition, each channel delegates or extends many of the socket methods.
+   Most of these are nearly identical to their socket partners.
+
+
+   .. method:: create_socket(family=socket.AF_INET, type=socket.SOCK_STREAM)
+
+      This is identical to the creation of a normal socket, and will use the
+      same options for creation.  Refer to the :mod:`socket` documentation for
+      information on creating sockets.
+
+      .. versionchanged:: 3.3
+         *family* and *type* arguments can be omitted.
+
+
+   .. method:: connect(address)
+
+      As with the normal socket object, *address* is a tuple with the first
+      element the host to connect to, and the second the port number.
+
+
+   .. method:: send(data)
+
+      Send *data* to the remote end-point of the socket.
+
+
+   .. method:: recv(buffer_size)
+
+      Read at most *buffer_size* bytes from the socket's remote end-point.  An
+      empty bytes object implies that the channel has been closed from the
+      other end.
+
+      Note that :meth:`recv` may raise :exc:`BlockingIOError` , even though
+      :func:`select.select` or :func:`select.poll` has reported the socket
+      ready for reading.
+
+
+   .. method:: listen(backlog)
+
+      Listen for connections made to the socket.  The *backlog* argument
+      specifies the maximum number of queued connections and should be at least
+      1; the maximum value is system-dependent (usually 5).
+
+
+   .. method:: bind(address)
+
+      Bind the socket to *address*.  The socket must not already be bound.  (The
+      format of *address* depends on the address family --- refer to the
+      :mod:`socket` documentation for more information.)  To mark
+      the socket as re-usable (setting the :const:`SO_REUSEADDR` option), call
+      the :class:`dispatcher` object's :meth:`set_reuse_addr` method.
+
+
+   .. method:: accept()
+
+      Accept a connection.  The socket must be bound to an address and listening
+      for connections.  The return value can be either ``None`` or a pair
+      ``(conn, address)`` where *conn* is a *new* socket object usable to send
+      and receive data on the connection, and *address* is the address bound to
+      the socket on the other end of the connection.
+      When ``None`` is returned it means the connection didn't take place, in
+      which case the server should just ignore this event and keep listening
+      for further incoming connections.
+
+
+   .. method:: close()
+
+      Close the socket.  All future operations on the socket object will fail.
+      The remote end-point will receive no more data (after queued data is
+      flushed).  Sockets are automatically closed when they are
+      garbage-collected.
+
+
+.. class:: dispatcher_with_send()
+
+   A :class:`dispatcher` subclass which adds simple buffered output capability,
+   useful for simple clients. For more sophisticated usage use
+   :class:`asynchat.async_chat`.
+
+.. class:: file_dispatcher()
+
+   A file_dispatcher takes a file descriptor or :term:`file object` along
+   with an optional map argument and wraps it for use with the :c:func:`poll`
+   or :c:func:`loop` functions.  If provided a file object or anything with a
+   :c:func:`fileno` method, that method will be called and passed to the
+   :class:`file_wrapper` constructor.
+
+   .. availability:: Unix.
+
+.. class:: file_wrapper()
+
+   A file_wrapper takes an integer file descriptor and calls :func:`os.dup` to
+   duplicate the handle so that the original handle may be closed independently
+   of the file_wrapper.  This class implements sufficient methods to emulate a
+   socket for use by the :class:`file_dispatcher` class.
+
+   .. availability:: Unix.
+
+
+.. _asyncore-example-1:
+
+asyncore Example basic HTTP client
+----------------------------------
+
+Here is a very basic HTTP client that uses the :class:`dispatcher` class to
+implement its socket handling::
+
+   import asyncore
+
+   class HTTPClient(asyncore.dispatcher):
+
+       def __init__(self, host, path):
+           asyncore.dispatcher.__init__(self)
+           self.create_socket()
+           self.connect( (host, 80) )
+           self.buffer = bytes('GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' %
+                               (path, host), 'ascii')
+
+       def handle_connect(self):
+           pass
+
+       def handle_close(self):
+           self.close()
+
+       def handle_read(self):
+           print(self.recv(8192))
+
+       def writable(self):
+           return (len(self.buffer) > 0)
+
+       def handle_write(self):
+           sent = self.send(self.buffer)
+           self.buffer = self.buffer[sent:]
+
+
+   client = HTTPClient('www.python.org', '/')
+   asyncore.loop()
+
+.. _asyncore-example-2:
+
+asyncore Example basic echo server
+----------------------------------
+
+Here is a basic echo server that uses the :class:`dispatcher` class to accept
+connections and dispatches the incoming connections to a handler::
+
+    import asyncore
+
+    class EchoHandler(asyncore.dispatcher_with_send):
+
+        def handle_read(self):
+            data = self.recv(8192)
+            if data:
+                self.send(data)
+
+    class EchoServer(asyncore.dispatcher):
+
+        def __init__(self, host, port):
+            asyncore.dispatcher.__init__(self)
+            self.create_socket()
+            self.set_reuse_addr()
+            self.bind((host, port))
+            self.listen(5)
+
+        def handle_accepted(self, sock, addr):
+            print('Incoming connection from %s' % repr(addr))
+            handler = EchoHandler(sock)
+
+    server = EchoServer('localhost', 8080)
+    asyncore.loop()
diff --git a/Doc/library/email.rst b/Doc/library/email.rst
index 816fae991d24c..5eebcd9e896d9 100644
--- a/Doc/library/email.rst
+++ b/Doc/library/email.rst
@@ -147,3 +147,6 @@ Legacy API:
    Module :mod:`mailbox`
       Tools for creating, reading, and managing collections of messages on disk
       using a variety standard formats.
+
+   Module :mod:`smtpd`
+      SMTP server framework (primarily useful for testing)
diff --git a/Doc/library/internet.rst b/Doc/library/internet.rst
index 65693c9b3b200..e745dd1243512 100644
--- a/Doc/library/internet.rst
+++ b/Doc/library/internet.rst
@@ -35,6 +35,7 @@ is currently supported on most popular platforms.  Here is an overview:
    imaplib.rst
    nntplib.rst
    smtplib.rst
+   smtpd.rst
    telnetlib.rst
    uuid.rst
    socketserver.rst
diff --git a/Doc/library/ipc.rst b/Doc/library/ipc.rst
index 4849c82f317d9..b88a174eb97f1 100644
--- a/Doc/library/ipc.rst
+++ b/Doc/library/ipc.rst
@@ -22,5 +22,7 @@ The list of modules described in this chapter is:
    ssl.rst
    select.rst
    selectors.rst
+   asyncore.rst
+   asynchat.rst
    signal.rst
    mmap.rst
diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst
new file mode 100644
index 0000000000000..611411ddd295b
--- /dev/null
+++ b/Doc/library/smtpd.rst
@@ -0,0 +1,264 @@
+:mod:`smtpd` --- SMTP Server
+============================
+
+.. module:: smtpd
+   :synopsis: A SMTP server implementation in Python.
+
+.. moduleauthor:: Barry Warsaw <barry at python.org>
+.. sectionauthor:: Moshe Zadka <moshez at moshez.org>
+
+**Source code:** :source:`Lib/smtpd.py`
+
+--------------
+
+This module offers several classes to implement SMTP (email) servers.
+
+.. deprecated:: 3.6
+   The `aiosmtpd <https://aiosmtpd.readthedocs.io/>`_ package is a recommended
+   replacement for this module.  It is based on :mod:`asyncio` and provides a
+   more straightforward API.
+
+Several server implementations are present; one is a generic
+do-nothing implementation, which can be overridden, while the other two offer
+specific mail-sending strategies.
+
+Additionally the SMTPChannel may be extended to implement very specific
+interaction behaviour with SMTP clients.
+
+The code supports :RFC:`5321`, plus the :rfc:`1870` SIZE and :rfc:`6531`
+SMTPUTF8 extensions.
+
+
+SMTPServer Objects
+------------------
+
+
+.. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432,\
+                      map=None, enable_SMTPUTF8=False, decode_data=False)
+
+   Create a new :class:`SMTPServer` object, which binds to local address
+   *localaddr*.  It will treat *remoteaddr* as an upstream SMTP relayer.  Both
+   *localaddr* and *remoteaddr* should be a :ref:`(host, port) <host_port>`
+   tuple.  The object inherits from :class:`asyncore.dispatcher`, and so will
+   insert itself into :mod:`asyncore`'s event loop on instantiation.
+
+   *data_size_limit* specifies the maximum number of bytes that will be
+   accepted in a ``DATA`` command.  A value of ``None`` or ``0`` means no
+   limit.
+
+   *map* is the socket map to use for connections (an initially empty
+   dictionary is a suitable value).  If not specified the :mod:`asyncore`
+   global socket map is used.
+
+   *enable_SMTPUTF8* determines whether the ``SMTPUTF8`` extension (as defined
+   in :RFC:`6531`) should be enabled.  The default is ``False``.
+   When ``True``, ``SMTPUTF8`` is accepted as a parameter to the ``MAIL``
+   command and when present is passed to :meth:`process_message` in the
+   ``kwargs['mail_options']`` list.  *decode_data* and *enable_SMTPUTF8*
+   cannot be set to ``True`` at the same time.
+
+   *decode_data* specifies whether the data portion of the SMTP transaction
+   should be decoded using UTF-8.  When *decode_data* is ``False`` (the
+   default), the server advertises the ``8BITMIME``
+   extension (:rfc:`6152`), accepts the ``BODY=8BITMIME`` parameter to
+   the ``MAIL`` command, and when present passes it to :meth:`process_message`
+   in the ``kwargs['mail_options']`` list. *decode_data* and *enable_SMTPUTF8*
+   cannot be set to ``True`` at the same time.
+
+   .. method:: process_message(peer, mailfrom, rcpttos, data, **kwargs)
+
+      Raise a :exc:`NotImplementedError` exception. Override this in subclasses to
+      do something useful with this message. Whatever was passed in the
+      constructor as *remoteaddr* will be available as the :attr:`_remoteaddr`
+      attribute. *peer* is the remote host's address, *mailfrom* is the envelope
+      originator, *rcpttos* are the envelope recipients and *data* is a string
+      containing the contents of the e-mail (which should be in :rfc:`5321`
+      format).
+
+      If the *decode_data* constructor keyword is set to ``True``, the *data*
+      argument will be a unicode string.  If it is set to ``False``, it
+      will be a bytes object.
+
+      *kwargs* is a dictionary containing additional information. It is empty
+      if ``decode_data=True`` was given as an init argument, otherwise
+      it contains the following keys:
+
+          *mail_options*:
+             a list of all received parameters to the ``MAIL``
+             command (the elements are uppercase strings; example:
+             ``['BODY=8BITMIME', 'SMTPUTF8']``).
+
+          *rcpt_options*:
+             same as *mail_options* but for the ``RCPT`` command.
+             Currently no ``RCPT TO`` options are supported, so for now
+             this will always be an empty list.
+
+      Implementations of ``process_message`` should use the ``**kwargs``
+      signature to accept arbitrary keyword arguments, since future feature
+      enhancements may add keys to the kwargs dictionary.
+
+      Return ``None`` to request a normal ``250 Ok`` response; otherwise
+      return the desired response string in :RFC:`5321` format.
+
+   .. attribute:: channel_class
+
+      Override this in subclasses to use a custom :class:`SMTPChannel` for
+      managing SMTP clients.
+
+   .. versionadded:: 3.4
+      The *map* constructor argument.
+
+   .. versionchanged:: 3.5
+      *localaddr* and *remoteaddr* may now contain IPv6 addresses.
+
+   .. versionadded:: 3.5
+      The *decode_data* and *enable_SMTPUTF8* constructor parameters, and the
+      *kwargs* parameter to :meth:`process_message` when *decode_data* is
+      ``False``.
+
+   .. versionchanged:: 3.6
+      *decode_data* is now ``False`` by default.
+
+
+DebuggingServer Objects
+-----------------------
+
+
+.. class:: DebuggingServer(localaddr, remoteaddr)
+
+   Create a new debugging server.  Arguments are as per :class:`SMTPServer`.
+   Messages will be discarded, and printed on stdout.
+
+
+PureProxy Objects
+-----------------
+
+
+.. class:: PureProxy(localaddr, remoteaddr)
+
+   Create a new pure proxy server. Arguments are as per :class:`SMTPServer`.
+   Everything will be relayed to *remoteaddr*.  Note that running this has a good
+   chance to make you into an open relay, so please be careful.
+
+
+SMTPChannel Objects
+-------------------
+
+.. class:: SMTPChannel(server, conn, addr, data_size_limit=33554432,\
+                       map=None, enable_SMTPUTF8=False, decode_data=False)
+
+   Create a new :class:`SMTPChannel` object which manages the communication
+   between the server and a single SMTP client.
+
+   *conn* and *addr* are as per the instance variables described below.
+
+   *data_size_limit* specifies the maximum number of bytes that will be
+   accepted in a ``DATA`` command.  A value of ``None`` or ``0`` means no
+   limit.
+
+   *enable_SMTPUTF8* determines whether the ``SMTPUTF8`` extension (as defined
+   in :RFC:`6531`) should be enabled.  The default is ``False``.
+   *decode_data* and *enable_SMTPUTF8* cannot be set to ``True`` at the same
+   time.
+
+   A dictionary can be specified in *map* to avoid using a global socket map.
+
+   *decode_data* specifies whether the data portion of the SMTP transaction
+   should be decoded using UTF-8.  The default is ``False``.
+   *decode_data* and *enable_SMTPUTF8* cannot be set to ``True`` at the same
+   time.
+
+   To use a custom SMTPChannel implementation you need to override the
+   :attr:`SMTPServer.channel_class` of your :class:`SMTPServer`.
+
+   .. versionchanged:: 3.5
+      The *decode_data* and *enable_SMTPUTF8* parameters were added.
+
+   .. versionchanged:: 3.6
+      *decode_data* is now ``False`` by default.
+
+   The :class:`SMTPChannel` has the following instance variables:
+
+   .. attribute:: smtp_server
+
+      Holds the :class:`SMTPServer` that spawned this channel.
+
+   .. attribute:: conn
+
+      Holds the socket object connecting to the client.
+
+   .. attribute:: addr
+
+      Holds the address of the client, the second value returned by
+      :func:`socket.accept <socket.socket.accept>`
+
+   .. attribute:: received_lines
+
+      Holds a list of the line strings (decoded using UTF-8) received from
+      the client. The lines have their ``"\r\n"`` line ending translated to
+      ``"\n"``.
+
+   .. attribute:: smtp_state
+
+      Holds the current state of the channel. This will be either
+      :attr:`COMMAND` initially and then :attr:`DATA` after the client sends
+      a "DATA" line.
+
+   .. attribute:: seen_greeting
+
+      Holds a string containing the greeting sent by the client in its "HELO".
+
+   .. attribute:: mailfrom
+
+      Holds a string containing the address identified in the "MAIL FROM:" line
+      from the client.
+
+   .. attribute:: rcpttos
+
+      Holds a list of strings containing the addresses identified in the
+      "RCPT TO:" lines from the client.
+
+   .. attribute:: received_data
+
+      Holds a string containing all of the data sent by the client during the
+      DATA state, up to but not including the terminating ``"\r\n.\r\n"``.
+
+   .. attribute:: fqdn
+
+      Holds the fully-qualified domain name of the server as returned by
+      :func:`socket.getfqdn`.
+
+   .. attribute:: peer
+
+      Holds the name of the client peer as returned by ``conn.getpeername()``
+      where ``conn`` is :attr:`conn`.
+
+   The :class:`SMTPChannel` operates by invoking methods named ``smtp_<command>``
+   upon reception of a command line from the client. Built into the base
+   :class:`SMTPChannel` class are methods for handling the following commands
+   (and responding to them appropriately):
+
+   ======== ===================================================================
+   Command  Action taken
+   ======== ===================================================================
+   HELO     Accepts the greeting from the client and stores it in
+            :attr:`seen_greeting`.  Sets server to base command mode.
+   EHLO     Accepts the greeting from the client and stores it in
+            :attr:`seen_greeting`.  Sets server to extended command mode.
+   NOOP     Takes no action.
+   QUIT     Closes the connection cleanly.
+   MAIL     Accepts the "MAIL FROM:" syntax and stores the supplied address as
+            :attr:`mailfrom`.  In extended command mode, accepts the
+            :rfc:`1870` SIZE attribute and responds appropriately based on the
+            value of *data_size_limit*.
+   RCPT     Accepts the "RCPT TO:" syntax and stores the supplied addresses in
+            the :attr:`rcpttos` list.
+   RSET     Resets the :attr:`mailfrom`, :attr:`rcpttos`, and
+            :attr:`received_data`, but not the greeting.
+   DATA     Sets the internal state to :attr:`DATA` and stores remaining lines
+            from the client in :attr:`received_data` until the terminator
+            ``"\r\n.\r\n"`` is received.
+   HELP     Returns minimal information on command syntax
+   VRFY     Returns code 252 (the server doesn't know if the address is valid)
+   EXPN     Reports that the command is not implemented.
+   ======== ===================================================================
diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst
index 8aa72bd7332ef..b65a3e8fb2b97 100644
--- a/Doc/library/socketserver.rst
+++ b/Doc/library/socketserver.rst
@@ -176,7 +176,7 @@ partially finished requests and to use :mod:`selectors` to decide which
 request to work on next (or whether to handle a new incoming request).  This is
 particularly important for stream services where each client can potentially be
 connected for a long time (if threads or subprocesses cannot be used).  See
-:mod:`asyncio` for another way to manage this.
+:mod:`asyncore` for another way to manage this.
 
 .. XXX should data and methods be intermingled, or separate?
    how should the distinction between class and instance variables be drawn?
diff --git a/Doc/license.rst b/Doc/license.rst
index 1d086b6fdd3fe..cd03411d6c94a 100644
--- a/Doc/license.rst
+++ b/Doc/license.rst
@@ -383,6 +383,33 @@ Project, http://www.wide.ad.jp/. ::
    SUCH DAMAGE.
 
 
+Asynchronous socket services
+----------------------------
+
+The :mod:`asynchat` and :mod:`asyncore` modules contain the following notice::
+
+   Copyright 1996 by Sam Rushing
+
+                           All Rights Reserved
+
+   Permission to use, copy, modify, and distribute this software and
+   its documentation for any purpose and without fee is hereby
+   granted, provided that the above copyright notice appear in all
+   copies and that both that copyright notice and this permission
+   notice appear in supporting documentation, and that the name of Sam
+   Rushing not be used in advertising or publicity pertaining to
+   distribution of the software without specific, written prior
+   permission.
+
+   SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+   INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+   NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+   CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+   OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+   NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+   CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
 Cookie management
 -----------------
 
diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index b06d8d4215033..a570864743d47 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -518,15 +518,6 @@ Removed
 
   (Contributed by Hugo van Kemenade in :issue:`45320`.)
 
-* Remove the ``asyncore`` and ``asynchat`` modules, deprecated in Python 3.6:
-  use the :mod:`asyncio` module instead.
-  (Contributed by Victor Stinner in :issue:`28533`.)
-
-* Remove the ``smtpd`` module, deprecated in Python 3.6: the `aiosmtpd
-  <https://aiosmtpd.readthedocs.io/>`__ module can be used instead, it is based
-  on asyncio.
-  (Contributed by Victor Stinner in :issue:`28533`.)
-
 
 Porting to Python 3.11
 ======================
diff --git a/Lib/test/support/_asynchat.py b/Lib/asynchat.py
similarity index 98%
rename from Lib/test/support/_asynchat.py
rename to Lib/asynchat.py
index 941cc1d22e6b0..de26ffa648ffe 100644
--- a/Lib/test/support/_asynchat.py
+++ b/Lib/asynchat.py
@@ -45,9 +45,17 @@
 method) up to the terminator, and then control will be returned to
 you - by calling your self.found_terminator() method.
 """
-from test.support import _asyncore as asyncore
+import asyncore
 from collections import deque
 
+from warnings import warn
+warn(
+    'The asynchat module is deprecated. '
+    'The recommended replacement is asyncio',
+    DeprecationWarning,
+    stacklevel=2)
+
+
 
 class async_chat(asyncore.dispatcher):
     """This is an abstract class.  You must derive from this class, and add
diff --git a/Lib/test/support/_asyncore.py b/Lib/asyncore.py
similarity index 99%
rename from Lib/test/support/_asyncore.py
rename to Lib/asyncore.py
index 7863efaea66c8..b1eea4bf65211 100644
--- a/Lib/test/support/_asyncore.py
+++ b/Lib/asyncore.py
@@ -57,6 +57,12 @@
      ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
      errorcode
 
+warnings.warn(
+    'The asyncore module is deprecated. '
+    'The recommended replacement is asyncio',
+    DeprecationWarning,
+    stacklevel=2)
+
 
 _DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
                            EBADF})
diff --git a/Lib/test/support/_smtpd.py b/Lib/smtpd.py
similarity index 83%
rename from Lib/test/support/_smtpd.py
rename to Lib/smtpd.py
index 0e37d08f2be34..1cd004fbc6fe5 100755
--- a/Lib/test/support/_smtpd.py
+++ b/Lib/smtpd.py
@@ -80,13 +80,22 @@
 from warnings import warn
 from email._header_value_parser import get_addr_spec, get_angle_addr
 
-from test.support import _asyncore as asyncore
-from test.support import _asynchat as asynchat
-
 __all__ = [
     "SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy",
 ]
 
+warn(
+    'The smtpd module is deprecated and unmaintained.  Please see aiosmtpd '
+    '(https://aiosmtpd.readthedocs.io/) for the recommended replacement.',
+    DeprecationWarning,
+    stacklevel=2)
+
+
+# These are imported after the above warning so that users get the correct
+# deprecation warning.
+import asyncore
+import asynchat
+
 
 program = sys.argv[0]
 __version__ = 'Python SMTP proxy version 0.3'
@@ -179,6 +188,128 @@ def _set_rset_state(self):
         self.received_lines = []
 
 
+    # properties for backwards-compatibility
+    @property
+    def __server(self):
+        warn("Access to __server attribute on SMTPChannel is deprecated, "
+            "use 'smtp_server' instead", DeprecationWarning, 2)
+        return self.smtp_server
+    @__server.setter
+    def __server(self, value):
+        warn("Setting __server attribute on SMTPChannel is deprecated, "
+            "set 'smtp_server' instead", DeprecationWarning, 2)
+        self.smtp_server = value
+
+    @property
+    def __line(self):
+        warn("Access to __line attribute on SMTPChannel is deprecated, "
+            "use 'received_lines' instead", DeprecationWarning, 2)
+        return self.received_lines
+    @__line.setter
+    def __line(self, value):
+        warn("Setting __line attribute on SMTPChannel is deprecated, "
+            "set 'received_lines' instead", DeprecationWarning, 2)
+        self.received_lines = value
+
+    @property
+    def __state(self):
+        warn("Access to __state attribute on SMTPChannel is deprecated, "
+            "use 'smtp_state' instead", DeprecationWarning, 2)
+        return self.smtp_state
+    @__state.setter
+    def __state(self, value):
+        warn("Setting __state attribute on SMTPChannel is deprecated, "
+            "set 'smtp_state' instead", DeprecationWarning, 2)
+        self.smtp_state = value
+
+    @property
+    def __greeting(self):
+        warn("Access to __greeting attribute on SMTPChannel is deprecated, "
+            "use 'seen_greeting' instead", DeprecationWarning, 2)
+        return self.seen_greeting
+    @__greeting.setter
+    def __greeting(self, value):
+        warn("Setting __greeting attribute on SMTPChannel is deprecated, "
+            "set 'seen_greeting' instead", DeprecationWarning, 2)
+        self.seen_greeting = value
+
+    @property
+    def __mailfrom(self):
+        warn("Access to __mailfrom attribute on SMTPChannel is deprecated, "
+            "use 'mailfrom' instead", DeprecationWarning, 2)
+        return self.mailfrom
+    @__mailfrom.setter
+    def __mailfrom(self, value):
+        warn("Setting __mailfrom attribute on SMTPChannel is deprecated, "
+            "set 'mailfrom' instead", DeprecationWarning, 2)
+        self.mailfrom = value
+
+    @property
+    def __rcpttos(self):
+        warn("Access to __rcpttos attribute on SMTPChannel is deprecated, "
+            "use 'rcpttos' instead", DeprecationWarning, 2)
+        return self.rcpttos
+    @__rcpttos.setter
+    def __rcpttos(self, value):
+        warn("Setting __rcpttos attribute on SMTPChannel is deprecated, "
+            "set 'rcpttos' instead", DeprecationWarning, 2)
+        self.rcpttos = value
+
+    @property
+    def __data(self):
+        warn("Access to __data attribute on SMTPChannel is deprecated, "
+            "use 'received_data' instead", DeprecationWarning, 2)
+        return self.received_data
+    @__data.setter
+    def __data(self, value):
+        warn("Setting __data attribute on SMTPChannel is deprecated, "
+            "set 'received_data' instead", DeprecationWarning, 2)
+        self.received_data = value
+
+    @property
+    def __fqdn(self):
+        warn("Access to __fqdn attribute on SMTPChannel is deprecated, "
+            "use 'fqdn' instead", DeprecationWarning, 2)
+        return self.fqdn
+    @__fqdn.setter
+    def __fqdn(self, value):
+        warn("Setting __fqdn attribute on SMTPChannel is deprecated, "
+            "set 'fqdn' instead", DeprecationWarning, 2)
+        self.fqdn = value
+
+    @property
+    def __peer(self):
+        warn("Access to __peer attribute on SMTPChannel is deprecated, "
+            "use 'peer' instead", DeprecationWarning, 2)
+        return self.peer
+    @__peer.setter
+    def __peer(self, value):
+        warn("Setting __peer attribute on SMTPChannel is deprecated, "
+            "set 'peer' instead", DeprecationWarning, 2)
+        self.peer = value
+
+    @property
+    def __conn(self):
+        warn("Access to __conn attribute on SMTPChannel is deprecated, "
+            "use 'conn' instead", DeprecationWarning, 2)
+        return self.conn
+    @__conn.setter
+    def __conn(self, value):
+        warn("Setting __conn attribute on SMTPChannel is deprecated, "
+            "set 'conn' instead", DeprecationWarning, 2)
+        self.conn = value
+
+    @property
+    def __addr(self):
+        warn("Access to __addr attribute on SMTPChannel is deprecated, "
+            "use 'addr' instead", DeprecationWarning, 2)
+        return self.addr
+    @__addr.setter
+    def __addr(self, value):
+        warn("Setting __addr attribute on SMTPChannel is deprecated, "
+            "set 'addr' instead", DeprecationWarning, 2)
+        self.addr = value
+
     # Overrides base class for convenience.
     def push(self, msg):
         asynchat.async_chat.push(self, bytes(
diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py
index 17dda99b64582..60c9be24617a6 100644
--- a/Lib/test/libregrtest/save_env.py
+++ b/Lib/test/libregrtest/save_env.py
@@ -52,7 +52,7 @@ def __init__(self, testname, verbose=0, quiet=False, *, pgo=False):
 
     resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr',
                  'os.environ', 'sys.path', 'sys.path_hooks', '__import__',
-                 'warnings.filters',
+                 'warnings.filters', 'asyncore.socket_map',
                  'logging._handlers', 'logging._handlerList', 'sys.gettrace',
                  'sys.warnoptions',
                  # multiprocessing.process._cleanup() may release ref
@@ -160,6 +160,16 @@ def restore_warnings_filters(self, saved_filters):
         warnings.filters = saved_filters[1]
         warnings.filters[:] = saved_filters[2]
 
+    def get_asyncore_socket_map(self):
+        asyncore = sys.modules.get('asyncore')
+        # XXX Making a copy keeps objects alive until __exit__ gets called.
+        return asyncore and asyncore.socket_map.copy() or {}
+    def restore_asyncore_socket_map(self, saved_map):
+        asyncore = sys.modules.get('asyncore')
+        if asyncore is not None:
+            asyncore.close_all(ignore_all=True)
+            asyncore.socket_map.update(saved_map)
+
     def get_shutil_archive_formats(self):
         shutil = self.try_get_module('shutil')
         # we could call get_archives_formats() but that only returns the
diff --git a/Lib/test/mock_socket.py b/Lib/test/mock_socket.py
index 9788d58b4c765..c7abddcf5fafd 100644
--- a/Lib/test/mock_socket.py
+++ b/Lib/test/mock_socket.py
@@ -1,4 +1,4 @@
-"""Mock socket module used by the smtplib tests.
+"""Mock socket module used by the smtpd and smtplib tests.
 """
 
 # imported for _GLOBAL_DEFAULT_TIMEOUT
@@ -33,7 +33,7 @@ def close(self):
 
 
 class MockSocket:
-    """Mock socket object used by smtplib tests.
+    """Mock socket object used by smtpd and smtplib tests.
     """
     def __init__(self, family=None):
         global _reply_data
diff --git a/Lib/test/test_asynchat.py b/Lib/test/test_asynchat.py
new file mode 100644
index 0000000000000..973ac1f7d97c9
--- /dev/null
+++ b/Lib/test/test_asynchat.py
@@ -0,0 +1,292 @@
+# test asynchat
+
+from test import support
+from test.support import socket_helper
+from test.support import threading_helper
+
+import errno
+import socket
+import sys
+import threading
+import time
+import unittest
+import unittest.mock
+
+import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', DeprecationWarning)
+    import asynchat
+    import asyncore
+
+HOST = socket_helper.HOST
+SERVER_QUIT = b'QUIT\n'
+
+
+class echo_server(threading.Thread):
+    # parameter to determine the number of bytes passed back to the
+    # client each send
+    chunk_size = 1
+
+    def __init__(self, event):
+        threading.Thread.__init__(self)
+        self.event = event
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.port = socket_helper.bind_port(self.sock)
+        # This will be set if the client wants us to wait before echoing
+        # data back.
+        self.start_resend_event = None
+
+    def run(self):
+        self.sock.listen()
+        self.event.set()
+        conn, client = self.sock.accept()
+        self.buffer = b""
+        # collect data until quit message is seen
+        while SERVER_QUIT not in self.buffer:
+            data = conn.recv(1)
+            if not data:
+                break
+            self.buffer = self.buffer + data
+
+        # remove the SERVER_QUIT message
+        self.buffer = self.buffer.replace(SERVER_QUIT, b'')
+
+        if self.start_resend_event:
+            self.start_resend_event.wait()
+
+        # re-send entire set of collected data
+        try:
+            # this may fail on some tests, such as test_close_when_done,
+            # since the client closes the channel when it's done sending
+            while self.buffer:
+                n = conn.send(self.buffer[:self.chunk_size])
+                time.sleep(0.001)
+                self.buffer = self.buffer[n:]
+        except:
+            pass
+
+        conn.close()
+        self.sock.close()
+
+class echo_client(asynchat.async_chat):
+
+    def __init__(self, terminator, server_port):
+        asynchat.async_chat.__init__(self)
+        self.contents = []
+        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.connect((HOST, server_port))
+        self.set_terminator(terminator)
+        self.buffer = b""
+
+    def handle_connect(self):
+        pass
+
+    if sys.platform == 'darwin':
+        # select.poll returns a select.POLLHUP at the end of the tests
+        # on darwin, so just ignore it
+        def handle_expt(self):
+            pass
+
+    def collect_incoming_data(self, data):
+        self.buffer += data
+
+    def found_terminator(self):
+        self.contents.append(self.buffer)
+        self.buffer = b""
+
+def start_echo_server():
+    event = threading.Event()
+    s = echo_server(event)
+    s.start()
+    event.wait()
+    event.clear()
+    time.sleep(0.01)   # Give server time to start accepting.
+    return s, event
+
+
+class TestAsynchat(unittest.TestCase):
+    usepoll = False
+
+    def setUp(self):
+        self._threads = threading_helper.threading_setup()
+
+    def tearDown(self):
+        threading_helper.threading_cleanup(*self._threads)
+
+    def line_terminator_check(self, term, server_chunk):
+        event = threading.Event()
+        s = echo_server(event)
+        s.chunk_size = server_chunk
+        s.start()
+        event.wait()
+        event.clear()
+        time.sleep(0.01)   # Give server time to start accepting.
+        c = echo_client(term, s.port)
+        c.push(b"hello ")
+        c.push(b"world" + term)
+        c.push(b"I'm not dead yet!" + term)
+        c.push(SERVER_QUIT)
+        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
+        threading_helper.join_thread(s)
+
+        self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"])
+
+    # the line terminator tests below check receiving variously-sized
+    # chunks back from the server in order to exercise all branches of
+    # async_chat.handle_read
+
+    def test_line_terminator1(self):
+        # test one-character terminator
+        for l in (1, 2, 3):
+            self.line_terminator_check(b'\n', l)
+
+    def test_line_terminator2(self):
+        # test two-character terminator
+        for l in (1, 2, 3):
+            self.line_terminator_check(b'\r\n', l)
+
+    def test_line_terminator3(self):
+        # test three-character terminator
+        for l in (1, 2, 3):
+            self.line_terminator_check(b'qqq', l)
+
+    def numeric_terminator_check(self, termlen):
+        # Try reading a fixed number of bytes
+        s, event = start_echo_server()
+        c = echo_client(termlen, s.port)
+        data = b"hello world, I'm not dead yet!\n"
+        c.push(data)
+        c.push(SERVER_QUIT)
+        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
+        threading_helper.join_thread(s)
+
+        self.assertEqual(c.contents, [data[:termlen]])
+
+    def test_numeric_terminator1(self):
+        # check that ints & longs both work (since type is
+        # explicitly checked in async_chat.handle_read)
+        self.numeric_terminator_check(1)
+
+    def test_numeric_terminator2(self):
+        self.numeric_terminator_check(6)
+
+    def test_none_terminator(self):
+        # Try reading a fixed number of bytes
+        s, event = start_echo_server()
+        c = echo_client(None, s.port)
+        data = b"hello world, I'm not dead yet!\n"
+        c.push(data)
+        c.push(SERVER_QUIT)
+        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
+        threading_helper.join_thread(s)
+
+        self.assertEqual(c.contents, [])
+        self.assertEqual(c.buffer, data)
+
+    def test_simple_producer(self):
+        s, event = start_echo_server()
+        c = echo_client(b'\n', s.port)
+        data = b"hello world\nI'm not dead yet!\n"
+        p = asynchat.simple_producer(data+SERVER_QUIT, buffer_size=8)
+        c.push_with_producer(p)
+        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
+        threading_helper.join_thread(s)
+
+        self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"])
+
+    def test_string_producer(self):
+        s, event = start_echo_server()
+        c = echo_client(b'\n', s.port)
+        data = b"hello world\nI'm not dead yet!\n"
+        c.push_with_producer(data+SERVER_QUIT)
+        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
+        threading_helper.join_thread(s)
+
+        self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"])
+
+    def test_empty_line(self):
+        # checks that empty lines are handled correctly
+        s, event = start_echo_server()
+        c = echo_client(b'\n', s.port)
+        c.push(b"hello world\n\nI'm not dead yet!\n")
+        c.push(SERVER_QUIT)
+        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
+        threading_helper.join_thread(s)
+
+        self.assertEqual(c.contents,
+                         [b"hello world", b"", b"I'm not dead yet!"])
+
+    def test_close_when_done(self):
+        s, event = start_echo_server()
+        s.start_resend_event = threading.Event()
+        c = echo_client(b'\n', s.port)
+        c.push(b"hello world\nI'm not dead yet!\n")
+        c.push(SERVER_QUIT)
+        c.close_when_done()
+        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
+
+        # Only allow the server to start echoing data back to the client after
+        # the client has closed its connection.  This prevents a race condition
+        # where the server echoes all of its data before we can check that it
+        # got any down below.
+        s.start_resend_event.set()
+        threading_helper.join_thread(s)
+
+        self.assertEqual(c.contents, [])
+        # the server might have been able to send a byte or two back, but this
+        # at least checks that it received something and didn't just fail
+        # (which could still result in the client not having received anything)
+        self.assertGreater(len(s.buffer), 0)
+
+    def test_push(self):
+        # Issue #12523: push() should raise a TypeError if it doesn't get
+        # a bytes string
+        s, event = start_echo_server()
+        c = echo_client(b'\n', s.port)
+        data = b'bytes\n'
+        c.push(data)
+        c.push(bytearray(data))
+        c.push(memoryview(data))
+        self.assertRaises(TypeError, c.push, 10)
+        self.assertRaises(TypeError, c.push, 'unicode')
+        c.push(SERVER_QUIT)
+        asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
+        threading_helper.join_thread(s)
+        self.assertEqual(c.contents, [b'bytes', b'bytes', b'bytes'])
+
+
+class TestAsynchat_WithPoll(TestAsynchat):
+    usepoll = True
+
+
+class TestAsynchatMocked(unittest.TestCase):
+    def test_blockingioerror(self):
+        # Issue #16133: handle_read() must ignore BlockingIOError
+        sock = unittest.mock.Mock()
+        sock.recv.side_effect = BlockingIOError(errno.EAGAIN)
+
+        dispatcher = asynchat.async_chat()
+        dispatcher.set_socket(sock)
+        self.addCleanup(dispatcher.del_channel)
+
+        with unittest.mock.patch.object(dispatcher, 'handle_error') as error:
+            dispatcher.handle_read()
+        self.assertFalse(error.called)
+
+
+class TestHelperFunctions(unittest.TestCase):
+    def test_find_prefix_at_end(self):
+        self.assertEqual(asynchat.find_prefix_at_end("qwerty\r", "\r\n"), 1)
+        self.assertEqual(asynchat.find_prefix_at_end("qwertydkjf", "\r\n"), 0)
+
+
+class TestNotConnected(unittest.TestCase):
+    def test_disallow_negative_terminator(self):
+        # Issue #11259
+        client = asynchat.async_chat()
+        self.assertRaises(ValueError, client.set_terminator, -1)
+
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py
new file mode 100644
index 0000000000000..ecd1e120ecb51
--- /dev/null
+++ b/Lib/test/test_asyncore.py
@@ -0,0 +1,841 @@
+import unittest
+import select
+import os
+import socket
+import sys
+import time
+import errno
+import struct
+import threading
+
+from test import support
+from test.support import os_helper
+from test.support import socket_helper
+from test.support import threading_helper
+from test.support import warnings_helper
+from io import BytesIO
+
+if support.PGO:
+    raise unittest.SkipTest("test is not helpful for PGO")
+
+import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', DeprecationWarning)
+    import asyncore
+
+
+HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX')
+
+class dummysocket:
+    def __init__(self):
+        self.closed = False
+
+    def close(self):
+        self.closed = True
+
+    def fileno(self):
+        return 42
+
+class dummychannel:
+    def __init__(self):
+        self.socket = dummysocket()
+
+    def close(self):
+        self.socket.close()
+
+class exitingdummy:
+    def __init__(self):
+        pass
+
+    def handle_read_event(self):
+        raise asyncore.ExitNow()
+
+    handle_write_event = handle_read_event
+    handle_close = handle_read_event
+    handle_expt_event = handle_read_event
+
+class crashingdummy:
+    def __init__(self):
+        self.error_handled = False
+
+    def handle_read_event(self):
+        raise Exception()
+
+    handle_write_event = handle_read_event
+    handle_close = handle_read_event
+    handle_expt_event = handle_read_event
+
+    def handle_error(self):
+        self.error_handled = True
+
+# used when testing senders; just collects what it gets until newline is sent
+def capture_server(evt, buf, serv):
+    try:
+        serv.listen()
+        conn, addr = serv.accept()
+    except TimeoutError:
+        pass
+    else:
+        n = 200
+        start = time.monotonic()
+        while n > 0 and time.monotonic() - start < 3.0:
+            r, w, e = select.select([conn], [], [], 0.1)
+            if r:
+                n -= 1
+                data = conn.recv(10)
+                # keep everything except for the newline terminator
+                buf.write(data.replace(b'\n', b''))
+                if b'\n' in data:
+                    break
+            time.sleep(0.01)
+
+        conn.close()
+    finally:
+        serv.close()
+        evt.set()
+
+def bind_af_aware(sock, addr):
+    """Helper function to bind a socket according to its family."""
+    if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX:
+        # Make sure the path doesn't exist.
+        os_helper.unlink(addr)
+        socket_helper.bind_unix_socket(sock, addr)
+    else:
+        sock.bind(addr)
+
+
+class HelperFunctionTests(unittest.TestCase):
+    def test_readwriteexc(self):
+        # Check exception handling behavior of read, write and _exception
+
+        # check that ExitNow exceptions in the object handler method
+        # bubbles all the way up through asyncore read/write/_exception calls
+        tr1 = exitingdummy()
+        self.assertRaises(asyncore.ExitNow, asyncore.read, tr1)
+        self.assertRaises(asyncore.ExitNow, asyncore.write, tr1)
+        self.assertRaises(asyncore.ExitNow, asyncore._exception, tr1)
+
+        # check that an exception other than ExitNow in the object handler
+        # method causes the handle_error method to get called
+        tr2 = crashingdummy()
+        asyncore.read(tr2)
+        self.assertEqual(tr2.error_handled, True)
+
+        tr2 = crashingdummy()
+        asyncore.write(tr2)
+        self.assertEqual(tr2.error_handled, True)
+
+        tr2 = crashingdummy()
+        asyncore._exception(tr2)
+        self.assertEqual(tr2.error_handled, True)
+
+    # asyncore.readwrite uses constants in the select module that
+    # are not present in Windows systems (see this thread:
+    # http://mail.python.org/pipermail/python-list/2001-October/109973.html)
+    # These constants should be present as long as poll is available
+
+    @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required')
+    def test_readwrite(self):
+        # Check that correct methods are called by readwrite()
+
+        attributes = ('read', 'expt', 'write', 'closed', 'error_handled')
+
+        expected = (
+            (select.POLLIN, 'read'),
+            (select.POLLPRI, 'expt'),
+            (select.POLLOUT, 'write'),
+            (select.POLLERR, 'closed'),
+            (select.POLLHUP, 'closed'),
+            (select.POLLNVAL, 'closed'),
+            )
+
+        class testobj:
+            def __init__(self):
+                self.read = False
+                self.write = False
+                self.closed = False
+                self.expt = False
+                self.error_handled = False
+
+            def handle_read_event(self):
+                self.read = True
+
+            def handle_write_event(self):
+                self.write = True
+
+            def handle_close(self):
+                self.closed = True
+
+            def handle_expt_event(self):
+                self.expt = True
+
+            def handle_error(self):
+                self.error_handled = True
+
+        for flag, expectedattr in expected:
+            tobj = testobj()
+            self.assertEqual(getattr(tobj, expectedattr), False)
+            asyncore.readwrite(tobj, flag)
+
+            # Only the attribute modified by the routine we expect to be
+            # called should be True.
+            for attr in attributes:
+                self.assertEqual(getattr(tobj, attr), attr==expectedattr)
+
+            # check that ExitNow exceptions in the object handler method
+            # bubbles all the way up through asyncore readwrite call
+            tr1 = exitingdummy()
+            self.assertRaises(asyncore.ExitNow, asyncore.readwrite, tr1, flag)
+
+            # check that an exception other than ExitNow in the object handler
+            # method causes the handle_error method to get called
+            tr2 = crashingdummy()
+            self.assertEqual(tr2.error_handled, False)
+            asyncore.readwrite(tr2, flag)
+            self.assertEqual(tr2.error_handled, True)
+
+    def test_closeall(self):
+        self.closeall_check(False)
+
+    def test_closeall_default(self):
+        self.closeall_check(True)
+
+    def closeall_check(self, usedefault):
+        # Check that close_all() closes everything in a given map
+
+        l = []
+        testmap = {}
+        for i in range(10):
+            c = dummychannel()
+            l.append(c)
+            self.assertEqual(c.socket.closed, False)
+            testmap[i] = c
+
+        if usedefault:
+            socketmap = asyncore.socket_map
+            try:
+                asyncore.socket_map = testmap
+                asyncore.close_all()
+            finally:
+                testmap, asyncore.socket_map = asyncore.socket_map, socketmap
+        else:
+            asyncore.close_all(testmap)
+
+        self.assertEqual(len(testmap), 0)
+
+        for c in l:
+            self.assertEqual(c.socket.closed, True)
+
+    def test_compact_traceback(self):
+        try:
+            raise Exception("I don't like spam!")
+        except:
+            real_t, real_v, real_tb = sys.exc_info()
+            r = asyncore.compact_traceback()
+        else:
+            self.fail("Expected exception")
+
+        (f, function, line), t, v, info = r
+        self.assertEqual(os.path.split(f)[-1], 'test_asyncore.py')
+        self.assertEqual(function, 'test_compact_traceback')
+        self.assertEqual(t, real_t)
+        self.assertEqual(v, real_v)
+        self.assertEqual(info, '[%s|%s|%s]' % (f, function, line))
+
+
+class DispatcherTests(unittest.TestCase):
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        asyncore.close_all()
+
+    def test_basic(self):
+        d = asyncore.dispatcher()
+        self.assertEqual(d.readable(), True)
+        self.assertEqual(d.writable(), True)
+
+    def test_repr(self):
+        d = asyncore.dispatcher()
+        self.assertEqual(repr(d), '<asyncore.dispatcher at %#x>' % id(d))
+
+    def test_log(self):
+        d = asyncore.dispatcher()
+
+        # capture output of dispatcher.log() (to stderr)
+        l1 = "Lovely spam! Wonderful spam!"
+        l2 = "I don't like spam!"
+        with support.captured_stderr() as stderr:
+            d.log(l1)
+            d.log(l2)
+
+        lines = stderr.getvalue().splitlines()
+        self.assertEqual(lines, ['log: %s' % l1, 'log: %s' % l2])
+
+    def test_log_info(self):
+        d = asyncore.dispatcher()
+
+        # capture output of dispatcher.log_info() (to stdout via print)
+        l1 = "Have you got anything without spam?"
+        l2 = "Why can't she have egg bacon spam and sausage?"
+        l3 = "THAT'S got spam in it!"
+        with support.captured_stdout() as stdout:
+            d.log_info(l1, 'EGGS')
+            d.log_info(l2)
+            d.log_info(l3, 'SPAM')
+
+        lines = stdout.getvalue().splitlines()
+        expected = ['EGGS: %s' % l1, 'info: %s' % l2, 'SPAM: %s' % l3]
+        self.assertEqual(lines, expected)
+
+    def test_unhandled(self):
+        d = asyncore.dispatcher()
+        d.ignore_log_types = ()
+
+        # capture output of dispatcher.log_info() (to stdout via print)
+        with support.captured_stdout() as stdout:
+            d.handle_expt()
+            d.handle_read()
+            d.handle_write()
+            d.handle_connect()
+
+        lines = stdout.getvalue().splitlines()
+        expected = ['warning: unhandled incoming priority event',
+                    'warning: unhandled read event',
+                    'warning: unhandled write event',
+                    'warning: unhandled connect event']
+        self.assertEqual(lines, expected)
+
+    def test_strerror(self):
+        # refers to bug #8573
+        err = asyncore._strerror(errno.EPERM)
+        if hasattr(os, 'strerror'):
+            self.assertEqual(err, os.strerror(errno.EPERM))
+        err = asyncore._strerror(-1)
+        self.assertTrue(err != "")
+
+
+class dispatcherwithsend_noread(asyncore.dispatcher_with_send):
+    def readable(self):
+        return False
+
+    def handle_connect(self):
+        pass
+
+
+class DispatcherWithSendTests(unittest.TestCase):
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        asyncore.close_all()
+
+    @threading_helper.reap_threads
+    def test_send(self):
+        evt = threading.Event()
+        sock = socket.socket()
+        sock.settimeout(3)
+        port = socket_helper.bind_port(sock)
+
+        cap = BytesIO()
+        args = (evt, cap, sock)
+        t = threading.Thread(target=capture_server, args=args)
+        t.start()
+        try:
+            # wait a little longer for the server to initialize (it sometimes
+            # refuses connections on slow machines without this wait)
+            time.sleep(0.2)
+
+            data = b"Suppose there isn't a 16-ton weight?"
+            d = dispatcherwithsend_noread()
+            d.create_socket()
+            d.connect((socket_helper.HOST, port))
+
+            # give time for socket to connect
+            time.sleep(0.1)
+
+            d.send(data)
+            d.send(data)
+            d.send(b'\n')
+
+            n = 1000
+            while d.out_buffer and n > 0:
+                asyncore.poll()
+                n -= 1
+
+            evt.wait()
+
+            self.assertEqual(cap.getvalue(), data*2)
+        finally:
+            threading_helper.join_thread(t)
+
+
+ at unittest.skipUnless(hasattr(asyncore, 'file_wrapper'),
+                     'asyncore.file_wrapper required')
+class FileWrapperTest(unittest.TestCase):
+    def setUp(self):
+        self.d = b"It's not dead, it's sleeping!"
+        with open(os_helper.TESTFN, 'wb') as file:
+            file.write(self.d)
+
+    def tearDown(self):
+        os_helper.unlink(os_helper.TESTFN)
+
+    def test_recv(self):
+        fd = os.open(os_helper.TESTFN, os.O_RDONLY)
+        w = asyncore.file_wrapper(fd)
+        os.close(fd)
+
+        self.assertNotEqual(w.fd, fd)
+        self.assertNotEqual(w.fileno(), fd)
+        self.assertEqual(w.recv(13), b"It's not dead")
+        self.assertEqual(w.read(6), b", it's")
+        w.close()
+        self.assertRaises(OSError, w.read, 1)
+
+    def test_send(self):
+        d1 = b"Come again?"
+        d2 = b"I want to buy some cheese."
+        fd = os.open(os_helper.TESTFN, os.O_WRONLY | os.O_APPEND)
+        w = asyncore.file_wrapper(fd)
+        os.close(fd)
+
+        w.write(d1)
+        w.send(d2)
+        w.close()
+        with open(os_helper.TESTFN, 'rb') as file:
+            self.assertEqual(file.read(), self.d + d1 + d2)
+
+    @unittest.skipUnless(hasattr(asyncore, 'file_dispatcher'),
+                         'asyncore.file_dispatcher required')
+    def test_dispatcher(self):
+        fd = os.open(os_helper.TESTFN, os.O_RDONLY)
+        data = []
+        class FileDispatcher(asyncore.file_dispatcher):
+            def handle_read(self):
+                data.append(self.recv(29))
+        s = FileDispatcher(fd)
+        os.close(fd)
+        asyncore.loop(timeout=0.01, use_poll=True, count=2)
+        self.assertEqual(b"".join(data), self.d)
+
+    def test_resource_warning(self):
+        # Issue #11453
+        fd = os.open(os_helper.TESTFN, os.O_RDONLY)
+        f = asyncore.file_wrapper(fd)
+
+        os.close(fd)
+        with warnings_helper.check_warnings(('', ResourceWarning)):
+            f = None
+            support.gc_collect()
+
+    def test_close_twice(self):
+        fd = os.open(os_helper.TESTFN, os.O_RDONLY)
+        f = asyncore.file_wrapper(fd)
+        os.close(fd)
+
+        os.close(f.fd)  # file_wrapper dupped fd
+        with self.assertRaises(OSError):
+            f.close()
+
+        self.assertEqual(f.fd, -1)
+        # calling close twice should not fail
+        f.close()
+
+
+class BaseTestHandler(asyncore.dispatcher):
+
+    def __init__(self, sock=None):
+        asyncore.dispatcher.__init__(self, sock)
+        self.flag = False
+
+    def handle_accept(self):
+        raise Exception("handle_accept not supposed to be called")
+
+    def handle_accepted(self):
+        raise Exception("handle_accepted not supposed to be called")
+
+    def handle_connect(self):
+        raise Exception("handle_connect not supposed to be called")
+
+    def handle_expt(self):
+        raise Exception("handle_expt not supposed to be called")
+
+    def handle_close(self):
+        raise Exception("handle_close not supposed to be called")
+
+    def handle_error(self):
+        raise
+
+
+class BaseServer(asyncore.dispatcher):
+    """A server which listens on an address and dispatches the
+    connection to a handler.
+    """
+
+    def __init__(self, family, addr, handler=BaseTestHandler):
+        asyncore.dispatcher.__init__(self)
+        self.create_socket(family)
+        self.set_reuse_addr()
+        bind_af_aware(self.socket, addr)
+        self.listen(5)
+        self.handler = handler
+
+    @property
+    def address(self):
+        return self.socket.getsockname()
+
+    def handle_accepted(self, sock, addr):
+        self.handler(sock)
+
+    def handle_error(self):
+        raise
+
+
+class BaseClient(BaseTestHandler):
+
+    def __init__(self, family, address):
+        BaseTestHandler.__init__(self)
+        self.create_socket(family)
+        self.connect(address)
+
+    def handle_connect(self):
+        pass
+
+
+class BaseTestAPI:
+
+    def tearDown(self):
+        asyncore.close_all(ignore_all=True)
+
+    def loop_waiting_for_flag(self, instance, timeout=5):
+        timeout = float(timeout) / 100
+        count = 100
+        while asyncore.socket_map and count > 0:
+            asyncore.loop(timeout=0.01, count=1, use_poll=self.use_poll)
+            if instance.flag:
+                return
+            count -= 1
+            time.sleep(timeout)
+        self.fail("flag not set")
+
+    def test_handle_connect(self):
+        # make sure handle_connect is called on connect()
+
+        class TestClient(BaseClient):
+            def handle_connect(self):
+                self.flag = True
+
+        server = BaseServer(self.family, self.addr)
+        client = TestClient(self.family, server.address)
+        self.loop_waiting_for_flag(client)
+
+    def test_handle_accept(self):
+        # make sure handle_accept() is called when a client connects
+
+        class TestListener(BaseTestHandler):
+
+            def __init__(self, family, addr):
+                BaseTestHandler.__init__(self)
+                self.create_socket(family)
+                bind_af_aware(self.socket, addr)
+                self.listen(5)
+                self.address = self.socket.getsockname()
+
+            def handle_accept(self):
+                self.flag = True
+
+        server = TestListener(self.family, self.addr)
+        client = BaseClient(self.family, server.address)
+        self.loop_waiting_for_flag(server)
+
+    def test_handle_accepted(self):
+        # make sure handle_accepted() is called when a client connects
+
+        class TestListener(BaseTestHandler):
+
+            def __init__(self, family, addr):
+                BaseTestHandler.__init__(self)
+                self.create_socket(family)
+                bind_af_aware(self.socket, addr)
+                self.listen(5)
+                self.address = self.socket.getsockname()
+
+            def handle_accept(self):
+                asyncore.dispatcher.handle_accept(self)
+
+            def handle_accepted(self, sock, addr):
+                sock.close()
+                self.flag = True
+
+        server = TestListener(self.family, self.addr)
+        client = BaseClient(self.family, server.address)
+        self.loop_waiting_for_flag(server)
+
+
+    def test_handle_read(self):
+        # make sure handle_read is called on data received
+
+        class TestClient(BaseClient):
+            def handle_read(self):
+                self.flag = True
+
+        class TestHandler(BaseTestHandler):
+            def __init__(self, conn):
+                BaseTestHandler.__init__(self, conn)
+                self.send(b'x' * 1024)
+
+        server = BaseServer(self.family, self.addr, TestHandler)
+        client = TestClient(self.family, server.address)
+        self.loop_waiting_for_flag(client)
+
+    def test_handle_write(self):
+        # make sure handle_write is called
+
+        class TestClient(BaseClient):
+            def handle_write(self):
+                self.flag = True
+
+        server = BaseServer(self.family, self.addr)
+        client = TestClient(self.family, server.address)
+        self.loop_waiting_for_flag(client)
+
+    def test_handle_close(self):
+        # make sure handle_close is called when the other end closes
+        # the connection
+
+        class TestClient(BaseClient):
+
+            def handle_read(self):
+                # in order to make handle_close be called we are supposed
+                # to make at least one recv() call
+                self.recv(1024)
+
+            def handle_close(self):
+                self.flag = True
+                self.close()
+
+        class TestHandler(BaseTestHandler):
+            def __init__(self, conn):
+                BaseTestHandler.__init__(self, conn)
+                self.close()
+
+        server = BaseServer(self.family, self.addr, TestHandler)
+        client = TestClient(self.family, server.address)
+        self.loop_waiting_for_flag(client)
+
+    def test_handle_close_after_conn_broken(self):
+        # Check that ECONNRESET/EPIPE is correctly handled (issues #5661 and
+        # #11265).
+
+        data = b'\0' * 128
+
+        class TestClient(BaseClient):
+
+            def handle_write(self):
+                self.send(data)
+
+            def handle_close(self):
+                self.flag = True
+                self.close()
+
+            def handle_expt(self):
+                self.flag = True
+                self.close()
+
+        class TestHandler(BaseTestHandler):
+
+            def handle_read(self):
+                self.recv(len(data))
+                self.close()
+
+            def writable(self):
+                return False
+
+        server = BaseServer(self.family, self.addr, TestHandler)
+        client = TestClient(self.family, server.address)
+        self.loop_waiting_for_flag(client)
+
+    @unittest.skipIf(sys.platform.startswith("sunos"),
+                     "OOB support is broken on Solaris")
+    def test_handle_expt(self):
+        # Make sure handle_expt is called on OOB data received.
+        # Note: this might fail on some platforms as OOB data is
+        # tenuously supported and rarely used.
+        if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX:
+            self.skipTest("Not applicable to AF_UNIX sockets.")
+
+        if sys.platform == "darwin" and self.use_poll:
+            self.skipTest("poll may fail on macOS; see issue #28087")
+
+        class TestClient(BaseClient):
+            def handle_expt(self):
+                self.socket.recv(1024, socket.MSG_OOB)
+                self.flag = True
+
+        class TestHandler(BaseTestHandler):
+            def __init__(self, conn):
+                BaseTestHandler.__init__(self, conn)
+                self.socket.send(bytes(chr(244), 'latin-1'), socket.MSG_OOB)
+
+        server = BaseServer(self.family, self.addr, TestHandler)
+        client = TestClient(self.family, server.address)
+        self.loop_waiting_for_flag(client)
+
+    def test_handle_error(self):
+
+        class TestClient(BaseClient):
+            def handle_write(self):
+                1.0 / 0
+            def handle_error(self):
+                self.flag = True
+                try:
+                    raise
+                except ZeroDivisionError:
+                    pass
+                else:
+                    raise Exception("exception not raised")
+
+        server = BaseServer(self.family, self.addr)
+        client = TestClient(self.family, server.address)
+        self.loop_waiting_for_flag(client)
+
+    def test_connection_attributes(self):
+        server = BaseServer(self.family, self.addr)
+        client = BaseClient(self.family, server.address)
+
+        # we start disconnected
+        self.assertFalse(server.connected)
+        self.assertTrue(server.accepting)
+        # this can't be taken for granted across all platforms
+        #self.assertFalse(client.connected)
+        self.assertFalse(client.accepting)
+
+        # execute some loops so that client connects to server
+        asyncore.loop(timeout=0.01, use_poll=self.use_poll, count=100)
+        self.assertFalse(server.connected)
+        self.assertTrue(server.accepting)
+        self.assertTrue(client.connected)
+        self.assertFalse(client.accepting)
+
+        # disconnect the client
+        client.close()
+        self.assertFalse(server.connected)
+        self.assertTrue(server.accepting)
+        self.assertFalse(client.connected)
+        self.assertFalse(client.accepting)
+
+        # stop serving
+        server.close()
+        self.assertFalse(server.connected)
+        self.assertFalse(server.accepting)
+
+    def test_create_socket(self):
+        s = asyncore.dispatcher()
+        s.create_socket(self.family)
+        self.assertEqual(s.socket.type, socket.SOCK_STREAM)
+        self.assertEqual(s.socket.family, self.family)
+        self.assertEqual(s.socket.gettimeout(), 0)
+        self.assertFalse(s.socket.get_inheritable())
+
+    def test_bind(self):
+        if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX:
+            self.skipTest("Not applicable to AF_UNIX sockets.")
+        s1 = asyncore.dispatcher()
+        s1.create_socket(self.family)
+        s1.bind(self.addr)
+        s1.listen(5)
+        port = s1.socket.getsockname()[1]
+
+        s2 = asyncore.dispatcher()
+        s2.create_socket(self.family)
+        # EADDRINUSE indicates the socket was correctly bound
+        self.assertRaises(OSError, s2.bind, (self.addr[0], port))
+
+    def test_set_reuse_addr(self):
+        if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX:
+            self.skipTest("Not applicable to AF_UNIX sockets.")
+
+        with socket.socket(self.family) as sock:
+            try:
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            except OSError:
+                unittest.skip("SO_REUSEADDR not supported on this platform")
+            else:
+                # if SO_REUSEADDR succeeded for sock we expect asyncore
+                # to do the same
+                s = asyncore.dispatcher(socket.socket(self.family))
+                self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET,
+                                                     socket.SO_REUSEADDR))
+                s.socket.close()
+                s.create_socket(self.family)
+                s.set_reuse_addr()
+                self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET,
+                                                     socket.SO_REUSEADDR))
+
+    @threading_helper.reap_threads
+    def test_quick_connect(self):
+        # see: http://bugs.python.org/issue10340
+        if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())):
+            self.skipTest("test specific to AF_INET and AF_INET6")
+
+        server = BaseServer(self.family, self.addr)
+        # run the thread 500 ms: the socket should be connected in 200 ms
+        t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1,
+                                                          count=5))
+        t.start()
+        try:
+            with socket.socket(self.family, socket.SOCK_STREAM) as s:
+                s.settimeout(.2)
+                s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
+                             struct.pack('ii', 1, 0))
+
+                try:
+                    s.connect(server.address)
+                except OSError:
+                    pass
+        finally:
+            threading_helper.join_thread(t)
+
+class TestAPI_UseIPv4Sockets(BaseTestAPI):
+    family = socket.AF_INET
+    addr = (socket_helper.HOST, 0)
+
+ at unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 support required')
+class TestAPI_UseIPv6Sockets(BaseTestAPI):
+    family = socket.AF_INET6
+    addr = (socket_helper.HOSTv6, 0)
+
+ at unittest.skipUnless(HAS_UNIX_SOCKETS, 'Unix sockets required')
+class TestAPI_UseUnixSockets(BaseTestAPI):
+    if HAS_UNIX_SOCKETS:
+        family = socket.AF_UNIX
+    addr = os_helper.TESTFN
+
+    def tearDown(self):
+        os_helper.unlink(self.addr)
+        BaseTestAPI.tearDown(self)
+
+class TestAPI_UseIPv4Select(TestAPI_UseIPv4Sockets, unittest.TestCase):
+    use_poll = False
+
+ at unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required')
+class TestAPI_UseIPv4Poll(TestAPI_UseIPv4Sockets, unittest.TestCase):
+    use_poll = True
+
+class TestAPI_UseIPv6Select(TestAPI_UseIPv6Sockets, unittest.TestCase):
+    use_poll = False
+
+ at unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required')
+class TestAPI_UseIPv6Poll(TestAPI_UseIPv6Sockets, unittest.TestCase):
+    use_poll = True
+
+class TestAPI_UseUnixSocketsSelect(TestAPI_UseUnixSockets, unittest.TestCase):
+    use_poll = False
+
+ at unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required')
+class TestAPI_UseUnixSocketsPoll(TestAPI_UseUnixSockets, unittest.TestCase):
+    use_poll = True
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py
index c9edfac96871a..56e3d8ab8528a 100644
--- a/Lib/test/test_ftplib.py
+++ b/Lib/test/test_ftplib.py
@@ -18,13 +18,17 @@
 
 from unittest import TestCase, skipUnless
 from test import support
-from test.support import _asynchat as asynchat
-from test.support import _asyncore as asyncore
-from test.support import socket_helper
 from test.support import threading_helper
+from test.support import socket_helper
 from test.support import warnings_helper
 from test.support.socket_helper import HOST, HOSTv6
 
+import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', DeprecationWarning)
+    import asyncore
+    import asynchat
+
 
 TIMEOUT = support.LOOPBACK_TIMEOUT
 DEFAULT_ENCODING = 'utf-8'
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
index e709ea327fbd3..85b6e5f392111 100644
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -42,8 +42,6 @@
 import tempfile
 from test.support.script_helper import assert_python_ok, assert_python_failure
 from test import support
-from test.support import _asyncore as asyncore
-from test.support import _smtpd as smtpd
 from test.support import os_helper
 from test.support import socket_helper
 from test.support import threading_helper
@@ -61,6 +59,11 @@
 from socketserver import (ThreadingUDPServer, DatagramRequestHandler,
                           ThreadingTCPServer, StreamRequestHandler)
 
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', DeprecationWarning)
+    import asyncore
+    import smtpd
+
 try:
     import win32evtlog, win32evtlogutil, pywintypes
 except ImportError:
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 5e15340515674..8da0aa3163fe2 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -30,8 +30,6 @@
 import uuid
 import warnings
 from test import support
-from test.support import _asynchat as asynchat
-from test.support import _asyncore as asyncore
 from test.support import import_helper
 from test.support import os_helper
 from test.support import socket_helper
@@ -39,6 +37,11 @@
 from test.support import warnings_helper
 from platform import win32_is_iot
 
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', DeprecationWarning)
+    import asynchat
+    import asyncore
+
 try:
     import resource
 except ImportError:
diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py
index 23c6be3a801bf..44cf5231f9d23 100644
--- a/Lib/test/test_poplib.py
+++ b/Lib/test/test_poplib.py
@@ -12,12 +12,16 @@
 import unittest
 from unittest import TestCase, skipUnless
 from test import support as test_support
-from test.support import _asynchat as asynchat
-from test.support import _asyncore as asyncore
 from test.support import hashlib_helper
 from test.support import socket_helper
 from test.support import threading_helper
 
+import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', DeprecationWarning)
+    import asynchat
+    import asyncore
+
 HOST = socket_helper.HOST
 PORT = 0
 
diff --git a/Lib/test/test_smtpd.py b/Lib/test/test_smtpd.py
new file mode 100644
index 0000000000000..d2e150d535ff6
--- /dev/null
+++ b/Lib/test/test_smtpd.py
@@ -0,0 +1,1018 @@
+import unittest
+import textwrap
+from test import support, mock_socket
+from test.support import socket_helper
+from test.support import warnings_helper
+import socket
+import io
+
+import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', DeprecationWarning)
+    import smtpd
+    import asyncore
+
+
+class DummyServer(smtpd.SMTPServer):
+    def __init__(self, *args, **kwargs):
+        smtpd.SMTPServer.__init__(self, *args, **kwargs)
+        self.messages = []
+        if self._decode_data:
+            self.return_status = 'return status'
+        else:
+            self.return_status = b'return status'
+
+    def process_message(self, peer, mailfrom, rcpttos, data, **kw):
+        self.messages.append((peer, mailfrom, rcpttos, data))
+        if data == self.return_status:
+            return '250 Okish'
+        if 'mail_options' in kw and 'SMTPUTF8' in kw['mail_options']:
+            return '250 SMTPUTF8 message okish'
+
+
+class DummyDispatcherBroken(Exception):
+    pass
+
+
+class BrokenDummyServer(DummyServer):
+    def listen(self, num):
+        raise DummyDispatcherBroken()
+
+
+class SMTPDServerTest(unittest.TestCase):
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+
+    def test_process_message_unimplemented(self):
+        server = smtpd.SMTPServer((socket_helper.HOST, 0), ('b', 0),
+                                  decode_data=True)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True)
+
+        def write_line(line):
+            channel.socket.queue_recv(line)
+            channel.handle_read()
+
+        write_line(b'HELO example')
+        write_line(b'MAIL From:eggs at example')
+        write_line(b'RCPT To:spam at example')
+        write_line(b'DATA')
+        self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n')
+
+    def test_decode_data_and_enable_SMTPUTF8_raises(self):
+        self.assertRaises(
+            ValueError,
+            smtpd.SMTPServer,
+            (socket_helper.HOST, 0),
+            ('b', 0),
+            enable_SMTPUTF8=True,
+            decode_data=True)
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+
+
+class DebuggingServerTest(unittest.TestCase):
+
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+
+    def send_data(self, channel, data, enable_SMTPUTF8=False):
+        def write_line(line):
+            channel.socket.queue_recv(line)
+            channel.handle_read()
+        write_line(b'EHLO example')
+        if enable_SMTPUTF8:
+            write_line(b'MAIL From:eggs at example BODY=8BITMIME SMTPUTF8')
+        else:
+            write_line(b'MAIL From:eggs at example')
+        write_line(b'RCPT To:spam at example')
+        write_line(b'DATA')
+        write_line(data)
+        write_line(b'.')
+
+    def test_process_message_with_decode_data_true(self):
+        server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0),
+                                       decode_data=True)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True)
+        with support.captured_stdout() as s:
+            self.send_data(channel, b'From: test\n\nhello\n')
+        stdout = s.getvalue()
+        self.assertEqual(stdout, textwrap.dedent("""\
+             ---------- MESSAGE FOLLOWS ----------
+             From: test
+             X-Peer: peer-address
+
+             hello
+             ------------ END MESSAGE ------------
+             """))
+
+    def test_process_message_with_decode_data_false(self):
+        server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0))
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr)
+        with support.captured_stdout() as s:
+            self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n')
+        stdout = s.getvalue()
+        self.assertEqual(stdout, textwrap.dedent("""\
+             ---------- MESSAGE FOLLOWS ----------
+             b'From: test'
+             b'X-Peer: peer-address'
+             b''
+             b'h\\xc3\\xa9llo\\xff'
+             ------------ END MESSAGE ------------
+             """))
+
+    def test_process_message_with_enable_SMTPUTF8_true(self):
+        server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0),
+                                       enable_SMTPUTF8=True)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True)
+        with support.captured_stdout() as s:
+            self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n')
+        stdout = s.getvalue()
+        self.assertEqual(stdout, textwrap.dedent("""\
+             ---------- MESSAGE FOLLOWS ----------
+             b'From: test'
+             b'X-Peer: peer-address'
+             b''
+             b'h\\xc3\\xa9llo\\xff'
+             ------------ END MESSAGE ------------
+             """))
+
+    def test_process_SMTPUTF8_message_with_enable_SMTPUTF8_true(self):
+        server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0),
+                                       enable_SMTPUTF8=True)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True)
+        with support.captured_stdout() as s:
+            self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n',
+                           enable_SMTPUTF8=True)
+        stdout = s.getvalue()
+        self.assertEqual(stdout, textwrap.dedent("""\
+             ---------- MESSAGE FOLLOWS ----------
+             mail options: ['BODY=8BITMIME', 'SMTPUTF8']
+             b'From: test'
+             b'X-Peer: peer-address'
+             b''
+             b'h\\xc3\\xa9llo\\xff'
+             ------------ END MESSAGE ------------
+             """))
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+
+
+class TestFamilyDetection(unittest.TestCase):
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+
+    @unittest.skipUnless(socket_helper.IPV6_ENABLED, "IPv6 not enabled")
+    def test_socket_uses_IPv6(self):
+        server = smtpd.SMTPServer((socket_helper.HOSTv6, 0), (socket_helper.HOSTv4, 0))
+        self.assertEqual(server.socket.family, socket.AF_INET6)
+
+    def test_socket_uses_IPv4(self):
+        server = smtpd.SMTPServer((socket_helper.HOSTv4, 0), (socket_helper.HOSTv6, 0))
+        self.assertEqual(server.socket.family, socket.AF_INET)
+
+
+class TestRcptOptionParsing(unittest.TestCase):
+    error_response = (b'555 RCPT TO parameters not recognized or not '
+                      b'implemented\r\n')
+
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, channel, line):
+        channel.socket.queue_recv(line)
+        channel.handle_read()
+
+    def test_params_rejected(self):
+        server = DummyServer((socket_helper.HOST, 0), ('b', 0))
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr)
+        self.write_line(channel, b'EHLO example')
+        self.write_line(channel, b'MAIL from: <foo at example.com> size=20')
+        self.write_line(channel, b'RCPT to: <foo at example.com> foo=bar')
+        self.assertEqual(channel.socket.last, self.error_response)
+
+    def test_nothing_accepted(self):
+        server = DummyServer((socket_helper.HOST, 0), ('b', 0))
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr)
+        self.write_line(channel, b'EHLO example')
+        self.write_line(channel, b'MAIL from: <foo at example.com> size=20')
+        self.write_line(channel, b'RCPT to: <foo at example.com>')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+
+class TestMailOptionParsing(unittest.TestCase):
+    error_response = (b'555 MAIL FROM parameters not recognized or not '
+                      b'implemented\r\n')
+
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, channel, line):
+        channel.socket.queue_recv(line)
+        channel.handle_read()
+
+    def test_with_decode_data_true(self):
+        server = DummyServer((socket_helper.HOST, 0), ('b', 0), decode_data=True)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True)
+        self.write_line(channel, b'EHLO example')
+        for line in [
+            b'MAIL from: <foo at example.com> size=20 SMTPUTF8',
+            b'MAIL from: <foo at example.com> size=20 SMTPUTF8 BODY=8BITMIME',
+            b'MAIL from: <foo at example.com> size=20 BODY=UNKNOWN',
+            b'MAIL from: <foo at example.com> size=20 body=8bitmime',
+        ]:
+            self.write_line(channel, line)
+            self.assertEqual(channel.socket.last, self.error_response)
+        self.write_line(channel, b'MAIL from: <foo at example.com> size=20')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+    def test_with_decode_data_false(self):
+        server = DummyServer((socket_helper.HOST, 0), ('b', 0))
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr)
+        self.write_line(channel, b'EHLO example')
+        for line in [
+            b'MAIL from: <foo at example.com> size=20 SMTPUTF8',
+            b'MAIL from: <foo at example.com> size=20 SMTPUTF8 BODY=8BITMIME',
+        ]:
+            self.write_line(channel, line)
+            self.assertEqual(channel.socket.last, self.error_response)
+        self.write_line(
+            channel,
+            b'MAIL from: <foo at example.com> size=20 SMTPUTF8 BODY=UNKNOWN')
+        self.assertEqual(
+            channel.socket.last,
+            b'501 Error: BODY can only be one of 7BIT, 8BITMIME\r\n')
+        self.write_line(
+            channel, b'MAIL from: <foo at example.com> size=20 body=8bitmime')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+    def test_with_enable_smtputf8_true(self):
+        server = DummyServer((socket_helper.HOST, 0), ('b', 0), enable_SMTPUTF8=True)
+        conn, addr = server.accept()
+        channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True)
+        self.write_line(channel, b'EHLO example')
+        self.write_line(
+            channel,
+            b'MAIL from: <foo at example.com> size=20 body=8bitmime smtputf8')
+        self.assertEqual(channel.socket.last, b'250 OK\r\n')
+
+
+class SMTPDChannelTest(unittest.TestCase):
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+        self.server = DummyServer((socket_helper.HOST, 0), ('b', 0),
+                                  decode_data=True)
+        conn, addr = self.server.accept()
+        self.channel = smtpd.SMTPChannel(self.server, conn, addr,
+                                         decode_data=True)
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, line):
+        self.channel.socket.queue_recv(line)
+        self.channel.handle_read()
+
+    def test_broken_connect(self):
+        self.assertRaises(
+            DummyDispatcherBroken, BrokenDummyServer,
+            (socket_helper.HOST, 0), ('b', 0), decode_data=True)
+
+    def test_decode_data_and_enable_SMTPUTF8_raises(self):
+        self.assertRaises(
+            ValueError, smtpd.SMTPChannel,
+            self.server, self.channel.conn, self.channel.addr,
+            enable_SMTPUTF8=True, decode_data=True)
+
+    def test_server_accept(self):
+        self.server.handle_accept()
+
+    def test_missing_data(self):
+        self.write_line(b'')
+        self.assertEqual(self.channel.socket.last,
+                         b'500 Error: bad syntax\r\n')
+
+    def test_EHLO(self):
+        self.write_line(b'EHLO example')
+        self.assertEqual(self.channel.socket.last, b'250 HELP\r\n')
+
+    def test_EHLO_bad_syntax(self):
+        self.write_line(b'EHLO')
+        self.assertEqual(self.channel.socket.last,
+                         b'501 Syntax: EHLO hostname\r\n')
+
+    def test_EHLO_duplicate(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'EHLO example')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Duplicate HELO/EHLO\r\n')
+
+    def test_EHLO_HELO_duplicate(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'HELO example')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Duplicate HELO/EHLO\r\n')
+
+    def test_HELO(self):
+        name = smtpd.socket.getfqdn()
+        self.write_line(b'HELO example')
+        self.assertEqual(self.channel.socket.last,
+                         '250 {}\r\n'.format(name).encode('ascii'))
+
+    def test_HELO_EHLO_duplicate(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'EHLO example')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Duplicate HELO/EHLO\r\n')
+
+    def test_HELP(self):
+        self.write_line(b'HELP')
+        self.assertEqual(self.channel.socket.last,
+                         b'250 Supported commands: EHLO HELO MAIL RCPT ' + \
+                         b'DATA RSET NOOP QUIT VRFY\r\n')
+
+    def test_HELP_command(self):
+        self.write_line(b'HELP MAIL')
+        self.assertEqual(self.channel.socket.last,
+                         b'250 Syntax: MAIL FROM: <address>\r\n')
+
+    def test_HELP_command_unknown(self):
+        self.write_line(b'HELP SPAM')
+        self.assertEqual(self.channel.socket.last,
+                         b'501 Supported commands: EHLO HELO MAIL RCPT ' + \
+                         b'DATA RSET NOOP QUIT VRFY\r\n')
+
+    def test_HELO_bad_syntax(self):
+        self.write_line(b'HELO')
+        self.assertEqual(self.channel.socket.last,
+                         b'501 Syntax: HELO hostname\r\n')
+
+    def test_HELO_duplicate(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'HELO example')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Duplicate HELO/EHLO\r\n')
+
+    def test_HELO_parameter_rejected_when_extensions_not_enabled(self):
+        self.extended_smtp = False
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL from:<foo at example.com> SIZE=1234')
+        self.assertEqual(self.channel.socket.last,
+                         b'501 Syntax: MAIL FROM: <address>\r\n')
+
+    def test_MAIL_allows_space_after_colon(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL from:   <foo at example.com>')
+        self.assertEqual(self.channel.socket.last,
+                         b'250 OK\r\n')
+
+    def test_extended_MAIL_allows_space_after_colon(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from:   <foo at example.com> size=20')
+        self.assertEqual(self.channel.socket.last,
+                         b'250 OK\r\n')
+
+    def test_NOOP(self):
+        self.write_line(b'NOOP')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_HELO_NOOP(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'NOOP')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_NOOP_bad_syntax(self):
+        self.write_line(b'NOOP hi')
+        self.assertEqual(self.channel.socket.last,
+                         b'501 Syntax: NOOP\r\n')
+
+    def test_QUIT(self):
+        self.write_line(b'QUIT')
+        self.assertEqual(self.channel.socket.last, b'221 Bye\r\n')
+
+    def test_HELO_QUIT(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'QUIT')
+        self.assertEqual(self.channel.socket.last, b'221 Bye\r\n')
+
+    def test_QUIT_arg_ignored(self):
+        self.write_line(b'QUIT bye bye')
+        self.assertEqual(self.channel.socket.last, b'221 Bye\r\n')
+
+    def test_bad_state(self):
+        self.channel.smtp_state = 'BAD STATE'
+        self.write_line(b'HELO example')
+        self.assertEqual(self.channel.socket.last,
+                         b'451 Internal confusion\r\n')
+
+    def test_command_too_long(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL from: ' +
+                        b'a' * self.channel.command_size_limit +
+                        b'@example')
+        self.assertEqual(self.channel.socket.last,
+                         b'500 Error: line too long\r\n')
+
+    def test_MAIL_command_limit_extended_with_SIZE(self):
+        self.write_line(b'EHLO example')
+        fill_len = self.channel.command_size_limit - len('MAIL from:<@example>')
+        self.write_line(b'MAIL from:<' +
+                        b'a' * fill_len +
+                        b'@example> SIZE=1234')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+        self.write_line(b'MAIL from:<' +
+                        b'a' * (fill_len + 26) +
+                        b'@example> SIZE=1234')
+        self.assertEqual(self.channel.socket.last,
+                         b'500 Error: line too long\r\n')
+
+    def test_MAIL_command_rejects_SMTPUTF8_by_default(self):
+        self.write_line(b'EHLO example')
+        self.write_line(
+            b'MAIL from: <naive at example.com> BODY=8BITMIME SMTPUTF8')
+        self.assertEqual(self.channel.socket.last[0:1], b'5')
+
+    def test_data_longer_than_default_data_size_limit(self):
+        # Hack the default so we don't have to generate so much data.
+        self.channel.data_size_limit = 1048
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.write_line(b'RCPT To:spam at example')
+        self.write_line(b'DATA')
+        self.write_line(b'A' * self.channel.data_size_limit +
+                        b'A\r\n.')
+        self.assertEqual(self.channel.socket.last,
+                         b'552 Error: Too much mail data\r\n')
+
+    def test_MAIL_size_parameter(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL FROM:<eggs at example> SIZE=512')
+        self.assertEqual(self.channel.socket.last,
+                         b'250 OK\r\n')
+
+    def test_MAIL_invalid_size_parameter(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL FROM:<eggs at example> SIZE=invalid')
+        self.assertEqual(self.channel.socket.last,
+            b'501 Syntax: MAIL FROM: <address> [SP <mail-parameters>]\r\n')
+
+    def test_MAIL_RCPT_unknown_parameters(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL FROM:<eggs at example> ham=green')
+        self.assertEqual(self.channel.socket.last,
+            b'555 MAIL FROM parameters not recognized or not implemented\r\n')
+
+        self.write_line(b'MAIL FROM:<eggs at example>')
+        self.write_line(b'RCPT TO:<eggs at example> ham=green')
+        self.assertEqual(self.channel.socket.last,
+            b'555 RCPT TO parameters not recognized or not implemented\r\n')
+
+    def test_MAIL_size_parameter_larger_than_default_data_size_limit(self):
+        self.channel.data_size_limit = 1048
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL FROM:<eggs at example> SIZE=2096')
+        self.assertEqual(self.channel.socket.last,
+            b'552 Error: message size exceeds fixed maximum message size\r\n')
+
+    def test_need_MAIL(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'RCPT to:spam at example')
+        self.assertEqual(self.channel.socket.last,
+            b'503 Error: need MAIL command\r\n')
+
+    def test_MAIL_syntax_HELO(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL from eggs at example')
+        self.assertEqual(self.channel.socket.last,
+            b'501 Syntax: MAIL FROM: <address>\r\n')
+
+    def test_MAIL_syntax_EHLO(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from eggs at example')
+        self.assertEqual(self.channel.socket.last,
+            b'501 Syntax: MAIL FROM: <address> [SP <mail-parameters>]\r\n')
+
+    def test_MAIL_missing_address(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL from:')
+        self.assertEqual(self.channel.socket.last,
+            b'501 Syntax: MAIL FROM: <address>\r\n')
+
+    def test_MAIL_chevrons(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL from:<eggs at example>')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_MAIL_empty_chevrons(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from:<>')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_MAIL_quoted_localpart(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from: <"Fred Blogs"@example.com>')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com')
+
+    def test_MAIL_quoted_localpart_no_angles(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from: "Fred Blogs"@example.com')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com')
+
+    def test_MAIL_quoted_localpart_with_size(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from: <"Fred Blogs"@example.com> SIZE=1000')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com')
+
+    def test_MAIL_quoted_localpart_with_size_no_angles(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL from: "Fred Blogs"@example.com SIZE=1000')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com')
+
+    def test_nested_MAIL(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL from:eggs at example')
+        self.write_line(b'MAIL from:spam at example')
+        self.assertEqual(self.channel.socket.last,
+            b'503 Error: nested MAIL command\r\n')
+
+    def test_VRFY(self):
+        self.write_line(b'VRFY eggs at example')
+        self.assertEqual(self.channel.socket.last,
+            b'252 Cannot VRFY user, but will accept message and attempt ' + \
+            b'delivery\r\n')
+
+    def test_VRFY_syntax(self):
+        self.write_line(b'VRFY')
+        self.assertEqual(self.channel.socket.last,
+            b'501 Syntax: VRFY <address>\r\n')
+
+    def test_EXPN_not_implemented(self):
+        self.write_line(b'EXPN')
+        self.assertEqual(self.channel.socket.last,
+            b'502 EXPN not implemented\r\n')
+
+    def test_no_HELO_MAIL(self):
+        self.write_line(b'MAIL from:<foo at example.com>')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Error: send HELO first\r\n')
+
+    def test_need_RCPT(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.write_line(b'DATA')
+        self.assertEqual(self.channel.socket.last,
+            b'503 Error: need RCPT command\r\n')
+
+    def test_RCPT_syntax_HELO(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From: eggs at example')
+        self.write_line(b'RCPT to eggs at example')
+        self.assertEqual(self.channel.socket.last,
+            b'501 Syntax: RCPT TO: <address>\r\n')
+
+    def test_RCPT_syntax_EHLO(self):
+        self.write_line(b'EHLO example')
+        self.write_line(b'MAIL From: eggs at example')
+        self.write_line(b'RCPT to eggs at example')
+        self.assertEqual(self.channel.socket.last,
+            b'501 Syntax: RCPT TO: <address> [SP <mail-parameters>]\r\n')
+
+    def test_RCPT_lowercase_to_OK(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From: eggs at example')
+        self.write_line(b'RCPT to: <eggs at example>')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_no_HELO_RCPT(self):
+        self.write_line(b'RCPT to eggs at example')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Error: send HELO first\r\n')
+
+    def test_data_dialog(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.write_line(b'RCPT To:spam at example')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+        self.write_line(b'DATA')
+        self.assertEqual(self.channel.socket.last,
+            b'354 End data with <CR><LF>.<CR><LF>\r\n')
+        self.write_line(b'data\r\nmore\r\n.')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.assertEqual(self.server.messages,
+            [(('peer-address', 'peer-port'),
+              'eggs at example',
+              ['spam at example'],
+              'data\nmore')])
+
+    def test_DATA_syntax(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.write_line(b'RCPT To:spam at example')
+        self.write_line(b'DATA spam')
+        self.assertEqual(self.channel.socket.last, b'501 Syntax: DATA\r\n')
+
+    def test_no_HELO_DATA(self):
+        self.write_line(b'DATA spam')
+        self.assertEqual(self.channel.socket.last,
+                         b'503 Error: send HELO first\r\n')
+
+    def test_data_transparency_section_4_5_2(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.write_line(b'RCPT To:spam at example')
+        self.write_line(b'DATA')
+        self.write_line(b'..\r\n.\r\n')
+        self.assertEqual(self.channel.received_data, '.')
+
+    def test_multiple_RCPT(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.write_line(b'RCPT To:spam at example')
+        self.write_line(b'RCPT To:ham at example')
+        self.write_line(b'DATA')
+        self.write_line(b'data\r\n.')
+        self.assertEqual(self.server.messages,
+            [(('peer-address', 'peer-port'),
+              'eggs at example',
+              ['spam at example','ham at example'],
+              'data')])
+
+    def test_manual_status(self):
+        # checks that the Channel is able to return a custom status message
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.write_line(b'RCPT To:spam at example')
+        self.write_line(b'DATA')
+        self.write_line(b'return status\r\n.')
+        self.assertEqual(self.channel.socket.last, b'250 Okish\r\n')
+
+    def test_RSET(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.write_line(b'RCPT To:spam at example')
+        self.write_line(b'RSET')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.write_line(b'MAIL From:foo at example')
+        self.write_line(b'RCPT To:eggs at example')
+        self.write_line(b'DATA')
+        self.write_line(b'data\r\n.')
+        self.assertEqual(self.server.messages,
+            [(('peer-address', 'peer-port'),
+               'foo at example',
+               ['eggs at example'],
+               'data')])
+
+    def test_HELO_RSET(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'RSET')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_RSET_syntax(self):
+        self.write_line(b'RSET hi')
+        self.assertEqual(self.channel.socket.last, b'501 Syntax: RSET\r\n')
+
+    def test_unknown_command(self):
+        self.write_line(b'UNKNOWN_CMD')
+        self.assertEqual(self.channel.socket.last,
+                         b'500 Error: command "UNKNOWN_CMD" not ' + \
+                         b'recognized\r\n')
+
+    def test_attribute_deprecations(self):
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            spam = self.channel._SMTPChannel__server
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            self.channel._SMTPChannel__server = 'spam'
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            spam = self.channel._SMTPChannel__line
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            self.channel._SMTPChannel__line = 'spam'
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            spam = self.channel._SMTPChannel__state
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            self.channel._SMTPChannel__state = 'spam'
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            spam = self.channel._SMTPChannel__greeting
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            self.channel._SMTPChannel__greeting = 'spam'
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            spam = self.channel._SMTPChannel__mailfrom
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            self.channel._SMTPChannel__mailfrom = 'spam'
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            spam = self.channel._SMTPChannel__rcpttos
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            self.channel._SMTPChannel__rcpttos = 'spam'
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            spam = self.channel._SMTPChannel__data
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            self.channel._SMTPChannel__data = 'spam'
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            spam = self.channel._SMTPChannel__fqdn
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            self.channel._SMTPChannel__fqdn = 'spam'
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            spam = self.channel._SMTPChannel__peer
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            self.channel._SMTPChannel__peer = 'spam'
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            spam = self.channel._SMTPChannel__conn
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            self.channel._SMTPChannel__conn = 'spam'
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            spam = self.channel._SMTPChannel__addr
+        with warnings_helper.check_warnings(('', DeprecationWarning)):
+            self.channel._SMTPChannel__addr = 'spam'
+
+ at unittest.skipUnless(socket_helper.IPV6_ENABLED, "IPv6 not enabled")
+class SMTPDChannelIPv6Test(SMTPDChannelTest):
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+        self.server = DummyServer((socket_helper.HOSTv6, 0), ('b', 0),
+                                  decode_data=True)
+        conn, addr = self.server.accept()
+        self.channel = smtpd.SMTPChannel(self.server, conn, addr,
+                                         decode_data=True)
+
+class SMTPDChannelWithDataSizeLimitTest(unittest.TestCase):
+
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+        self.server = DummyServer((socket_helper.HOST, 0), ('b', 0),
+                                  decode_data=True)
+        conn, addr = self.server.accept()
+        # Set DATA size limit to 32 bytes for easy testing
+        self.channel = smtpd.SMTPChannel(self.server, conn, addr, 32,
+                                         decode_data=True)
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, line):
+        self.channel.socket.queue_recv(line)
+        self.channel.handle_read()
+
+    def test_data_limit_dialog(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.write_line(b'RCPT To:spam at example')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+        self.write_line(b'DATA')
+        self.assertEqual(self.channel.socket.last,
+            b'354 End data with <CR><LF>.<CR><LF>\r\n')
+        self.write_line(b'data\r\nmore\r\n.')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.assertEqual(self.server.messages,
+            [(('peer-address', 'peer-port'),
+              'eggs at example',
+              ['spam at example'],
+              'data\nmore')])
+
+    def test_data_limit_dialog_too_much_data(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+        self.write_line(b'RCPT To:spam at example')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+        self.write_line(b'DATA')
+        self.assertEqual(self.channel.socket.last,
+            b'354 End data with <CR><LF>.<CR><LF>\r\n')
+        self.write_line(b'This message is longer than 32 bytes\r\n.')
+        self.assertEqual(self.channel.socket.last,
+                         b'552 Error: Too much mail data\r\n')
+
+
+class SMTPDChannelWithDecodeDataFalse(unittest.TestCase):
+
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+        self.server = DummyServer((socket_helper.HOST, 0), ('b', 0))
+        conn, addr = self.server.accept()
+        self.channel = smtpd.SMTPChannel(self.server, conn, addr)
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, line):
+        self.channel.socket.queue_recv(line)
+        self.channel.handle_read()
+
+    def test_ascii_data(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.write_line(b'RCPT To:spam at example')
+        self.write_line(b'DATA')
+        self.write_line(b'plain ascii text')
+        self.write_line(b'.')
+        self.assertEqual(self.channel.received_data, b'plain ascii text')
+
+    def test_utf8_data(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.write_line(b'RCPT To:spam at example')
+        self.write_line(b'DATA')
+        self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87')
+        self.write_line(b'and some plain ascii')
+        self.write_line(b'.')
+        self.assertEqual(
+            self.channel.received_data,
+            b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87\n'
+                b'and some plain ascii')
+
+
+class SMTPDChannelWithDecodeDataTrue(unittest.TestCase):
+
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+        self.server = DummyServer((socket_helper.HOST, 0), ('b', 0),
+                                  decode_data=True)
+        conn, addr = self.server.accept()
+        # Set decode_data to True
+        self.channel = smtpd.SMTPChannel(self.server, conn, addr,
+                decode_data=True)
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, line):
+        self.channel.socket.queue_recv(line)
+        self.channel.handle_read()
+
+    def test_ascii_data(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.write_line(b'RCPT To:spam at example')
+        self.write_line(b'DATA')
+        self.write_line(b'plain ascii text')
+        self.write_line(b'.')
+        self.assertEqual(self.channel.received_data, 'plain ascii text')
+
+    def test_utf8_data(self):
+        self.write_line(b'HELO example')
+        self.write_line(b'MAIL From:eggs at example')
+        self.write_line(b'RCPT To:spam at example')
+        self.write_line(b'DATA')
+        self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87')
+        self.write_line(b'and some plain ascii')
+        self.write_line(b'.')
+        self.assertEqual(
+            self.channel.received_data,
+            'utf8 enriched text: ???\nand some plain ascii')
+
+
+class SMTPDChannelTestWithEnableSMTPUTF8True(unittest.TestCase):
+    def setUp(self):
+        smtpd.socket = asyncore.socket = mock_socket
+        self.old_debugstream = smtpd.DEBUGSTREAM
+        self.debug = smtpd.DEBUGSTREAM = io.StringIO()
+        self.server = DummyServer((socket_helper.HOST, 0), ('b', 0),
+                                  enable_SMTPUTF8=True)
+        conn, addr = self.server.accept()
+        self.channel = smtpd.SMTPChannel(self.server, conn, addr,
+                                         enable_SMTPUTF8=True)
+
+    def tearDown(self):
+        asyncore.close_all()
+        asyncore.socket = smtpd.socket = socket
+        smtpd.DEBUGSTREAM = self.old_debugstream
+
+    def write_line(self, line):
+        self.channel.socket.queue_recv(line)
+        self.channel.handle_read()
+
+    def test_MAIL_command_accepts_SMTPUTF8_when_announced(self):
+        self.write_line(b'EHLO example')
+        self.write_line(
+            'MAIL from: <nai?ve at example.com> BODY=8BITMIME SMTPUTF8'.encode(
+                'utf-8')
+        )
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_process_smtputf8_message(self):
+        self.write_line(b'EHLO example')
+        for mail_parameters in [b'', b'BODY=8BITMIME SMTPUTF8']:
+            self.write_line(b'MAIL from: <a at example> ' + mail_parameters)
+            self.assertEqual(self.channel.socket.last[0:3], b'250')
+            self.write_line(b'rcpt to:<b at example.com>')
+            self.assertEqual(self.channel.socket.last[0:3], b'250')
+            self.write_line(b'data')
+            self.assertEqual(self.channel.socket.last[0:3], b'354')
+            self.write_line(b'c\r\n.')
+            if mail_parameters == b'':
+                self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+            else:
+                self.assertEqual(self.channel.socket.last,
+                                 b'250 SMTPUTF8 message okish\r\n')
+
+    def test_utf8_data(self):
+        self.write_line(b'EHLO example')
+        self.write_line(
+            'MAIL From: nai?ve at exampl? BODY=8BITMIME SMTPUTF8'.encode('utf-8'))
+        self.assertEqual(self.channel.socket.last[0:3], b'250')
+        self.write_line('RCPT To:sp?m at exampl?'.encode('utf-8'))
+        self.assertEqual(self.channel.socket.last[0:3], b'250')
+        self.write_line(b'DATA')
+        self.assertEqual(self.channel.socket.last[0:3], b'354')
+        self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87')
+        self.write_line(b'.')
+        self.assertEqual(
+            self.channel.received_data,
+            b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87')
+
+    def test_MAIL_command_limit_extended_with_SIZE_and_SMTPUTF8(self):
+        self.write_line(b'ehlo example')
+        fill_len = (512 + 26 + 10) - len('mail from:<@example>')
+        self.write_line(b'MAIL from:<' +
+                        b'a' * (fill_len + 1) +
+                        b'@example>')
+        self.assertEqual(self.channel.socket.last,
+                         b'500 Error: line too long\r\n')
+        self.write_line(b'MAIL from:<' +
+                        b'a' * fill_len +
+                        b'@example>')
+        self.assertEqual(self.channel.socket.last, b'250 OK\r\n')
+
+    def test_multiple_emails_with_extended_command_length(self):
+        self.write_line(b'ehlo example')
+        fill_len = (512 + 26 + 10) - len('mail from:<@example>')
+        for char in [b'a', b'b', b'c']:
+            self.write_line(b'MAIL from:<' + char * fill_len + b'a at example>')
+            self.assertEqual(self.channel.socket.last[0:3], b'500')
+            self.write_line(b'MAIL from:<' + char * fill_len + b'@example>')
+            self.assertEqual(self.channel.socket.last[0:3], b'250')
+            self.write_line(b'rcpt to:<hans at example.com>')
+            self.assertEqual(self.channel.socket.last[0:3], b'250')
+            self.write_line(b'data')
+            self.assertEqual(self.channel.socket.last[0:3], b'354')
+            self.write_line(b'test\r\n.')
+            self.assertEqual(self.channel.socket.last[0:3], b'250')
+
+
+class MiscTestCase(unittest.TestCase):
+    def test__all__(self):
+        not_exported = {
+            "program", "Devnull", "DEBUGSTREAM", "NEWLINE", "COMMASPACE",
+            "DATA_SIZE_DEFAULT", "usage", "Options", "parseargs",
+        }
+        support.check__all__(self, smtpd, not_exported=not_exported)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py
index bdce3e37095e4..9761a37251731 100644
--- a/Lib/test/test_smtplib.py
+++ b/Lib/test/test_smtplib.py
@@ -18,13 +18,17 @@
 
 import unittest
 from test import support, mock_socket
-from test.support import _asyncore as asyncore
-from test.support import _smtpd as smtpd
 from test.support import hashlib_helper
 from test.support import socket_helper
 from test.support import threading_helper
 from unittest.mock import Mock
 
+import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', DeprecationWarning)
+    import asyncore
+    import smtpd
+
 HOST = socket_helper.HOST
 
 if sys.platform == 'darwin':
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 88eeb07afa036..981e2fe82ee46 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -4,7 +4,6 @@
 import unittest
 import unittest.mock
 from test import support
-from test.support import _asyncore as asyncore
 from test.support import import_helper
 from test.support import os_helper
 from test.support import socket_helper
@@ -31,6 +30,10 @@
 except ImportError:
     ctypes = None
 
+import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', DeprecationWarning)
+    import asyncore
 
 ssl = import_helper.import_module("ssl")
 import _ssl
diff --git a/Misc/NEWS.d/next/Library/2021-11-11-12-59-10.bpo-28533.68mMZa.rst b/Misc/NEWS.d/next/Library/2021-11-11-12-59-10.bpo-28533.68mMZa.rst
deleted file mode 100644
index 49243815e31f2..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-11-11-12-59-10.bpo-28533.68mMZa.rst
+++ /dev/null
@@ -1,2 +0,0 @@
-Remove the ``asyncore`` and ``asynchat`` modules, deprecated in Python 3.6:
-use the :mod:`asyncio` module instead. Patch by Victor Stinner.
diff --git a/Misc/NEWS.d/next/Library/2021-11-11-12-59-49.bpo-28533.LvIFCQ.rst b/Misc/NEWS.d/next/Library/2021-11-11-12-59-49.bpo-28533.LvIFCQ.rst
deleted file mode 100644
index 67fb3686b1d86..0000000000000
--- a/Misc/NEWS.d/next/Library/2021-11-11-12-59-49.bpo-28533.LvIFCQ.rst
+++ /dev/null
@@ -1,3 +0,0 @@
-Remove the ``smtpd`` module, deprecated in Python 3.6: the `aiosmtpd
-<https://aiosmtpd.readthedocs.io/>`__ module can be used instead, it is based
-on asyncio. Patch by Victor Stinner.
diff --git a/PCbuild/lib.pyproj b/PCbuild/lib.pyproj
index 7dd40ad64a7e3..43c570f1dab37 100644
--- a/PCbuild/lib.pyproj
+++ b/PCbuild/lib.pyproj
@@ -24,6 +24,7 @@
     <Compile Include="antigravity.py" />
     <Compile Include="argparse.py" />
     <Compile Include="ast.py" />
+    <Compile Include="asynchat.py" />
     <Compile Include="asyncio\base_events.py" />
     <Compile Include="asyncio\base_futures.py" />
     <Compile Include="asyncio\base_subprocess.py" />
@@ -49,6 +50,7 @@
     <Compile Include="asyncio\windows_events.py" />
     <Compile Include="asyncio\windows_utils.py" />
     <Compile Include="asyncio\__init__.py" />
+    <Compile Include="asyncore.py" />
     <Compile Include="base64.py" />
     <Compile Include="bdb.py" />
     <Compile Include="bisect.py" />
@@ -720,6 +722,7 @@
     <Compile Include="shutil.py" />
     <Compile Include="signal.py" />
     <Compile Include="site.py" />
+    <Compile Include="smtpd.py" />
     <Compile Include="smtplib.py" />
     <Compile Include="sndhdr.py" />
     <Compile Include="socket.py" />
@@ -857,6 +860,7 @@
     <Compile Include="test\test_asdl_parser.py" />
     <Compile Include="test\test_ast.py" />
     <Compile Include="test\test_asyncgen.py" />
+    <Compile Include="test\test_asynchat.py" />
     <Compile Include="test\test_asyncio\echo.py" />
     <Compile Include="test\test_asyncio\echo2.py" />
     <Compile Include="test\test_asyncio\echo3.py" />
@@ -878,6 +882,7 @@
     <Compile Include="test\test_asyncio\test_windows_utils.py" />
     <Compile Include="test\test_asyncio\__init__.py" />
     <Compile Include="test\test_asyncio\__main__.py" />
+    <Compile Include="test\test_asyncore.py" />
     <Compile Include="test\test_atexit.py" />
     <Compile Include="test\test_audioop.py" />
     <Compile Include="test\test_augassign.py" />
@@ -1258,6 +1263,7 @@
     <Compile Include="test\test_signal.py" />
     <Compile Include="test\test_site.py" />
     <Compile Include="test\test_slice.py" />
+    <Compile Include="test\test_smtpd.py" />
     <Compile Include="test\test_smtplib.py" />
     <Compile Include="test\test_smtpnet.py" />
     <Compile Include="test\test_sndhdr.py" />
diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h
index 7d421081b2ab8..754fa94e35eba 100644
--- a/Python/stdlib_module_names.h
+++ b/Python/stdlib_module_names.h
@@ -96,7 +96,9 @@ static const char* _Py_stdlib_module_names[] = {
 "argparse",
 "array",
 "ast",
+"asynchat",
 "asyncio",
+"asyncore",
 "atexit",
 "audioop",
 "base64",
@@ -240,6 +242,7 @@ static const char* _Py_stdlib_module_names[] = {
 "shutil",
 "signal",
 "site",
+"smtpd",
 "smtplib",
 "sndhdr",
 "socket",


From webhook-mailer at python.org  Tue Dec  7 06:45:22 2021
From: webhook-mailer at python.org (vsajip)
Date: Tue, 07 Dec 2021 11:45:22 -0000
Subject: [Python-checkins] [3.10] bpo-35821: Add an example to
 Logger.propagate documentation. (GH-29841) (GH-29957)
Message-ID: <mailman.75.1638877523.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/f78c229b4ec8621a9b15c6396b6c91518e8975d6
commit: f78c229b4ec8621a9b15c6396b6c91518e8975d6
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: vsajip <vinay_sajip at yahoo.co.uk>
date: 2021-12-07T11:45:13Z
summary:

[3.10] bpo-35821: Add an example to Logger.propagate documentation. (GH-29841) (GH-29957)

files:
M Doc/library/logging.rst

diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
index bb1bbf0e3708c..ade4a3c6b261e 100644
--- a/Doc/library/logging.rst
+++ b/Doc/library/logging.rst
@@ -80,6 +80,15 @@ is the module's name in the Python package namespace.
       If this evaluates to false, logging messages are not passed to the handlers
       of ancestor loggers.
 
+      Spelling it out with an example: If the propagate attribute of the logger named
+      `A.B.C` evaluates to true, any event logged to `A.B.C` via a method call such as
+      `logging.getLogger('A.B.C').error(...)` will [subject to passing that logger's
+      level and filter settings] be passed in turn to any handlers attached to loggers
+      named `A.B`, `A` and the root logger, after first being passed to any handlers
+      attached to `A.B.C`. If any logger in the chain `A.B.C`, `A.B`, `A` has its
+      `propagate` attribute set to false, then that is the last logger whose handlers
+      are offered the event to handle, and propagation stops at that point.
+
       The constructor sets this attribute to ``True``.
 
       .. note:: If you attach a handler to a logger *and* one or more of its


From webhook-mailer at python.org  Tue Dec  7 06:45:53 2021
From: webhook-mailer at python.org (vsajip)
Date: Tue, 07 Dec 2021 11:45:53 -0000
Subject: [Python-checkins] [3.9] bpo-35821: Add an example to
 Logger.propagate documentation. (GH-29841) (GH-29958)
Message-ID: <mailman.76.1638877554.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/e688568cdfe758a2316ecaf0c8df868d5dde0d83
commit: e688568cdfe758a2316ecaf0c8df868d5dde0d83
branch: 3.9
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: vsajip <vinay_sajip at yahoo.co.uk>
date: 2021-12-07T11:45:49Z
summary:

[3.9] bpo-35821: Add an example to Logger.propagate documentation. (GH-29841) (GH-29958)

files:
M Doc/library/logging.rst

diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
index e72a898a11363..705b31d604500 100644
--- a/Doc/library/logging.rst
+++ b/Doc/library/logging.rst
@@ -80,6 +80,15 @@ is the module's name in the Python package namespace.
       If this evaluates to false, logging messages are not passed to the handlers
       of ancestor loggers.
 
+      Spelling it out with an example: If the propagate attribute of the logger named
+      `A.B.C` evaluates to true, any event logged to `A.B.C` via a method call such as
+      `logging.getLogger('A.B.C').error(...)` will [subject to passing that logger's
+      level and filter settings] be passed in turn to any handlers attached to loggers
+      named `A.B`, `A` and the root logger, after first being passed to any handlers
+      attached to `A.B.C`. If any logger in the chain `A.B.C`, `A.B`, `A` has its
+      `propagate` attribute set to false, then that is the last logger whose handlers
+      are offered the event to handle, and propagation stops at that point.
+
       The constructor sets this attribute to ``True``.
 
       .. note:: If you attach a handler to a logger *and* one or more of its


From webhook-mailer at python.org  Tue Dec  7 07:00:11 2021
From: webhook-mailer at python.org (serhiy-storchaka)
Date: Tue, 07 Dec 2021 12:00:11 -0000
Subject: [Python-checkins] bpo-45664: Fix resolve_bases() and new_class()
 for GenericAlias instance as a base (GH-29298) (GH-29928)
Message-ID: <mailman.77.1638878412.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/bffce2cbb5543bc63a67e33ad599328a12f2b00a
commit: bffce2cbb5543bc63a67e33ad599328a12f2b00a
branch: 3.9
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2021-12-07T14:00:06+02:00
summary:

bpo-45664: Fix resolve_bases() and new_class() for GenericAlias instance as a base (GH-29298) (GH-29928)

(cherry picked from commit 2b318ce1c988b7b6e3caf293d55f289e066b6e0f)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
A Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
M Lib/test/test_types.py
M Lib/types.py

diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py
index 49dc5bf40e3ed..fe432a4deb803 100644
--- a/Lib/test/test_types.py
+++ b/Lib/test/test_types.py
@@ -7,6 +7,7 @@
 import locale
 import sys
 import types
+import typing
 import unittest.mock
 import weakref
 
@@ -888,6 +889,17 @@ def __mro_entries__(self, bases):
         self.assertEqual(D.__orig_bases__, (c,))
         self.assertEqual(D.__mro__, (D, A, object))
 
+    def test_new_class_with_mro_entry_genericalias(self):
+        L1 = types.new_class('L1', (typing.List[int],), {})
+        self.assertEqual(L1.__bases__, (list, typing.Generic))
+        self.assertEqual(L1.__orig_bases__, (typing.List[int],))
+        self.assertEqual(L1.__mro__, (L1, list, typing.Generic, object))
+
+        L2 = types.new_class('L2', (list[int],), {})
+        self.assertEqual(L2.__bases__, (list,))
+        self.assertEqual(L2.__orig_bases__, (list[int],))
+        self.assertEqual(L2.__mro__, (L2, list, object))
+
     def test_new_class_with_mro_entry_none(self):
         class A: pass
         class B: pass
@@ -1003,6 +1015,11 @@ def __mro_entries__(self, bases):
         for bases in [x, y, z, t]:
             self.assertIs(types.resolve_bases(bases), bases)
 
+    def test_resolve_bases_with_mro_entry(self):
+        self.assertEqual(types.resolve_bases((typing.List[int],)),
+                         (list, typing.Generic))
+        self.assertEqual(types.resolve_bases((list[int],)), (list,))
+
     def test_metaclass_derivation(self):
         # issue1294232: correct metaclass calculation
         new_calls = []  # to check the order of __new__ calls
diff --git a/Lib/types.py b/Lib/types.py
index ad2020ec69b63..f15fda5933a3f 100644
--- a/Lib/types.py
+++ b/Lib/types.py
@@ -82,7 +82,7 @@ def resolve_bases(bases):
     updated = False
     shift = 0
     for i, base in enumerate(bases):
-        if isinstance(base, type):
+        if isinstance(base, type) and not isinstance(base, GenericAlias):
             continue
         if not hasattr(base, "__mro_entries__"):
             continue
diff --git a/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst b/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
new file mode 100644
index 0000000000000..573a569845a87
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-10-28-23-40-54.bpo-45664.7dqtxQ.rst
@@ -0,0 +1,2 @@
+Fix :func:`types.resolve_bases` and :func:`types.new_class` for
+:class:`types.GenericAlias` instance as a base.


From webhook-mailer at python.org  Tue Dec  7 08:02:34 2021
From: webhook-mailer at python.org (pablogsal)
Date: Tue, 07 Dec 2021 13:02:34 -0000
Subject: [Python-checkins] bpo-46004: Fix error location for loops with
 invalid targets (GH-29959)
Message-ID: <mailman.78.1638882155.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/1c7a1c3be08ee911d347fffd2716f3911ba751f9
commit: 1c7a1c3be08ee911d347fffd2716f3911ba751f9
branch: main
author: Pablo Galindo Salgado <Pablogsal at gmail.com>
committer: pablogsal <Pablogsal at gmail.com>
date: 2021-12-07T13:02:15Z
summary:

bpo-46004: Fix error location for loops with invalid targets (GH-29959)

files:
A Misc/NEWS.d/next/Core and Builtins/2021-12-07-11-24-24.bpo-46004.TTEU1p.rst
M Lib/test/test_exceptions.py
M Parser/pegen.h

diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index e4b7b8f0a6406..a954ecb8f7c25 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -234,6 +234,7 @@ def testSyntaxErrorOffset(self):
         check("ages = {'Alice'=22, 'Bob'=23}", 1, 16)
         check('match ...:\n    case {**rest, "key": value}:\n        ...', 2, 19)
         check("[a b c d e f]", 1, 2)
+        check("for x yfff:", 1, 7)
 
         # Errors thrown by compile.c
         check('class foo:return 1', 1, 11)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-07-11-24-24.bpo-46004.TTEU1p.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-07-11-24-24.bpo-46004.TTEU1p.rst
new file mode 100644
index 0000000000000..199bccf8166f0
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-12-07-11-24-24.bpo-46004.TTEU1p.rst	
@@ -0,0 +1,2 @@
+Fix the :exc:`SyntaxError` location for errors involving for loops with
+invalid targets. Patch by Pablo Galindo
diff --git a/Parser/pegen.h b/Parser/pegen.h
index 78e75d7060cf1..caba34e535b6a 100644
--- a/Parser/pegen.h
+++ b/Parser/pegen.h
@@ -227,8 +227,9 @@ _RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e)
             msg,
             _PyPegen_get_expr_name(invalid_target)
         );
+        return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(invalid_target, "invalid syntax");
     }
-    return RAISE_SYNTAX_ERROR("invalid syntax");
+    return NULL;
 }
 
 // Action utility functions


From webhook-mailer at python.org  Tue Dec  7 10:23:42 2021
From: webhook-mailer at python.org (pablogsal)
Date: Tue, 07 Dec 2021 15:23:42 -0000
Subject: [Python-checkins] [3.10] bpo-46004: Fix error location for loops
 with invalid targets (GH-29959). (GH-29961)
Message-ID: <mailman.79.1638890624.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/c52141200364898818956a73b955f7c04f634dc8
commit: c52141200364898818956a73b955f7c04f634dc8
branch: 3.10
author: Pablo Galindo Salgado <Pablogsal at gmail.com>
committer: pablogsal <Pablogsal at gmail.com>
date: 2021-12-07T15:23:33Z
summary:

[3.10] bpo-46004: Fix error location for loops with invalid targets (GH-29959). (GH-29961)

(cherry picked from commit 1c7a1c3be08ee911d347fffd2716f3911ba751f9)

Co-authored-by: Pablo Galindo Salgado <Pablogsal at gmail.com>

files:
A Misc/NEWS.d/next/Core and Builtins/2021-12-07-11-24-24.bpo-46004.TTEU1p.rst
M Lib/test/test_exceptions.py
M Parser/pegen.h

diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index ad2864bc41637..bc5f83a5d2dc8 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -234,6 +234,7 @@ def testSyntaxErrorOffset(self):
         check("ages = {'Alice'=22, 'Bob'=23}", 1, 16)
         check('match ...:\n    case {**rest, "key": value}:\n        ...', 2, 19)
         check("[a b c d e f]", 1, 2)
+        check("for x yfff:", 1, 7)
 
         # Errors thrown by compile.c
         check('class foo:return 1', 1, 11)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-07-11-24-24.bpo-46004.TTEU1p.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-07-11-24-24.bpo-46004.TTEU1p.rst
new file mode 100644
index 0000000000000..199bccf8166f0
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-12-07-11-24-24.bpo-46004.TTEU1p.rst	
@@ -0,0 +1,2 @@
+Fix the :exc:`SyntaxError` location for errors involving for loops with
+invalid targets. Patch by Pablo Galindo
diff --git a/Parser/pegen.h b/Parser/pegen.h
index 04c7b9d19bb8a..29d48052e4bc9 100644
--- a/Parser/pegen.h
+++ b/Parser/pegen.h
@@ -321,8 +321,9 @@ _RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e)
             msg,
             _PyPegen_get_expr_name(invalid_target)
         );
+        return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(invalid_target, "invalid syntax");
     }
-    return RAISE_SYNTAX_ERROR("invalid syntax");
+    return NULL;
 }
 
 void *_PyPegen_arguments_parsing_error(Parser *, expr_ty);


From webhook-mailer at python.org  Tue Dec  7 10:25:23 2021
From: webhook-mailer at python.org (Fidget-Spinner)
Date: Tue, 07 Dec 2021 15:25:23 -0000
Subject: [Python-checkins] bpo-35821: Fix restructuredtext code formatting
 in logging.rst (GH-29963)
Message-ID: <mailman.80.1638890724.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/c7e7a4b969b5728d4b4f3c59bf98e1e830d5c6d6
commit: c7e7a4b969b5728d4b4f3c59bf98e1e830d5c6d6
branch: main
author: Ken Jin <28750310+Fidget-Spinner at users.noreply.github.com>
committer: Fidget-Spinner <28750310+Fidget-Spinner at users.noreply.github.com>
date: 2021-12-07T23:25:13+08:00
summary:

bpo-35821: Fix restructuredtext code formatting in logging.rst (GH-29963)

files:
M Doc/library/logging.rst

diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
index e7c64e5ba4036..ea6494f219ae7 100644
--- a/Doc/library/logging.rst
+++ b/Doc/library/logging.rst
@@ -81,12 +81,12 @@ is the module's name in the Python package namespace.
       of ancestor loggers.
 
       Spelling it out with an example: If the propagate attribute of the logger named
-      `A.B.C` evaluates to true, any event logged to `A.B.C` via a method call such as
-      `logging.getLogger('A.B.C').error(...)` will [subject to passing that logger's
+      ``A.B.C`` evaluates to true, any event logged to ``A.B.C`` via a method call such as
+      ``logging.getLogger('A.B.C').error(...)`` will [subject to passing that logger's
       level and filter settings] be passed in turn to any handlers attached to loggers
-      named `A.B`, `A` and the root logger, after first being passed to any handlers
-      attached to `A.B.C`. If any logger in the chain `A.B.C`, `A.B`, `A` has its
-      `propagate` attribute set to false, then that is the last logger whose handlers
+      named ``A.B``, ``A`` and the root logger, after first being passed to any handlers
+      attached to ``A.B.C``. If any logger in the chain ``A.B.C``, ``A.B``, ``A`` has its
+      ``propagate`` attribute set to false, then that is the last logger whose handlers
       are offered the event to handle, and propagation stops at that point.
 
       The constructor sets this attribute to ``True``.


From webhook-mailer at python.org  Tue Dec  7 10:47:43 2021
From: webhook-mailer at python.org (miss-islington)
Date: Tue, 07 Dec 2021 15:47:43 -0000
Subject: [Python-checkins] [3.10] bpo-35821: Fix restructuredtext code
 formatting in logging.rst (GH-29963) (GH-29965)
Message-ID: <mailman.81.1638892064.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/14f03ce6e8a33cc8b45f11c4d428193fc7c4a145
commit: 14f03ce6e8a33cc8b45f11c4d428193fc7c4a145
branch: 3.10
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-07T07:47:31-08:00
summary:

[3.10] bpo-35821: Fix restructuredtext code formatting in logging.rst (GH-29963) (GH-29965)



(cherry picked from commit c7e7a4b969b5728d4b4f3c59bf98e1e830d5c6d6)


Co-authored-by: Ken Jin <28750310+Fidget-Spinner at users.noreply.github.com>

Automerge-Triggered-By: GH:Fidget-Spinner

files:
M Doc/library/logging.rst

diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
index ade4a3c6b261e..74b1f69caec5c 100644
--- a/Doc/library/logging.rst
+++ b/Doc/library/logging.rst
@@ -81,12 +81,12 @@ is the module's name in the Python package namespace.
       of ancestor loggers.
 
       Spelling it out with an example: If the propagate attribute of the logger named
-      `A.B.C` evaluates to true, any event logged to `A.B.C` via a method call such as
-      `logging.getLogger('A.B.C').error(...)` will [subject to passing that logger's
+      ``A.B.C`` evaluates to true, any event logged to ``A.B.C`` via a method call such as
+      ``logging.getLogger('A.B.C').error(...)`` will [subject to passing that logger's
       level and filter settings] be passed in turn to any handlers attached to loggers
-      named `A.B`, `A` and the root logger, after first being passed to any handlers
-      attached to `A.B.C`. If any logger in the chain `A.B.C`, `A.B`, `A` has its
-      `propagate` attribute set to false, then that is the last logger whose handlers
+      named ``A.B``, ``A`` and the root logger, after first being passed to any handlers
+      attached to ``A.B.C``. If any logger in the chain ``A.B.C``, ``A.B``, ``A`` has its
+      ``propagate`` attribute set to false, then that is the last logger whose handlers
       are offered the event to handle, and propagation stops at that point.
 
       The constructor sets this attribute to ``True``.


From webhook-mailer at python.org  Tue Dec  7 10:48:42 2021
From: webhook-mailer at python.org (miss-islington)
Date: Tue, 07 Dec 2021 15:48:42 -0000
Subject: [Python-checkins] bpo-35821: Fix restructuredtext code formatting
 in logging.rst (GH-29963)
Message-ID: <mailman.82.1638892123.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/db42809d299d1bc3a07b29fabe8f74fa02a7e59e
commit: db42809d299d1bc3a07b29fabe8f74fa02a7e59e
branch: 3.9
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: miss-islington <31488909+miss-islington at users.noreply.github.com>
date: 2021-12-07T07:48:32-08:00
summary:

bpo-35821: Fix restructuredtext code formatting in logging.rst (GH-29963)

(cherry picked from commit c7e7a4b969b5728d4b4f3c59bf98e1e830d5c6d6)

Co-authored-by: Ken Jin <28750310+Fidget-Spinner at users.noreply.github.com>

files:
M Doc/library/logging.rst

diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst
index 705b31d604500..9d6167cca5294 100644
--- a/Doc/library/logging.rst
+++ b/Doc/library/logging.rst
@@ -81,12 +81,12 @@ is the module's name in the Python package namespace.
       of ancestor loggers.
 
       Spelling it out with an example: If the propagate attribute of the logger named
-      `A.B.C` evaluates to true, any event logged to `A.B.C` via a method call such as
-      `logging.getLogger('A.B.C').error(...)` will [subject to passing that logger's
+      ``A.B.C`` evaluates to true, any event logged to ``A.B.C`` via a method call such as
+      ``logging.getLogger('A.B.C').error(...)`` will [subject to passing that logger's
       level and filter settings] be passed in turn to any handlers attached to loggers
-      named `A.B`, `A` and the root logger, after first being passed to any handlers
-      attached to `A.B.C`. If any logger in the chain `A.B.C`, `A.B`, `A` has its
-      `propagate` attribute set to false, then that is the last logger whose handlers
+      named ``A.B``, ``A`` and the root logger, after first being passed to any handlers
+      attached to ``A.B.C``. If any logger in the chain ``A.B.C``, ``A.B``, ``A`` has its
+      ``propagate`` attribute set to false, then that is the last logger whose handlers
       are offered the event to handle, and propagation stops at that point.
 
       The constructor sets this attribute to ``True``.


From webhook-mailer at python.org  Tue Dec  7 11:03:04 2021
From: webhook-mailer at python.org (markshannon)
Date: Tue, 07 Dec 2021 16:03:04 -0000
Subject: [Python-checkins] bpo-45947: Place dict and values pointer at fixed
 (negative) offset just before GC header. (GH-29879)
Message-ID: <mailman.83.1638892986.15285.python-checkins@python.org>

https://github.com/python/cpython/commit/8319114feedd2a5b77378bba24eb9fb2689c5033
commit: 8319114feedd2a5b77378bba24eb9fb2689c5033
branch: main
author: Mark Shannon <mark at hotpy.org>
committer: markshannon <mark at hotpy.org>
date: 2021-12-07T16:02:53Z
summary:

bpo-45947: Place dict and values pointer at fixed (negative) offset just before GC header. (GH-29879)

* Place __dict__ immediately before GC header for plain Python objects.

* Fix up lazy dict creation logic to use managed dict pointers.

* Manage values pointer, placing them directly before managed dict pointers.

* Convert hint-based load/store attr specialization target managed dict classes.

* Specialize LOAD_METHOD for managed dict objects.

* Remove unsafe _PyObject_GC_Calloc function.

* Remove unsafe _PyObject_GC_Malloc() function.

* Add comment explaning use of Py_TPFLAGS_MANAGED_DICT.

files:
A Misc/NEWS.d/next/Core and Builtins/2021-12-01-14-06-36.bpo-45947.1XPPm_.rst
M Include/cpython/object.h
M Include/cpython/objimpl.h
M Include/internal/pycore_object.h
M Include/object.h
M Lib/test/test_stable_abi_ctypes.py
M Lib/test/test_sys.py
M Misc/stable_abi.txt
M Modules/_testcapimodule.c
M Modules/gcmodule.c
M Objects/dictobject.c
M Objects/exceptions.c
M Objects/object.c
M Objects/typeobject.c
M PC/python3dll.c
M Python/ceval.c
M Python/specialize.c
M Python/sysmodule.c
M Tools/gdb/libpython.py

diff --git a/Include/cpython/object.h b/Include/cpython/object.h
index 3a8a256e3b9ae..0c3957aff4b5d 100644
--- a/Include/cpython/object.h
+++ b/Include/cpython/object.h
@@ -270,7 +270,6 @@ struct _typeobject {
 
     destructor tp_finalize;
     vectorcallfunc tp_vectorcall;
-    Py_ssize_t tp_inline_values_offset;
 };
 
 /* The *real* layout of a type object when allocated on the heap */
diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h
index d83700e2a4647..4a905c25cc845 100644
--- a/Include/cpython/objimpl.h
+++ b/Include/cpython/objimpl.h
@@ -90,9 +90,6 @@ PyAPI_FUNC(int) PyObject_IS_GC(PyObject *obj);
 #  define _PyGC_FINALIZED(o) PyObject_GC_IsFinalized(o)
 #endif
 
-PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t size);
-PyAPI_FUNC(PyObject *) _PyObject_GC_Calloc(size_t size);
-
 
 /* Test if a type supports weak references */
 #define PyType_SUPPORTS_WEAKREFS(t) ((t)->tp_weaklistoffset > 0)
diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
index 3c126aaef1187..9041a4dc8a3ce 100644
--- a/Include/internal/pycore_object.h
+++ b/Include/internal/pycore_object.h
@@ -168,6 +168,15 @@ _PyObject_IS_GC(PyObject *obj)
 // Fast inlined version of PyType_IS_GC()
 #define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC)
 
+static inline size_t
+_PyType_PreHeaderSize(PyTypeObject *tp)
+{
+    return _PyType_IS_GC(tp) * sizeof(PyGC_Head) +
+        _PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT) * 2 * sizeof(PyObject *);
+}
+
+void _PyObject_GC_Link(PyObject *op);
+
 // Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL));
 extern int _Py_CheckSlotResult(
     PyObject *obj,
@@ -185,7 +194,19 @@ extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
                                           PyObject *name, PyObject *value);
 PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
                                         PyObject *name);
-PyDictValues ** _PyObject_ValuesPointer(PyObject *);
+
+static inline PyDictValues **_PyObject_ValuesPointer(PyObject *obj)
+{
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+    return ((PyDictValues **)obj)-4;
+}
+
+static inline PyObject **_PyObject_ManagedDictPointer(PyObject *obj)
+{
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+    return ((PyObject **)obj)-3;
+}
+
 PyObject ** _PyObject_DictPointer(PyObject *);
 int _PyObject_VisitInstanceAttributes(PyObject *self, visitproc visit, void *arg);
 void _PyObject_ClearInstanceAttributes(PyObject *self);
diff --git a/Include/object.h b/Include/object.h
index 33df303a44eb7..e5544e8b588ed 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -334,6 +334,12 @@ given type object has a specified feature.
 
 #ifndef Py_LIMITED_API
 
+/* Placement of dict (and values) pointers are managed by the VM, not by the type.
+ * The VM will automatically set tp_dictoffset. Should not be used for variable sized
+ * classes, such as classes that extend tuple.
+ */
+#define Py_TPFLAGS_MANAGED_DICT (1 << 4)
+
 /* Set if instances of the type object are treated as sequences for pattern matching */
 #define Py_TPFLAGS_SEQUENCE (1 << 5)
 /* Set if instances of the type object are treated as mappings for pattern matching */
diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py
index 1e27bcaf889a2..d0cd5c20dd5f4 100644
--- a/Lib/test/test_stable_abi_ctypes.py
+++ b/Lib/test/test_stable_abi_ctypes.py
@@ -817,7 +817,6 @@ def test_available_symbols(self):
     "_PyErr_BadInternalCall",
     "_PyObject_CallFunction_SizeT",
     "_PyObject_CallMethod_SizeT",
-    "_PyObject_GC_Malloc",
     "_PyObject_GC_New",
     "_PyObject_GC_NewVar",
     "_PyObject_GC_Resize",
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index 81802893a8ee5..41ac03b920e64 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -1421,8 +1421,8 @@ def delx(self): del self.__x
         check((1,2,3), vsize('') + 3*self.P)
         # type
         # static type: PyTypeObject
-        fmt = 'P2nPI13Pl4Pn9Pn12PIPP'
-        s = vsize(fmt)
+        fmt = 'P2nPI13Pl4Pn9Pn12PIP'
+        s = vsize('2P' + fmt)
         check(int, s)
         # class
         s = vsize(fmt +                 # PyTypeObject
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-01-14-06-36.bpo-45947.1XPPm_.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-01-14-06-36.bpo-45947.1XPPm_.rst
new file mode 100644
index 0000000000000..5156bd35369e1
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-12-01-14-06-36.bpo-45947.1XPPm_.rst	
@@ -0,0 +1,3 @@
+Place pointers to dict and values immediately before GC header. This reduces
+number of dependent memory loads to access either dict or values from 3 to
+1.
diff --git a/Misc/stable_abi.txt b/Misc/stable_abi.txt
index 9f5a85bdec40f..de6caa8c80746 100644
--- a/Misc/stable_abi.txt
+++ b/Misc/stable_abi.txt
@@ -1577,9 +1577,6 @@ function _PyObject_CallFunction_SizeT
 function _PyObject_CallMethod_SizeT
     added 3.2
     abi_only
-function _PyObject_GC_Malloc
-    added 3.2
-    abi_only
 function _PyObject_GC_New
     added 3.2
     abi_only
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 0216c985415ce..56d394985eb7c 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -5861,6 +5861,7 @@ test_tstate_capi(PyObject *self, PyObject *Py_UNUSED(args))
 }
 
 
+static PyObject *negative_dictoffset(PyObject *, PyObject *);
 static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *);
 static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*);
 
@@ -5929,14 +5930,15 @@ static PyMethodDef TestMethods[] = {
 #if (defined(__linux__) || defined(__FreeBSD__)) && defined(__GNUC__)
     {"test_pep3118_obsolete_write_locks", (PyCFunction)test_pep3118_obsolete_write_locks, METH_NOARGS},
 #endif
-    {"getbuffer_with_null_view", getbuffer_with_null_view, METH_O},
-    {"PyBuffer_SizeFromFormat",  test_PyBuffer_SizeFromFormat, METH_VARARGS},
-    {"test_buildvalue_N",       test_buildvalue_N,               METH_NOARGS},
+    {"getbuffer_with_null_view", getbuffer_with_null_view,       METH_O},
+    {"PyBuffer_SizeFromFormat",  test_PyBuffer_SizeFromFormat,   METH_VARARGS},
+    {"test_buildvalue_N",        test_buildvalue_N,              METH_NOARGS},
+    {"negative_dictoffset",      negative_dictoffset,            METH_NOARGS},
     {"test_buildvalue_issue38913", test_buildvalue_issue38913,   METH_NOARGS},
-    {"get_args",                get_args,                        METH_VARARGS},
+    {"get_args",                  get_args,                      METH_VARARGS},
     {"test_get_statictype_slots", test_get_statictype_slots,     METH_NOARGS},
     {"test_get_type_name",        test_get_type_name,            METH_NOARGS},
-    {"test_get_type_qualname",   test_get_type_qualname,       METH_NOARGS},
+    {"test_get_type_qualname",    test_get_type_qualname,        METH_NOARGS},
     {"test_type_from_ephemeral_spec", test_type_from_ephemeral_spec, METH_NOARGS},
     {"get_kwargs", (PyCFunction)(void(*)(void))get_kwargs,
       METH_VARARGS|METH_KEYWORDS},
@@ -7629,6 +7631,11 @@ PyInit__testcapi(void)
     return m;
 }
 
+static PyObject *
+negative_dictoffset(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    return PyType_FromSpec(&HeapCTypeWithNegativeDict_spec);
+}
 
 /* Test the C API exposed when PY_SSIZE_T_CLEAN is not defined */
 
diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
index b505676636d38..1808057a650e9 100644
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -69,10 +69,10 @@ module gc
 #define NEXT_MASK_UNREACHABLE  (1)
 
 /* Get an object's GC head */
-#define AS_GC(o) ((PyGC_Head *)(o)-1)
+#define AS_GC(o) ((PyGC_Head *)(((char *)(o))-sizeof(PyGC_Head)))
 
 /* Get the object given the GC head */
-#define FROM_GC(g) ((PyObject *)(((PyGC_Head *)g)+1))
+#define FROM_GC(g) ((PyObject *)(((char *)(g))+sizeof(PyGC_Head)))
 
 static inline int
 gc_is_collecting(PyGC_Head *g)
@@ -2231,28 +2231,14 @@ PyObject_IS_GC(PyObject *obj)
     return _PyObject_IS_GC(obj);
 }
 
-static PyObject *
-_PyObject_GC_Alloc(int use_calloc, size_t basicsize)
+void
+_PyObject_GC_Link(PyObject *op)
 {
+    PyGC_Head *g = AS_GC(op);
+    assert(((uintptr_t)g & (sizeof(uintptr_t)-1)) == 0);  // g must be correctly aligned
+
     PyThreadState *tstate = _PyThreadState_GET();
     GCState *gcstate = &tstate->interp->gc;
-    if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head)) {
-        return _PyErr_NoMemory(tstate);
-    }
-    size_t size = sizeof(PyGC_Head) + basicsize;
-
-    PyGC_Head *g;
-    if (use_calloc) {
-        g = (PyGC_Head *)PyObject_Calloc(1, size);
-    }
-    else {
-        g = (PyGC_Head *)PyObject_Malloc(size);
-    }
-    if (g == NULL) {
-        return _PyErr_NoMemory(tstate);
-    }
-    assert(((uintptr_t)g & 3) == 0);  // g must be aligned 4bytes boundary
-
     g->_gc_next = 0;
     g->_gc_prev = 0;
     gcstate->generations[0].count++; /* number of allocated GC objects */
@@ -2266,26 +2252,32 @@ _PyObject_GC_Alloc(int use_calloc, size_t basicsize)
         gc_collect_generations(tstate);
         gcstate->collecting = 0;
     }
-    PyObject *op = FROM_GC(g);
-    return op;
 }
 
-PyObject *
-_PyObject_GC_Malloc(size_t basicsize)
-{
-    return _PyObject_GC_Alloc(0, basicsize);
-}
-
-PyObject *
-_PyObject_GC_Calloc(size_t basicsize)
+static PyObject *
+gc_alloc(size_t basicsize, size_t presize)
 {
-    return _PyObject_GC_Alloc(1, basicsize);
+    PyThreadState *tstate = _PyThreadState_GET();
+    if (basicsize > PY_SSIZE_T_MAX - presize) {
+        return _PyErr_NoMemory(tstate);
+    }
+    size_t size = presize + basicsize;
+    char *mem = PyObject_Malloc(size);
+    if (mem == NULL) {
+        return _PyErr_NoMemory(tstate);
+    }
+    ((PyObject **)mem)[0] = NULL;
+    ((PyObject **)mem)[1] = NULL;
+    PyObject *op = (PyObject *)(mem + presize);
+    _PyObject_GC_Link(op);
+    return op;
 }
 
 PyObject *
 _PyObject_GC_New(PyTypeObject *tp)
 {
-    PyObject *op = _PyObject_GC_Malloc(_PyObject_SIZE(tp));
+    size_t presize = _PyType_PreHeaderSize(tp);
+    PyObject *op = gc_alloc(_PyObject_SIZE(tp), presize);
     if (op == NULL) {
         return NULL;
     }
@@ -2303,8 +2295,9 @@ _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems)
         PyErr_BadInternalCall();
         return NULL;
     }
+    size_t presize = _PyType_PreHeaderSize(tp);
     size = _PyObject_VAR_SIZE(tp, nitems);
-    op = (PyVarObject *) _PyObject_GC_Malloc(size);
+    op = (PyVarObject *)gc_alloc(size, presize);
     if (op == NULL) {
         return NULL;
     }
@@ -2333,6 +2326,7 @@ _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems)
 void
 PyObject_GC_Del(void *op)
 {
+    size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type);
     PyGC_Head *g = AS_GC(op);
     if (_PyObject_GC_IS_TRACKED(op)) {
         gc_list_remove(g);
@@ -2341,7 +2335,7 @@ PyObject_GC_Del(void *op)
     if (gcstate->generations[0].count > 0) {
         gcstate->generations[0].count--;
     }
-    PyObject_Free(g);
+    PyObject_Free(((char *)op)-presize);
 }
 
 int
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 475d92d329828..7ce4b9069f77e 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -4961,8 +4961,8 @@ static int
 init_inline_values(PyObject *obj, PyTypeObject *tp)
 {
     assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
-    assert(tp->tp_dictoffset > 0);
-    assert(tp->tp_inline_values_offset > 0);
+    // assert(type->tp_dictoffset > 0);  -- TO DO Update this assert.
+    assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
     PyDictKeysObject *keys = CACHED_KEYS(tp);
     assert(keys != NULL);
     if (keys->dk_usable > 1) {
@@ -4979,7 +4979,7 @@ init_inline_values(PyObject *obj, PyTypeObject *tp)
     for (int i = 0; i < size; i++) {
         values->values[i] = NULL;
     }
-    *((PyDictValues **)((char *)obj + tp->tp_inline_values_offset)) = values;
+    *_PyObject_ValuesPointer(obj) = values;
     return 0;
 }
 
@@ -4990,7 +4990,7 @@ _PyObject_InitializeDict(PyObject *obj)
     if (tp->tp_dictoffset == 0) {
         return 0;
     }
-    if (tp->tp_inline_values_offset) {
+    if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
         return init_inline_values(obj, tp);
     }
     PyObject *dict;
@@ -5032,7 +5032,7 @@ make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values)
 PyObject *
 _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values)
 {
-    assert(Py_TYPE(obj)->tp_inline_values_offset != 0);
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
     PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
     return make_dict_from_instance_attributes(keys, values);
 }
@@ -5042,10 +5042,10 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
                               PyObject *name, PyObject *value)
 {
     assert(PyUnicode_CheckExact(name));
-    PyTypeObject *tp = Py_TYPE(obj);
     PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
     assert(keys != NULL);
     assert(values != NULL);
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
     int ix = insert_into_dictkeys(keys, name);
     if (ix == DKIX_EMPTY) {
         if (value == NULL) {
@@ -5056,8 +5056,8 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
         if (dict == NULL) {
             return -1;
         }
-        *((PyDictValues **)((char *)obj + tp->tp_inline_values_offset)) = NULL;
-        *((PyObject **) ((char *)obj + tp->tp_dictoffset)) = dict;
+        *_PyObject_ValuesPointer(obj) = NULL;
+        *_PyObject_ManagedDictPointer(obj) = dict;
         return PyDict_SetItem(dict, name, value);
     }
     PyObject *old_value = values->values[ix];
@@ -5102,17 +5102,23 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj)
     if (tp->tp_dictoffset == 0) {
         return 1;
     }
-    PyDictValues **values_ptr = _PyObject_ValuesPointer(obj);
-    if (values_ptr && *values_ptr) {
-        PyDictKeysObject *keys = CACHED_KEYS(tp);
-        for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
-            if ((*values_ptr)->values[i] != NULL) {
-                return 0;
+    PyObject **dictptr;
+    if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+        PyDictValues *values = *_PyObject_ValuesPointer(obj);
+        if (values) {
+            PyDictKeysObject *keys = CACHED_KEYS(tp);
+            for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
+                if (values->values[i] != NULL) {
+                    return 0;
+                }
             }
+            return 1;
         }
-        return 1;
+        dictptr = _PyObject_ManagedDictPointer(obj);
+    }
+    else {
+       dictptr = _PyObject_DictPointer(obj);
     }
-    PyObject **dictptr = _PyObject_DictPointer(obj);
     PyObject *dict = *dictptr;
     if (dict == NULL) {
         return 1;
@@ -5125,7 +5131,7 @@ int
 _PyObject_VisitInstanceAttributes(PyObject *self, visitproc visit, void *arg)
 {
     PyTypeObject *tp = Py_TYPE(self);
-    assert(tp->tp_inline_values_offset);
+    assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
     PyDictValues **values_ptr = _PyObject_ValuesPointer(self);
     if (*values_ptr == NULL) {
         return 0;
@@ -5141,7 +5147,7 @@ void
 _PyObject_ClearInstanceAttributes(PyObject *self)
 {
     PyTypeObject *tp = Py_TYPE(self);
-    assert(tp->tp_inline_values_offset);
+    assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
     PyDictValues **values_ptr = _PyObject_ValuesPointer(self);
     if (*values_ptr == NULL) {
         return;
@@ -5156,7 +5162,7 @@ void
 _PyObject_FreeInstanceAttributes(PyObject *self)
 {
     PyTypeObject *tp = Py_TYPE(self);
-    assert(tp->tp_inline_values_offset);
+    assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
     PyDictValues **values_ptr = _PyObject_ValuesPointer(self);
     if (*values_ptr == NULL) {
         return;
@@ -5171,28 +5177,42 @@ _PyObject_FreeInstanceAttributes(PyObject *self)
 PyObject *
 PyObject_GenericGetDict(PyObject *obj, void *context)
 {
-    PyObject **dictptr = _PyObject_DictPointer(obj);
-    if (dictptr == NULL) {
-        PyErr_SetString(PyExc_AttributeError,
-                        "This object has no __dict__");
-        return NULL;
-    }
-    PyObject *dict = *dictptr;
-    if (dict == NULL) {
-        PyTypeObject *tp = Py_TYPE(obj);
+    PyObject *dict;
+    PyTypeObject *tp = Py_TYPE(obj);
+    if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) {
         PyDictValues **values_ptr = _PyObject_ValuesPointer(obj);
-        if (values_ptr && *values_ptr) {
+        PyObject **dictptr = _PyObject_ManagedDictPointer(obj);
+        if (*values_ptr) {
+            assert(*dictptr == NULL);
             *dictptr = dict = make_dict_from_instance_attributes(CACHED_KEYS(tp), *values_ptr);
             if (dict != NULL) {
                 *values_ptr = NULL;
             }
         }
-        else if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) {
-            dictkeys_incref(CACHED_KEYS(tp));
-            *dictptr = dict = new_dict_with_shared_keys(CACHED_KEYS(tp));
+        else if (*dictptr == NULL) {
+            *dictptr = dict = PyDict_New();
         }
         else {
-            *dictptr = dict = PyDict_New();
+            dict = *dictptr;
+        }
+    }
+    else {
+        PyObject **dictptr = _PyObject_DictPointer(obj);
+        if (dictptr == NULL) {
+            PyErr_SetString(PyExc_AttributeError,
+                            "This object has no __dict__");
+            return NULL;
+        }
+        dict = *dictptr;
+        if (dict == NULL) {
+            PyTypeObject *tp = Py_TYPE(obj);
+            if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) {
+                dictkeys_incref(CACHED_KEYS(tp));
+                *dictptr = dict = new_dict_with_shared_keys(CACHED_KEYS(tp));
+            }
+            else {
+                *dictptr = dict = PyDict_New();
+            }
         }
     }
     Py_XINCREF(dict);
diff --git a/Objects/exceptions.c b/Objects/exceptions.c
index c99f17a30f169..e1a8c1363ef62 100644
--- a/Objects/exceptions.c
+++ b/Objects/exceptions.c
@@ -3484,7 +3484,6 @@ _PyErr_TrySetFromCause(const char *format, ...)
     PyObject* msg_prefix;
     PyObject *exc, *val, *tb;
     PyTypeObject *caught_type;
-    PyObject **dictptr;
     PyObject *instance_args;
     Py_ssize_t num_args, caught_type_size, base_exc_size;
     PyObject *new_exc, *new_val, *new_tb;
@@ -3530,9 +3529,7 @@ _PyErr_TrySetFromCause(const char *format, ...)
     }
 
     /* Ensure the instance dict is also empty */
-    dictptr = _PyObject_GetDictPtr(val);
-    if (dictptr != NULL && *dictptr != NULL &&
-        PyDict_GET_SIZE(*dictptr) > 0) {
+    if (!_PyObject_IsInstanceDictEmpty(val)) {
         /* While we could potentially copy a non-empty instance dictionary
          * to the replacement exception, for now we take the more
          * conservative path of leaving exceptions with attributes set
diff --git a/Objects/object.c b/Objects/object.c
index 25f5a2133d574..a1c2e16b6fa2c 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -1073,6 +1073,9 @@ _PyObject_DictPointer(PyObject *obj)
     Py_ssize_t dictoffset;
     PyTypeObject *tp = Py_TYPE(obj);
 
+    if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+        return _PyObject_ManagedDictPointer(obj);
+    }
     dictoffset = tp->tp_dictoffset;
     if (dictoffset == 0)
         return NULL;
@@ -1096,24 +1099,20 @@ _PyObject_DictPointer(PyObject *obj)
 PyObject **
 _PyObject_GetDictPtr(PyObject *obj)
 {
-    PyObject **dict_ptr = _PyObject_DictPointer(obj);
-    if (dict_ptr == NULL) {
-        return NULL;
-    }
-    if (*dict_ptr != NULL) {
-        return dict_ptr;
+    if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
+        return _PyObject_DictPointer(obj);
     }
+    PyObject **dict_ptr = _PyObject_ManagedDictPointer(obj);
     PyDictValues **values_ptr = _PyObject_ValuesPointer(obj);
-    if (values_ptr == NULL || *values_ptr == NULL) {
+    if (*values_ptr == NULL) {
         return dict_ptr;
     }
+    assert(*dict_ptr == NULL);
     PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, *values_ptr);
     if (dict == NULL) {
         PyErr_Clear();
         return NULL;
     }
-    assert(*dict_ptr == NULL);
-    assert(*values_ptr != NULL);
     *values_ptr = NULL;
     *dict_ptr = dict;
     return dict_ptr;
@@ -1185,10 +1184,12 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method)
             }
         }
     }
-    PyDictValues **values_ptr = _PyObject_ValuesPointer(obj);
-    if (values_ptr && *values_ptr) {
+    PyDictValues *values;
+    if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) &&
+        (values = *_PyObject_ValuesPointer(obj)))
+    {
         assert(*_PyObject_DictPointer(obj) == NULL);
-        PyObject *attr = _PyObject_GetInstanceAttribute(obj, *values_ptr, name);
+        PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name);
         if (attr != NULL) {
             *method = attr;
             Py_XDECREF(descr);
@@ -1240,17 +1241,6 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method)
     return 0;
 }
 
-PyDictValues **
-_PyObject_ValuesPointer(PyObject *obj)
-{
-    PyTypeObject *tp = Py_TYPE(obj);
-    Py_ssize_t offset = tp->tp_inline_values_offset;
-    if (offset == 0) {
-        return NULL;
-    }
-    return (PyDictValues **) ((char *)obj + offset);
-}
-
 /* Generic GetAttr functions - put these in your tp_[gs]etattro slot. */
 
 PyObject *
@@ -1267,7 +1257,6 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name,
     PyObject *descr = NULL;
     PyObject *res = NULL;
     descrgetfunc f;
-    Py_ssize_t dictoffset;
     PyObject **dictptr;
 
     if (!PyUnicode_Check(name)){
@@ -1299,8 +1288,10 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name,
         }
     }
     if (dict == NULL) {
-        PyDictValues **values_ptr = _PyObject_ValuesPointer(obj);
-        if (values_ptr && *values_ptr) {
+        if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) &&
+            *_PyObject_ValuesPointer(obj))
+        {
+            PyDictValues **values_ptr = _PyObject_ValuesPointer(obj);
             if (PyUnicode_CheckExact(name)) {
                 assert(*_PyObject_DictPointer(obj) == NULL);
                 res = _PyObject_GetInstanceAttribute(obj, *values_ptr, name);
@@ -1320,22 +1311,8 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name,
             }
         }
         else {
-            /* Inline _PyObject_DictPointer */
-            dictoffset = tp->tp_dictoffset;
-            if (dictoffset != 0) {
-                if (dictoffset < 0) {
-                    Py_ssize_t tsize = Py_SIZE(obj);
-                    if (tsize < 0) {
-                        tsize = -tsize;
-                    }
-                    size_t size = _PyObject_VAR_SIZE(tp, tsize);
-                    _PyObject_ASSERT(obj, size <= PY_SSIZE_T_MAX);
-
-                    dictoffset += (Py_ssize_t)size;
-                    _PyObject_ASSERT(obj, dictoffset > 0);
-                    _PyObject_ASSERT(obj, dictoffset % SIZEOF_VOID_P == 0);
-                }
-                dictptr = (PyObject **) ((char *)obj + dictoffset);
+            dictptr = _PyObject_DictPointer(obj);
+            if (dictptr) {
                 dict = *dictptr;
             }
         }
@@ -1426,9 +1403,8 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name,
     }
 
     if (dict == NULL) {
-        PyDictValues **values_ptr = _PyObject_ValuesPointer(obj);
-        if (values_ptr && *values_ptr) {
-            res = _PyObject_StoreInstanceAttribute(obj, *values_ptr, name, value);
+        if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) && *_PyObject_ValuesPointer(obj)) {
+            res = _PyObject_StoreInstanceAttribute(obj, *_PyObject_ValuesPointer(obj), name, value);
         }
         else {
             PyObject **dictptr = _PyObject_DictPointer(obj);
@@ -1478,8 +1454,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context)
 {
     PyObject **dictptr = _PyObject_GetDictPtr(obj);
     if (dictptr == NULL) {
-        PyDictValues** values_ptr = _PyObject_ValuesPointer(obj);
-        if (values_ptr != NULL && *values_ptr != NULL) {
+        if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) &&
+            *_PyObject_ValuesPointer(obj) != NULL)
+        {
             /* Was unable to convert to dict */
             PyErr_NoMemory();
         }
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 22e509be26fff..2fd93b61c0b2b 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1146,17 +1146,17 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)
     const size_t size = _PyObject_VAR_SIZE(type, nitems+1);
     /* note that we need to add one, for the sentinel */
 
-    if (_PyType_IS_GC(type)) {
-        obj = _PyObject_GC_Malloc(size);
-    }
-    else {
-        obj = (PyObject *)PyObject_Malloc(size);
-    }
-
-    if (obj == NULL) {
+    const size_t presize = _PyType_PreHeaderSize(type);
+    char *alloc = PyObject_Malloc(size + presize);
+    if (alloc  == NULL) {
         return PyErr_NoMemory();
     }
-
+    obj = (PyObject *)(alloc + presize);
+    if (presize) {
+        ((PyObject **)alloc)[0] = NULL;
+        ((PyObject **)alloc)[1] = NULL;
+        _PyObject_GC_Link(obj);
+    }
     memset(obj, '\0', size);
 
     if (type->tp_itemsize == 0) {
@@ -1232,7 +1232,7 @@ subtype_traverse(PyObject *self, visitproc visit, void *arg)
         assert(base);
     }
 
-    if (type->tp_inline_values_offset) {
+    if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
         assert(type->tp_dictoffset);
         int err = _PyObject_VisitInstanceAttributes(self, visit, arg);
         if (err) {
@@ -1301,7 +1301,7 @@ subtype_clear(PyObject *self)
 
     /* Clear the instance dict (if any), to break cycles involving only
        __dict__ slots (as in the case 'self.__dict__ is self'). */
-    if (type->tp_inline_values_offset) {
+    if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
         _PyObject_ClearInstanceAttributes(self);
     }
     if (type->tp_dictoffset != base->tp_dictoffset) {
@@ -1360,6 +1360,8 @@ subtype_dealloc(PyObject *self)
         int type_needs_decref = (type->tp_flags & Py_TPFLAGS_HEAPTYPE
                                  && !(base->tp_flags & Py_TPFLAGS_HEAPTYPE));
 
+        assert((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
+
         /* Call the base tp_dealloc() */
         assert(basedealloc);
         basedealloc(self);
@@ -1445,10 +1447,18 @@ subtype_dealloc(PyObject *self)
     }
 
     /* If we added a dict, DECREF it, or free inline values. */
-    if (type->tp_inline_values_offset) {
-        _PyObject_FreeInstanceAttributes(self);
+    if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+        PyObject **dictptr = _PyObject_ManagedDictPointer(self);
+        if (*dictptr != NULL) {
+            assert(*_PyObject_ValuesPointer(self) == NULL);
+            Py_DECREF(*dictptr);
+            *dictptr = NULL;
+        }
+        else {
+            _PyObject_FreeInstanceAttributes(self);
+        }
     }
-    if (type->tp_dictoffset && !base->tp_dictoffset) {
+    else if (type->tp_dictoffset && !base->tp_dictoffset) {
         PyObject **dictptr = _PyObject_DictPointer(self);
         if (dictptr != NULL) {
             PyObject *dict = *dictptr;
@@ -2243,18 +2253,10 @@ extra_ivars(PyTypeObject *type, PyTypeObject *base)
         return t_size != b_size ||
             type->tp_itemsize != base->tp_itemsize;
     }
-    if (type->tp_inline_values_offset && base->tp_inline_values_offset == 0 &&
-        type->tp_inline_values_offset + sizeof(PyDictValues *) == t_size &&
-        type->tp_flags & Py_TPFLAGS_HEAPTYPE)
-        t_size -= sizeof(PyDictValues *);
     if (type->tp_weaklistoffset && base->tp_weaklistoffset == 0 &&
         type->tp_weaklistoffset + sizeof(PyObject *) == t_size &&
         type->tp_flags & Py_TPFLAGS_HEAPTYPE)
         t_size -= sizeof(PyObject *);
-    if (type->tp_dictoffset && base->tp_dictoffset == 0 &&
-        type->tp_dictoffset + sizeof(PyObject *) == t_size &&
-        type->tp_flags & Py_TPFLAGS_HEAPTYPE)
-        t_size -= sizeof(PyObject *);
     return t_size != b_size;
 }
 
@@ -2982,13 +2984,8 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type)
         }
     }
 
-    if (ctx->add_dict) {
-        if (ctx->base->tp_itemsize) {
-            type->tp_dictoffset = -(long)sizeof(PyObject *);
-        }
-        else {
-            type->tp_dictoffset = slotoffset;
-        }
+    if (ctx->add_dict && ctx->base->tp_itemsize) {
+        type->tp_dictoffset = -(long)sizeof(PyObject *);
         slotoffset += sizeof(PyObject *);
     }
 
@@ -2997,12 +2994,10 @@ type_new_descriptors(const type_new_ctx *ctx, PyTypeObject *type)
         type->tp_weaklistoffset = slotoffset;
         slotoffset += sizeof(PyObject *);
     }
-    if (type->tp_dictoffset > 0) {
-        type->tp_inline_values_offset = slotoffset;
-        slotoffset += sizeof(PyDictValues *);
-    }
-    else {
-        type->tp_inline_values_offset = 0;
+    if (ctx->add_dict && ctx->base->tp_itemsize == 0) {
+        assert((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
+        type->tp_flags |= Py_TPFLAGS_MANAGED_DICT;
+        type->tp_dictoffset = -slotoffset - sizeof(PyObject *)*3;
     }
 
     type->tp_basicsize = slotoffset;
@@ -3206,8 +3201,7 @@ type_new_impl(type_new_ctx *ctx)
     // Put the proper slots in place
     fixup_slot_dispatchers(type);
 
-    if (type->tp_inline_values_offset) {
-        assert(type->tp_dictoffset > 0);
+    if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
         PyHeapTypeObject *et = (PyHeapTypeObject*)type;
         et->ht_cached_keys = _PyDict_NewKeysForClass();
     }
@@ -3594,8 +3588,7 @@ PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases)
     if (PyType_Ready(type) < 0)
         goto fail;
 
-    if (type->tp_inline_values_offset) {
-        assert(type->tp_dictoffset > 0);
+    if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
         res->ht_cached_keys = _PyDict_NewKeysForClass();
     }
 
@@ -4658,7 +4651,6 @@ compatible_with_tp_base(PyTypeObject *child)
             child->tp_itemsize == parent->tp_itemsize &&
             child->tp_dictoffset == parent->tp_dictoffset &&
             child->tp_weaklistoffset == parent->tp_weaklistoffset &&
-            child->tp_inline_values_offset == parent->tp_inline_values_offset &&
             ((child->tp_flags & Py_TPFLAGS_HAVE_GC) ==
              (parent->tp_flags & Py_TPFLAGS_HAVE_GC)) &&
             (child->tp_dealloc == subtype_dealloc ||
@@ -4678,8 +4670,6 @@ same_slots_added(PyTypeObject *a, PyTypeObject *b)