[Python-checkins] bpo-46841: Quicken code in-place (GH-31888)
markshannon
webhook-mailer at python.org
Mon Mar 21 07:11:48 EDT 2022
https://github.com/python/cpython/commit/2bde6827ea4f136297b2d882480b981ff26262b6
commit: 2bde6827ea4f136297b2d882480b981ff26262b6
branch: main
author: Brandt Bucher <brandtbucher at microsoft.com>
committer: markshannon <mark at hotpy.org>
date: 2022-03-21T11:11:17Z
summary:
bpo-46841: Quicken code in-place (GH-31888)
* Moves the bytecode to the end of the corresponding PyCodeObject, and quickens it in-place.
* Removes the almost-always-unused co_varnames, co_freevars, and co_cellvars member caches
* _PyOpcode_Deopt is a new mapping from all opcodes to their un-quickened forms.
* _PyOpcode_InlineCacheEntries is renamed to _PyOpcode_Caches
* _Py_IncrementCountAndMaybeQuicken is renamed to _PyCode_Warmup
* _Py_Quicken is renamed to _PyCode_Quicken
* _co_quickened is renamed to _co_code_adaptive (and is now a read-only memoryview).
* Do not emit unused nonzero opargs anymore in the compiler.
files:
A Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst
M Include/cpython/code.h
M Include/internal/pycore_code.h
M Include/opcode.h
M Lib/opcode.py
M Lib/test/test_compile.py
M Objects/clinic/codeobject.c.h
M Objects/codeobject.c
M Objects/frameobject.c
M Objects/genobject.c
M Objects/typeobject.c
M Python/ceval.c
M Python/compile.c
M Python/marshal.c
M Python/opcode_targets.h
M Python/specialize.c
M Tools/scripts/deepfreeze.py
M Tools/scripts/generate_opcode_h.py
diff --git a/Include/cpython/code.h b/Include/cpython/code.h
index ab827c5ae87ff..157678317931e 100644
--- a/Include/cpython/code.h
+++ b/Include/cpython/code.h
@@ -26,91 +26,80 @@ typedef uint16_t _Py_CODEUNIT;
// Use "unsigned char" instead of "uint8_t" here to avoid illegal aliasing:
#define _Py_SET_OPCODE(word, opcode) (((unsigned char *)&(word))[0] = (opcode))
+// To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are
+// defined in this macro:
+#define _PyCode_DEF(SIZE) { \
+ PyObject_VAR_HEAD \
+ \
+ /* Note only the following fields are used in hash and/or comparisons \
+ * \
+ * - co_name \
+ * - co_argcount \
+ * - co_posonlyargcount \
+ * - co_kwonlyargcount \
+ * - co_nlocals \
+ * - co_stacksize \
+ * - co_flags \
+ * - co_firstlineno \
+ * - co_consts \
+ * - co_names \
+ * - co_localsplusnames \
+ * This is done to preserve the name and line number for tracebacks \
+ * and debuggers; otherwise, constant de-duplication would collapse \
+ * identical functions/lambdas defined on different lines. \
+ */ \
+ \
+ /* These fields are set with provided values on new code objects. */ \
+ \
+ /* The hottest fields (in the eval loop) are grouped here at the top. */ \
+ PyObject *co_consts; /* list (constants used) */ \
+ PyObject *co_names; /* list of strings (names used) */ \
+ PyObject *co_exceptiontable; /* Byte string encoding exception handling \
+ table */ \
+ int co_flags; /* CO_..., see below */ \
+ int co_warmup; /* Warmup counter for quickening */ \
+ \
+ /* The rest are not so impactful on performance. */ \
+ int co_argcount; /* #arguments, except *args */ \
+ int co_posonlyargcount; /* #positional only arguments */ \
+ int co_kwonlyargcount; /* #keyword only arguments */ \
+ int co_stacksize; /* #entries needed for evaluation stack */ \
+ int co_firstlineno; /* first source line number */ \
+ \
+ /* redundant values (derived from co_localsplusnames and \
+ co_localspluskinds) */ \
+ int co_nlocalsplus; /* number of local + cell + free variables \
+ */ \
+ int co_nlocals; /* number of local variables */ \
+ int co_nplaincellvars; /* number of non-arg cell variables */ \
+ int co_ncellvars; /* total number of cell variables */ \
+ int co_nfreevars; /* number of free variables */ \
+ \
+ PyObject *co_localsplusnames; /* tuple mapping offsets to names */ \
+ PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte \
+ per variable) */ \
+ PyObject *co_filename; /* unicode (where it was loaded from) */ \
+ PyObject *co_name; /* unicode (name, for reference) */ \
+ PyObject *co_qualname; /* unicode (qualname, for reference) */ \
+ PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) \
+ See Objects/lnotab_notes.txt for details. \
+ */ \
+ PyObject *co_endlinetable; /* bytes object that holds end lineno for \
+ instructions separated across different \
+ lines */ \
+ PyObject *co_columntable; /* bytes object that holds start/end column \
+ offset each instruction */ \
+ \
+ PyObject *co_weakreflist; /* to support weakrefs to code objects */ \
+ /* Scratch space for extra data relating to the code object. \
+ Type is a void* to keep the format private in codeobject.c to force \
+ people to go through the proper APIs. */ \
+ void *co_extra; \
+ char co_code_adaptive[(SIZE)]; \
+}
/* Bytecode object */
-struct PyCodeObject {
- PyObject_HEAD
-
- /* Note only the following fields are used in hash and/or comparisons
- *
- * - co_name
- * - co_argcount
- * - co_posonlyargcount
- * - co_kwonlyargcount
- * - co_nlocals
- * - co_stacksize
- * - co_flags
- * - co_firstlineno
- * - co_code
- * - co_consts
- * - co_names
- * - co_varnames
- * - co_freevars
- * - co_cellvars
- *
- * This is done to preserve the name and line number for tracebacks
- * and debuggers; otherwise, constant de-duplication would collapse
- * identical functions/lambdas defined on different lines.
- */
-
- /* These fields are set with provided values on new code objects. */
-
- // The hottest fields (in the eval loop) are grouped here at the top.
- PyObject *co_consts; /* list (constants used) */
- PyObject *co_names; /* list of strings (names used) */
- _Py_CODEUNIT *co_firstinstr; /* Pointer to first instruction, used for quickening.
- Unlike the other "hot" fields, this one is
- actually derived from co_code. */
- PyObject *co_exceptiontable; /* Byte string encoding exception handling table */
- int co_flags; /* CO_..., see below */
- int co_warmup; /* Warmup counter for quickening */
-
- // The rest are not so impactful on performance.
- int co_argcount; /* #arguments, except *args */
- int co_posonlyargcount; /* #positional only arguments */
- int co_kwonlyargcount; /* #keyword only arguments */
- int co_stacksize; /* #entries needed for evaluation stack */
- int co_firstlineno; /* first source line number */
- PyObject *co_code; /* instruction opcodes */
- PyObject *co_localsplusnames; /* tuple mapping offsets to names */
- PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */
- PyObject *co_filename; /* unicode (where it was loaded from) */
- PyObject *co_name; /* unicode (name, for reference) */
- PyObject *co_qualname; /* unicode (qualname, for reference) */
- PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) See
- Objects/lnotab_notes.txt for details. */
- PyObject *co_endlinetable; /* bytes object that holds end lineno for
- instructions separated across different
- lines */
- PyObject *co_columntable; /* bytes object that holds start/end column
- offset each instruction */
-
- /* These fields are set with computed values on new code objects. */
-
- // redundant values (derived from co_localsplusnames and co_localspluskinds)
- int co_nlocalsplus; /* number of local + cell + free variables */
- int co_nlocals; /* number of local variables */
- int co_nplaincellvars; /* number of non-arg cell variables */
- int co_ncellvars; /* total number of cell variables */
- int co_nfreevars; /* number of free variables */
- // lazily-computed values
- PyObject *co_varnames; /* tuple of strings (local variable names) */
- PyObject *co_cellvars; /* tuple of strings (cell variable names) */
- PyObject *co_freevars; /* tuple of strings (free variable names) */
-
- /* The remaining fields are zeroed out on new code objects. */
-
- PyObject *co_weakreflist; /* to support weakrefs to code objects */
- /* Scratch space for extra data relating to the code object.
- Type is a void* to keep the format private in codeobject.c to force
- people to go through the proper APIs. */
- void *co_extra;
- /* Quickened instructions and cache, or NULL
- This should be treated as opaque by all code except the specializer and
- interpreter. */
- _Py_CODEUNIT *co_quickened;
-
-};
+struct PyCodeObject _PyCode_DEF(1);
/* Masks for co_flags above */
#define CO_OPTIMIZED 0x0001
@@ -151,6 +140,8 @@ PyAPI_DATA(PyTypeObject) PyCode_Type;
#define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type)
#define PyCode_GetNumFree(op) ((op)->co_nfreevars)
+#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive)
+#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT))
/* Public interface */
PyAPI_FUNC(PyCodeObject *) PyCode_New(
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 0d324e9e4c0f5..82dc9e4bdc62f 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -92,30 +92,22 @@ typedef struct {
#define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)
-/* Maximum size of code to quicken, in code units. */
-#define MAX_SIZE_TO_QUICKEN 10000
-
#define QUICKENING_WARMUP_DELAY 8
/* We want to compare to zero for efficiency, so we offset values accordingly */
#define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
-#define QUICKENING_WARMUP_COLDEST 1
-int _Py_Quicken(PyCodeObject *code);
+void _PyCode_Quicken(PyCodeObject *code);
-/* Returns 1 if quickening occurs.
- * -1 if an error occurs
- * 0 otherwise */
-static inline int
-_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code)
+static inline void
+_PyCode_Warmup(PyCodeObject *code)
{
if (code->co_warmup != 0) {
code->co_warmup++;
if (code->co_warmup == 0) {
- return _Py_Quicken(code) ? -1 : 1;
+ _PyCode_Quicken(code);
}
}
- return 0;
}
extern Py_ssize_t _Py_QuickenedCount;
@@ -225,6 +217,7 @@ PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *);
extern PyObject* _PyCode_GetVarnames(PyCodeObject *);
extern PyObject* _PyCode_GetCellvars(PyCodeObject *);
extern PyObject* _PyCode_GetFreevars(PyCodeObject *);
+extern PyObject* _PyCode_GetCode(PyCodeObject *);
/* Return the ending source code line number from a bytecode index. */
extern int _PyCode_Addr2EndLine(PyCodeObject *, int);
diff --git a/Include/opcode.h b/Include/opcode.h
index 7bf0ba70fd7de..dfc7b72e3cdc6 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -7,185 +7,187 @@ extern "C" {
/* Instruction opcodes for compiled code */
-#define CACHE 0
-#define POP_TOP 1
-#define PUSH_NULL 2
-#define NOP 9
-#define UNARY_POSITIVE 10
-#define UNARY_NEGATIVE 11
-#define UNARY_NOT 12
-#define UNARY_INVERT 15
-#define BINARY_SUBSCR 25
-#define GET_LEN 30
-#define MATCH_MAPPING 31
-#define MATCH_SEQUENCE 32
-#define MATCH_KEYS 33
-#define PUSH_EXC_INFO 35
-#define WITH_EXCEPT_START 49
-#define GET_AITER 50
-#define GET_ANEXT 51
-#define BEFORE_ASYNC_WITH 52
-#define BEFORE_WITH 53
-#define END_ASYNC_FOR 54
-#define STORE_SUBSCR 60
-#define DELETE_SUBSCR 61
-#define GET_ITER 68
-#define GET_YIELD_FROM_ITER 69
-#define PRINT_EXPR 70
-#define LOAD_BUILD_CLASS 71
-#define LOAD_ASSERTION_ERROR 74
-#define RETURN_GENERATOR 75
-#define LIST_TO_TUPLE 82
-#define RETURN_VALUE 83
-#define IMPORT_STAR 84
-#define SETUP_ANNOTATIONS 85
-#define YIELD_VALUE 86
-#define ASYNC_GEN_WRAP 87
-#define PREP_RERAISE_STAR 88
-#define POP_EXCEPT 89
-#define HAVE_ARGUMENT 90
-#define STORE_NAME 90
-#define DELETE_NAME 91
-#define UNPACK_SEQUENCE 92
-#define FOR_ITER 93
-#define UNPACK_EX 94
-#define STORE_ATTR 95
-#define DELETE_ATTR 96
-#define STORE_GLOBAL 97
-#define DELETE_GLOBAL 98
-#define SWAP 99
-#define LOAD_CONST 100
-#define LOAD_NAME 101
-#define BUILD_TUPLE 102
-#define BUILD_LIST 103
-#define BUILD_SET 104
-#define BUILD_MAP 105
-#define LOAD_ATTR 106
-#define COMPARE_OP 107
-#define IMPORT_NAME 108
-#define IMPORT_FROM 109
-#define JUMP_FORWARD 110
-#define JUMP_IF_FALSE_OR_POP 111
-#define JUMP_IF_TRUE_OR_POP 112
-#define JUMP_ABSOLUTE 113
-#define POP_JUMP_IF_FALSE 114
-#define POP_JUMP_IF_TRUE 115
-#define LOAD_GLOBAL 116
-#define IS_OP 117
-#define CONTAINS_OP 118
-#define RERAISE 119
-#define COPY 120
-#define JUMP_IF_NOT_EXC_MATCH 121
-#define BINARY_OP 122
-#define SEND 123
-#define LOAD_FAST 124
-#define STORE_FAST 125
-#define DELETE_FAST 126
-#define JUMP_IF_NOT_EG_MATCH 127
-#define POP_JUMP_IF_NOT_NONE 128
-#define POP_JUMP_IF_NONE 129
-#define RAISE_VARARGS 130
-#define GET_AWAITABLE 131
-#define MAKE_FUNCTION 132
-#define BUILD_SLICE 133
-#define JUMP_NO_INTERRUPT 134
-#define MAKE_CELL 135
-#define LOAD_CLOSURE 136
-#define LOAD_DEREF 137
-#define STORE_DEREF 138
-#define DELETE_DEREF 139
-#define CALL_FUNCTION_EX 142
-#define EXTENDED_ARG 144
-#define LIST_APPEND 145
-#define SET_ADD 146
-#define MAP_ADD 147
-#define LOAD_CLASSDEREF 148
-#define COPY_FREE_VARS 149
-#define RESUME 151
-#define MATCH_CLASS 152
-#define FORMAT_VALUE 155
-#define BUILD_CONST_KEY_MAP 156
-#define BUILD_STRING 157
-#define LOAD_METHOD 160
-#define LIST_EXTEND 162
-#define SET_UPDATE 163
-#define DICT_MERGE 164
-#define DICT_UPDATE 165
-#define PRECALL 166
-#define CALL 171
-#define KW_NAMES 172
-#define BINARY_OP_ADAPTIVE 3
-#define BINARY_OP_ADD_INT 4
-#define BINARY_OP_ADD_FLOAT 5
-#define BINARY_OP_ADD_UNICODE 6
-#define BINARY_OP_INPLACE_ADD_UNICODE 7
-#define BINARY_OP_MULTIPLY_INT 8
-#define BINARY_OP_MULTIPLY_FLOAT 13
-#define BINARY_OP_SUBTRACT_INT 14
-#define BINARY_OP_SUBTRACT_FLOAT 16
-#define COMPARE_OP_ADAPTIVE 17
-#define COMPARE_OP_FLOAT_JUMP 18
-#define COMPARE_OP_INT_JUMP 19
-#define COMPARE_OP_STR_JUMP 20
-#define BINARY_SUBSCR_ADAPTIVE 21
-#define BINARY_SUBSCR_GETITEM 22
-#define BINARY_SUBSCR_LIST_INT 23
-#define BINARY_SUBSCR_TUPLE_INT 24
-#define BINARY_SUBSCR_DICT 26
-#define STORE_SUBSCR_ADAPTIVE 27
-#define STORE_SUBSCR_LIST_INT 28
-#define STORE_SUBSCR_DICT 29
-#define CALL_ADAPTIVE 34
-#define CALL_PY_EXACT_ARGS 36
-#define CALL_PY_WITH_DEFAULTS 37
-#define JUMP_ABSOLUTE_QUICK 38
-#define LOAD_ATTR_ADAPTIVE 39
-#define LOAD_ATTR_INSTANCE_VALUE 40
-#define LOAD_ATTR_WITH_HINT 41
-#define LOAD_ATTR_SLOT 42
-#define LOAD_ATTR_MODULE 43
-#define LOAD_GLOBAL_ADAPTIVE 44
-#define LOAD_GLOBAL_MODULE 45
-#define LOAD_GLOBAL_BUILTIN 46
-#define LOAD_METHOD_ADAPTIVE 47
-#define LOAD_METHOD_CLASS 48
-#define LOAD_METHOD_MODULE 55
-#define LOAD_METHOD_NO_DICT 56
-#define LOAD_METHOD_WITH_DICT 57
-#define LOAD_METHOD_WITH_VALUES 58
-#define PRECALL_ADAPTIVE 59
-#define PRECALL_BUILTIN_CLASS 62
-#define PRECALL_NO_KW_BUILTIN_O 63
-#define PRECALL_NO_KW_BUILTIN_FAST 64
-#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 65
-#define PRECALL_NO_KW_LEN 66
-#define PRECALL_NO_KW_ISINSTANCE 67
-#define PRECALL_NO_KW_LIST_APPEND 72
-#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 73
-#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 76
-#define PRECALL_NO_KW_STR_1 77
-#define PRECALL_NO_KW_TUPLE_1 78
-#define PRECALL_NO_KW_TYPE_1 79
-#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 80
-#define PRECALL_BOUND_METHOD 81
-#define PRECALL_PYFUNC 140
-#define RESUME_QUICK 141
-#define STORE_ATTR_ADAPTIVE 143
-#define STORE_ATTR_INSTANCE_VALUE 150
-#define STORE_ATTR_SLOT 153
-#define STORE_ATTR_WITH_HINT 154
-#define UNPACK_SEQUENCE_ADAPTIVE 158
-#define UNPACK_SEQUENCE_LIST 159
-#define UNPACK_SEQUENCE_TUPLE 161
-#define UNPACK_SEQUENCE_TWO_TUPLE 167
-#define LOAD_FAST__LOAD_FAST 168
-#define STORE_FAST__LOAD_FAST 169
-#define LOAD_FAST__LOAD_CONST 170
-#define LOAD_CONST__LOAD_FAST 173
-#define STORE_FAST__STORE_FAST 174
-#define DO_TRACING 255
+#define CACHE 0
+#define POP_TOP 1
+#define PUSH_NULL 2
+#define NOP 9
+#define UNARY_POSITIVE 10
+#define UNARY_NEGATIVE 11
+#define UNARY_NOT 12
+#define UNARY_INVERT 15
+#define BINARY_SUBSCR 25
+#define GET_LEN 30
+#define MATCH_MAPPING 31
+#define MATCH_SEQUENCE 32
+#define MATCH_KEYS 33
+#define PUSH_EXC_INFO 35
+#define WITH_EXCEPT_START 49
+#define GET_AITER 50
+#define GET_ANEXT 51
+#define BEFORE_ASYNC_WITH 52
+#define BEFORE_WITH 53
+#define END_ASYNC_FOR 54
+#define STORE_SUBSCR 60
+#define DELETE_SUBSCR 61
+#define GET_ITER 68
+#define GET_YIELD_FROM_ITER 69
+#define PRINT_EXPR 70
+#define LOAD_BUILD_CLASS 71
+#define LOAD_ASSERTION_ERROR 74
+#define RETURN_GENERATOR 75
+#define LIST_TO_TUPLE 82
+#define RETURN_VALUE 83
+#define IMPORT_STAR 84
+#define SETUP_ANNOTATIONS 85
+#define YIELD_VALUE 86
+#define ASYNC_GEN_WRAP 87
+#define PREP_RERAISE_STAR 88
+#define POP_EXCEPT 89
+#define HAVE_ARGUMENT 90
+#define STORE_NAME 90
+#define DELETE_NAME 91
+#define UNPACK_SEQUENCE 92
+#define FOR_ITER 93
+#define UNPACK_EX 94
+#define STORE_ATTR 95
+#define DELETE_ATTR 96
+#define STORE_GLOBAL 97
+#define DELETE_GLOBAL 98
+#define SWAP 99
+#define LOAD_CONST 100
+#define LOAD_NAME 101
+#define BUILD_TUPLE 102
+#define BUILD_LIST 103
+#define BUILD_SET 104
+#define BUILD_MAP 105
+#define LOAD_ATTR 106
+#define COMPARE_OP 107
+#define IMPORT_NAME 108
+#define IMPORT_FROM 109
+#define JUMP_FORWARD 110
+#define JUMP_IF_FALSE_OR_POP 111
+#define JUMP_IF_TRUE_OR_POP 112
+#define JUMP_ABSOLUTE 113
+#define POP_JUMP_IF_FALSE 114
+#define POP_JUMP_IF_TRUE 115
+#define LOAD_GLOBAL 116
+#define IS_OP 117
+#define CONTAINS_OP 118
+#define RERAISE 119
+#define COPY 120
+#define JUMP_IF_NOT_EXC_MATCH 121
+#define BINARY_OP 122
+#define SEND 123
+#define LOAD_FAST 124
+#define STORE_FAST 125
+#define DELETE_FAST 126
+#define JUMP_IF_NOT_EG_MATCH 127
+#define POP_JUMP_IF_NOT_NONE 128
+#define POP_JUMP_IF_NONE 129
+#define RAISE_VARARGS 130
+#define GET_AWAITABLE 131
+#define MAKE_FUNCTION 132
+#define BUILD_SLICE 133
+#define JUMP_NO_INTERRUPT 134
+#define MAKE_CELL 135
+#define LOAD_CLOSURE 136
+#define LOAD_DEREF 137
+#define STORE_DEREF 138
+#define DELETE_DEREF 139
+#define CALL_FUNCTION_EX 142
+#define EXTENDED_ARG 144
+#define LIST_APPEND 145
+#define SET_ADD 146
+#define MAP_ADD 147
+#define LOAD_CLASSDEREF 148
+#define COPY_FREE_VARS 149
+#define RESUME 151
+#define MATCH_CLASS 152
+#define FORMAT_VALUE 155
+#define BUILD_CONST_KEY_MAP 156
+#define BUILD_STRING 157
+#define LOAD_METHOD 160
+#define LIST_EXTEND 162
+#define SET_UPDATE 163
+#define DICT_MERGE 164
+#define DICT_UPDATE 165
+#define PRECALL 166
+#define CALL 171
+#define KW_NAMES 172
+#define BINARY_OP_ADAPTIVE 3
+#define BINARY_OP_ADD_FLOAT 4
+#define BINARY_OP_ADD_INT 5
+#define BINARY_OP_ADD_UNICODE 6
+#define BINARY_OP_INPLACE_ADD_UNICODE 7
+#define BINARY_OP_MULTIPLY_FLOAT 8
+#define BINARY_OP_MULTIPLY_INT 13
+#define BINARY_OP_SUBTRACT_FLOAT 14
+#define BINARY_OP_SUBTRACT_INT 16
+#define BINARY_SUBSCR_ADAPTIVE 17
+#define BINARY_SUBSCR_DICT 18
+#define BINARY_SUBSCR_GETITEM 19
+#define BINARY_SUBSCR_LIST_INT 20
+#define BINARY_SUBSCR_TUPLE_INT 21
+#define CALL_ADAPTIVE 22
+#define CALL_PY_EXACT_ARGS 23
+#define CALL_PY_WITH_DEFAULTS 24
+#define COMPARE_OP_ADAPTIVE 26
+#define COMPARE_OP_FLOAT_JUMP 27
+#define COMPARE_OP_INT_JUMP 28
+#define COMPARE_OP_STR_JUMP 29
+#define JUMP_ABSOLUTE_QUICK 34
+#define LOAD_ATTR_ADAPTIVE 36
+#define LOAD_ATTR_INSTANCE_VALUE 37
+#define LOAD_ATTR_MODULE 38
+#define LOAD_ATTR_SLOT 39
+#define LOAD_ATTR_WITH_HINT 40
+#define LOAD_CONST__LOAD_FAST 41
+#define LOAD_FAST__LOAD_CONST 42
+#define LOAD_FAST__LOAD_FAST 43
+#define LOAD_GLOBAL_ADAPTIVE 44
+#define LOAD_GLOBAL_BUILTIN 45
+#define LOAD_GLOBAL_MODULE 46
+#define LOAD_METHOD_ADAPTIVE 47
+#define LOAD_METHOD_CLASS 48
+#define LOAD_METHOD_MODULE 55
+#define LOAD_METHOD_NO_DICT 56
+#define LOAD_METHOD_WITH_DICT 57
+#define LOAD_METHOD_WITH_VALUES 58
+#define PRECALL_ADAPTIVE 59
+#define PRECALL_BOUND_METHOD 62
+#define PRECALL_BUILTIN_CLASS 63
+#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 64
+#define PRECALL_NO_KW_BUILTIN_FAST 65
+#define PRECALL_NO_KW_BUILTIN_O 66
+#define PRECALL_NO_KW_ISINSTANCE 67
+#define PRECALL_NO_KW_LEN 72
+#define PRECALL_NO_KW_LIST_APPEND 73
+#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 76
+#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 77
+#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 78
+#define PRECALL_NO_KW_STR_1 79
+#define PRECALL_NO_KW_TUPLE_1 80
+#define PRECALL_NO_KW_TYPE_1 81
+#define PRECALL_PYFUNC 140
+#define RESUME_QUICK 141
+#define STORE_ATTR_ADAPTIVE 143
+#define STORE_ATTR_INSTANCE_VALUE 150
+#define STORE_ATTR_SLOT 153
+#define STORE_ATTR_WITH_HINT 154
+#define STORE_FAST__LOAD_FAST 158
+#define STORE_FAST__STORE_FAST 159
+#define STORE_SUBSCR_ADAPTIVE 161
+#define STORE_SUBSCR_DICT 167
+#define STORE_SUBSCR_LIST_INT 168
+#define UNPACK_SEQUENCE_ADAPTIVE 169
+#define UNPACK_SEQUENCE_LIST 170
+#define UNPACK_SEQUENCE_TUPLE 173
+#define UNPACK_SEQUENCE_TWO_TUPLE 174
+#define DO_TRACING 255
-extern const uint8_t _PyOpcode_InlineCacheEntries[256];
+extern const uint8_t _PyOpcode_Caches[256];
+
+extern const uint8_t _PyOpcode_Deopt[256];
#ifdef NEED_OPCODE_TABLES
static const uint32_t _PyOpcode_RelativeJump[8] = {
@@ -209,7 +211,7 @@ static const uint32_t _PyOpcode_Jump[8] = {
0U,
};
-const uint8_t _PyOpcode_InlineCacheEntries[256] = {
+const uint8_t _PyOpcode_Caches[256] = {
[BINARY_SUBSCR] = 4,
[STORE_SUBSCR] = 1,
[UNPACK_SEQUENCE] = 1,
@@ -222,6 +224,184 @@ const uint8_t _PyOpcode_InlineCacheEntries[256] = {
[PRECALL] = 1,
[CALL] = 4,
};
+
+const uint8_t _PyOpcode_Deopt[256] = {
+ [ASYNC_GEN_WRAP] = ASYNC_GEN_WRAP,
+ [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH,
+ [BEFORE_WITH] = BEFORE_WITH,
+ [BINARY_OP] = BINARY_OP,
+ [BINARY_OP_ADAPTIVE] = BINARY_OP,
+ [BINARY_OP_ADD_FLOAT] = BINARY_OP,
+ [BINARY_OP_ADD_INT] = BINARY_OP,
+ [BINARY_OP_ADD_UNICODE] = BINARY_OP,
+ [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP,
+ [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP,
+ [BINARY_OP_MULTIPLY_INT] = BINARY_OP,
+ [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP,
+ [BINARY_OP_SUBTRACT_INT] = BINARY_OP,
+ [BINARY_SUBSCR] = BINARY_SUBSCR,
+ [BINARY_SUBSCR_ADAPTIVE] = BINARY_SUBSCR,
+ [BINARY_SUBSCR_DICT] = BINARY_SUBSCR,
+ [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR,
+ [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR,
+ [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR,
+ [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP,
+ [BUILD_LIST] = BUILD_LIST,
+ [BUILD_MAP] = BUILD_MAP,
+ [BUILD_SET] = BUILD_SET,
+ [BUILD_SLICE] = BUILD_SLICE,
+ [BUILD_STRING] = BUILD_STRING,
+ [BUILD_TUPLE] = BUILD_TUPLE,
+ [CACHE] = CACHE,
+ [CALL] = CALL,
+ [CALL_ADAPTIVE] = CALL,
+ [CALL_FUNCTION_EX] = CALL_FUNCTION_EX,
+ [CALL_PY_EXACT_ARGS] = CALL,
+ [CALL_PY_WITH_DEFAULTS] = CALL,
+ [COMPARE_OP] = COMPARE_OP,
+ [COMPARE_OP_ADAPTIVE] = COMPARE_OP,
+ [COMPARE_OP_FLOAT_JUMP] = COMPARE_OP,
+ [COMPARE_OP_INT_JUMP] = COMPARE_OP,
+ [COMPARE_OP_STR_JUMP] = COMPARE_OP,
+ [CONTAINS_OP] = CONTAINS_OP,
+ [COPY] = COPY,
+ [COPY_FREE_VARS] = COPY_FREE_VARS,
+ [DELETE_ATTR] = DELETE_ATTR,
+ [DELETE_DEREF] = DELETE_DEREF,
+ [DELETE_FAST] = DELETE_FAST,
+ [DELETE_GLOBAL] = DELETE_GLOBAL,
+ [DELETE_NAME] = DELETE_NAME,
+ [DELETE_SUBSCR] = DELETE_SUBSCR,
+ [DICT_MERGE] = DICT_MERGE,
+ [DICT_UPDATE] = DICT_UPDATE,
+ [END_ASYNC_FOR] = END_ASYNC_FOR,
+ [EXTENDED_ARG] = EXTENDED_ARG,
+ [FORMAT_VALUE] = FORMAT_VALUE,
+ [FOR_ITER] = FOR_ITER,
+ [GET_AITER] = GET_AITER,
+ [GET_ANEXT] = GET_ANEXT,
+ [GET_AWAITABLE] = GET_AWAITABLE,
+ [GET_ITER] = GET_ITER,
+ [GET_LEN] = GET_LEN,
+ [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER,
+ [IMPORT_FROM] = IMPORT_FROM,
+ [IMPORT_NAME] = IMPORT_NAME,
+ [IMPORT_STAR] = IMPORT_STAR,
+ [IS_OP] = IS_OP,
+ [JUMP_ABSOLUTE] = JUMP_ABSOLUTE,
+ [JUMP_ABSOLUTE_QUICK] = JUMP_ABSOLUTE,
+ [JUMP_FORWARD] = JUMP_FORWARD,
+ [JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP,
+ [JUMP_IF_NOT_EG_MATCH] = JUMP_IF_NOT_EG_MATCH,
+ [JUMP_IF_NOT_EXC_MATCH] = JUMP_IF_NOT_EXC_MATCH,
+ [JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP,
+ [JUMP_NO_INTERRUPT] = JUMP_NO_INTERRUPT,
+ [KW_NAMES] = KW_NAMES,
+ [LIST_APPEND] = LIST_APPEND,
+ [LIST_EXTEND] = LIST_EXTEND,
+ [LIST_TO_TUPLE] = LIST_TO_TUPLE,
+ [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR,
+ [LOAD_ATTR] = LOAD_ATTR,
+ [LOAD_ATTR_ADAPTIVE] = LOAD_ATTR,
+ [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR,
+ [LOAD_ATTR_MODULE] = LOAD_ATTR,
+ [LOAD_ATTR_SLOT] = LOAD_ATTR,
+ [LOAD_ATTR_WITH_HINT] = LOAD_ATTR,
+ [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS,
+ [LOAD_CLASSDEREF] = LOAD_CLASSDEREF,
+ [LOAD_CLOSURE] = LOAD_CLOSURE,
+ [LOAD_CONST] = LOAD_CONST,
+ [LOAD_CONST__LOAD_FAST] = LOAD_CONST,
+ [LOAD_DEREF] = LOAD_DEREF,
+ [LOAD_FAST] = LOAD_FAST,
+ [LOAD_FAST__LOAD_CONST] = LOAD_FAST,
+ [LOAD_FAST__LOAD_FAST] = LOAD_FAST,
+ [LOAD_GLOBAL] = LOAD_GLOBAL,
+ [LOAD_GLOBAL_ADAPTIVE] = LOAD_GLOBAL,
+ [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL,
+ [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL,
+ [LOAD_METHOD] = LOAD_METHOD,
+ [LOAD_METHOD_ADAPTIVE] = LOAD_METHOD,
+ [LOAD_METHOD_CLASS] = LOAD_METHOD,
+ [LOAD_METHOD_MODULE] = LOAD_METHOD,
+ [LOAD_METHOD_NO_DICT] = LOAD_METHOD,
+ [LOAD_METHOD_WITH_DICT] = LOAD_METHOD,
+ [LOAD_METHOD_WITH_VALUES] = LOAD_METHOD,
+ [LOAD_NAME] = LOAD_NAME,
+ [MAKE_CELL] = MAKE_CELL,
+ [MAKE_FUNCTION] = MAKE_FUNCTION,
+ [MAP_ADD] = MAP_ADD,
+ [MATCH_CLASS] = MATCH_CLASS,
+ [MATCH_KEYS] = MATCH_KEYS,
+ [MATCH_MAPPING] = MATCH_MAPPING,
+ [MATCH_SEQUENCE] = MATCH_SEQUENCE,
+ [NOP] = NOP,
+ [POP_EXCEPT] = POP_EXCEPT,
+ [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE,
+ [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE,
+ [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE,
+ [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE,
+ [POP_TOP] = POP_TOP,
+ [PRECALL] = PRECALL,
+ [PRECALL_ADAPTIVE] = PRECALL,
+ [PRECALL_BOUND_METHOD] = PRECALL,
+ [PRECALL_BUILTIN_CLASS] = PRECALL,
+ [PRECALL_BUILTIN_FAST_WITH_KEYWORDS] = PRECALL,
+ [PRECALL_NO_KW_BUILTIN_FAST] = PRECALL,
+ [PRECALL_NO_KW_BUILTIN_O] = PRECALL,
+ [PRECALL_NO_KW_ISINSTANCE] = PRECALL,
+ [PRECALL_NO_KW_LEN] = PRECALL,
+ [PRECALL_NO_KW_LIST_APPEND] = PRECALL,
+ [PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST] = PRECALL,
+ [PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = PRECALL,
+ [PRECALL_NO_KW_METHOD_DESCRIPTOR_O] = PRECALL,
+ [PRECALL_NO_KW_STR_1] = PRECALL,
+ [PRECALL_NO_KW_TUPLE_1] = PRECALL,
+ [PRECALL_NO_KW_TYPE_1] = PRECALL,
+ [PRECALL_PYFUNC] = PRECALL,
+ [PREP_RERAISE_STAR] = PREP_RERAISE_STAR,
+ [PRINT_EXPR] = PRINT_EXPR,
+ [PUSH_EXC_INFO] = PUSH_EXC_INFO,
+ [PUSH_NULL] = PUSH_NULL,
+ [RAISE_VARARGS] = RAISE_VARARGS,
+ [RERAISE] = RERAISE,
+ [RESUME] = RESUME,
+ [RESUME_QUICK] = RESUME,
+ [RETURN_GENERATOR] = RETURN_GENERATOR,
+ [RETURN_VALUE] = RETURN_VALUE,
+ [SEND] = SEND,
+ [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS,
+ [SET_ADD] = SET_ADD,
+ [SET_UPDATE] = SET_UPDATE,
+ [STORE_ATTR] = STORE_ATTR,
+ [STORE_ATTR_ADAPTIVE] = STORE_ATTR,
+ [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR,
+ [STORE_ATTR_SLOT] = STORE_ATTR,
+ [STORE_ATTR_WITH_HINT] = STORE_ATTR,
+ [STORE_DEREF] = STORE_DEREF,
+ [STORE_FAST] = STORE_FAST,
+ [STORE_FAST__LOAD_FAST] = STORE_FAST,
+ [STORE_FAST__STORE_FAST] = STORE_FAST,
+ [STORE_GLOBAL] = STORE_GLOBAL,
+ [STORE_NAME] = STORE_NAME,
+ [STORE_SUBSCR] = STORE_SUBSCR,
+ [STORE_SUBSCR_ADAPTIVE] = STORE_SUBSCR,
+ [STORE_SUBSCR_DICT] = STORE_SUBSCR,
+ [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR,
+ [SWAP] = SWAP,
+ [UNARY_INVERT] = UNARY_INVERT,
+ [UNARY_NEGATIVE] = UNARY_NEGATIVE,
+ [UNARY_NOT] = UNARY_NOT,
+ [UNARY_POSITIVE] = UNARY_POSITIVE,
+ [UNPACK_EX] = UNPACK_EX,
+ [UNPACK_SEQUENCE] = UNPACK_SEQUENCE,
+ [UNPACK_SEQUENCE_ADAPTIVE] = UNPACK_SEQUENCE,
+ [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE,
+ [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE,
+ [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE,
+ [WITH_EXCEPT_START] = WITH_EXCEPT_START,
+ [YIELD_VALUE] = YIELD_VALUE,
+};
#endif /* OPCODE_TABLES */
#define HAS_CONST(op) (false\
@@ -229,32 +409,32 @@ const uint8_t _PyOpcode_InlineCacheEntries[256] = {
|| ((op) == 172) \
)
-#define NB_ADD 0
-#define NB_AND 1
-#define NB_FLOOR_DIVIDE 2
-#define NB_LSHIFT 3
-#define NB_MATRIX_MULTIPLY 4
-#define NB_MULTIPLY 5
-#define NB_REMAINDER 6
-#define NB_OR 7
-#define NB_POWER 8
-#define NB_RSHIFT 9
-#define NB_SUBTRACT 10
-#define NB_TRUE_DIVIDE 11
-#define NB_XOR 12
-#define NB_INPLACE_ADD 13
-#define NB_INPLACE_AND 14
-#define NB_INPLACE_FLOOR_DIVIDE 15
-#define NB_INPLACE_LSHIFT 16
-#define NB_INPLACE_MATRIX_MULTIPLY 17
-#define NB_INPLACE_MULTIPLY 18
-#define NB_INPLACE_REMAINDER 19
-#define NB_INPLACE_OR 20
-#define NB_INPLACE_POWER 21
-#define NB_INPLACE_RSHIFT 22
-#define NB_INPLACE_SUBTRACT 23
-#define NB_INPLACE_TRUE_DIVIDE 24
-#define NB_INPLACE_XOR 25
+#define NB_ADD 0
+#define NB_AND 1
+#define NB_FLOOR_DIVIDE 2
+#define NB_LSHIFT 3
+#define NB_MATRIX_MULTIPLY 4
+#define NB_MULTIPLY 5
+#define NB_REMAINDER 6
+#define NB_OR 7
+#define NB_POWER 8
+#define NB_RSHIFT 9
+#define NB_SUBTRACT 10
+#define NB_TRUE_DIVIDE 11
+#define NB_XOR 12
+#define NB_INPLACE_ADD 13
+#define NB_INPLACE_AND 14
+#define NB_INPLACE_FLOOR_DIVIDE 15
+#define NB_INPLACE_LSHIFT 16
+#define NB_INPLACE_MATRIX_MULTIPLY 17
+#define NB_INPLACE_MULTIPLY 18
+#define NB_INPLACE_REMAINDER 19
+#define NB_INPLACE_OR 20
+#define NB_INPLACE_POWER 21
+#define NB_INPLACE_RSHIFT 22
+#define NB_INPLACE_SUBTRACT 23
+#define NB_INPLACE_TRUE_DIVIDE 24
+#define NB_INPLACE_XOR 25
#define HAS_ARG(op) ((op) >= HAVE_ARGUMENT)
diff --git a/Lib/opcode.py b/Lib/opcode.py
index eb9dd35fabf8d..7a52c13579af7 100644
--- a/Lib/opcode.py
+++ b/Lib/opcode.py
@@ -229,77 +229,111 @@ def jabs_op(name, op, entries=0):
("NB_INPLACE_XOR", "^="),
]
+_specializations = {
+ "BINARY_OP": [
+ "BINARY_OP_ADAPTIVE",
+ "BINARY_OP_ADD_FLOAT",
+ "BINARY_OP_ADD_INT",
+ "BINARY_OP_ADD_UNICODE",
+ "BINARY_OP_INPLACE_ADD_UNICODE",
+ "BINARY_OP_MULTIPLY_FLOAT",
+ "BINARY_OP_MULTIPLY_INT",
+ "BINARY_OP_SUBTRACT_FLOAT",
+ "BINARY_OP_SUBTRACT_INT",
+ ],
+ "BINARY_SUBSCR": [
+ "BINARY_SUBSCR_ADAPTIVE",
+ "BINARY_SUBSCR_DICT",
+ "BINARY_SUBSCR_GETITEM",
+ "BINARY_SUBSCR_LIST_INT",
+ "BINARY_SUBSCR_TUPLE_INT",
+ ],
+ "CALL": [
+ "CALL_ADAPTIVE",
+ "CALL_PY_EXACT_ARGS",
+ "CALL_PY_WITH_DEFAULTS",
+ ],
+ "COMPARE_OP": [
+ "COMPARE_OP_ADAPTIVE",
+ "COMPARE_OP_FLOAT_JUMP",
+ "COMPARE_OP_INT_JUMP",
+ "COMPARE_OP_STR_JUMP",
+ ],
+ "JUMP_ABSOLUTE": [
+ "JUMP_ABSOLUTE_QUICK",
+ ],
+ "LOAD_ATTR": [
+ "LOAD_ATTR_ADAPTIVE",
+ "LOAD_ATTR_INSTANCE_VALUE",
+ "LOAD_ATTR_MODULE",
+ "LOAD_ATTR_SLOT",
+ "LOAD_ATTR_WITH_HINT",
+ ],
+ "LOAD_CONST": [
+ "LOAD_CONST__LOAD_FAST",
+ ],
+ "LOAD_FAST": [
+ "LOAD_FAST__LOAD_CONST",
+ "LOAD_FAST__LOAD_FAST",
+ ],
+ "LOAD_GLOBAL": [
+ "LOAD_GLOBAL_ADAPTIVE",
+ "LOAD_GLOBAL_BUILTIN",
+ "LOAD_GLOBAL_MODULE",
+ ],
+ "LOAD_METHOD": [
+ "LOAD_METHOD_ADAPTIVE",
+ "LOAD_METHOD_CLASS",
+ "LOAD_METHOD_MODULE",
+ "LOAD_METHOD_NO_DICT",
+ "LOAD_METHOD_WITH_DICT",
+ "LOAD_METHOD_WITH_VALUES",
+ ],
+ "PRECALL": [
+ "PRECALL_ADAPTIVE",
+ "PRECALL_BOUND_METHOD",
+ "PRECALL_BUILTIN_CLASS",
+ "PRECALL_BUILTIN_FAST_WITH_KEYWORDS",
+ "PRECALL_NO_KW_BUILTIN_FAST",
+ "PRECALL_NO_KW_BUILTIN_O",
+ "PRECALL_NO_KW_ISINSTANCE",
+ "PRECALL_NO_KW_LEN",
+ "PRECALL_NO_KW_LIST_APPEND",
+ "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST",
+ "PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS",
+ "PRECALL_NO_KW_METHOD_DESCRIPTOR_O",
+ "PRECALL_NO_KW_STR_1",
+ "PRECALL_NO_KW_TUPLE_1",
+ "PRECALL_NO_KW_TYPE_1",
+ "PRECALL_PYFUNC",
+ ],
+ "RESUME": [
+ "RESUME_QUICK",
+ ],
+ "STORE_ATTR": [
+ "STORE_ATTR_ADAPTIVE",
+ "STORE_ATTR_INSTANCE_VALUE",
+ "STORE_ATTR_SLOT",
+ "STORE_ATTR_WITH_HINT",
+ ],
+ "STORE_FAST": [
+ "STORE_FAST__LOAD_FAST",
+ "STORE_FAST__STORE_FAST",
+ ],
+ "STORE_SUBSCR": [
+ "STORE_SUBSCR_ADAPTIVE",
+ "STORE_SUBSCR_DICT",
+ "STORE_SUBSCR_LIST_INT",
+ ],
+ "UNPACK_SEQUENCE": [
+ "UNPACK_SEQUENCE_ADAPTIVE",
+ "UNPACK_SEQUENCE_LIST",
+ "UNPACK_SEQUENCE_TUPLE",
+ "UNPACK_SEQUENCE_TWO_TUPLE",
+ ],
+}
_specialized_instructions = [
- "BINARY_OP_ADAPTIVE",
- "BINARY_OP_ADD_INT",
- "BINARY_OP_ADD_FLOAT",
- "BINARY_OP_ADD_UNICODE",
- "BINARY_OP_INPLACE_ADD_UNICODE",
- "BINARY_OP_MULTIPLY_INT",
- "BINARY_OP_MULTIPLY_FLOAT",
- "BINARY_OP_SUBTRACT_INT",
- "BINARY_OP_SUBTRACT_FLOAT",
- "COMPARE_OP_ADAPTIVE",
- "COMPARE_OP_FLOAT_JUMP",
- "COMPARE_OP_INT_JUMP",
- "COMPARE_OP_STR_JUMP",
- "BINARY_SUBSCR_ADAPTIVE",
- "BINARY_SUBSCR_GETITEM",
- "BINARY_SUBSCR_LIST_INT",
- "BINARY_SUBSCR_TUPLE_INT",
- "BINARY_SUBSCR_DICT",
- "STORE_SUBSCR_ADAPTIVE",
- "STORE_SUBSCR_LIST_INT",
- "STORE_SUBSCR_DICT",
- "CALL_ADAPTIVE",
- "CALL_PY_EXACT_ARGS",
- "CALL_PY_WITH_DEFAULTS",
- "JUMP_ABSOLUTE_QUICK",
- "LOAD_ATTR_ADAPTIVE",
- "LOAD_ATTR_INSTANCE_VALUE",
- "LOAD_ATTR_WITH_HINT",
- "LOAD_ATTR_SLOT",
- "LOAD_ATTR_MODULE",
- "LOAD_GLOBAL_ADAPTIVE",
- "LOAD_GLOBAL_MODULE",
- "LOAD_GLOBAL_BUILTIN",
- "LOAD_METHOD_ADAPTIVE",
- "LOAD_METHOD_CLASS",
- "LOAD_METHOD_MODULE",
- "LOAD_METHOD_NO_DICT",
- "LOAD_METHOD_WITH_DICT",
- "LOAD_METHOD_WITH_VALUES",
- "PRECALL_ADAPTIVE",
- "PRECALL_BUILTIN_CLASS",
- "PRECALL_NO_KW_BUILTIN_O",
- "PRECALL_NO_KW_BUILTIN_FAST",
- "PRECALL_BUILTIN_FAST_WITH_KEYWORDS",
- "PRECALL_NO_KW_LEN",
- "PRECALL_NO_KW_ISINSTANCE",
- "PRECALL_NO_KW_LIST_APPEND",
- "PRECALL_NO_KW_METHOD_DESCRIPTOR_O",
- "PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS",
- "PRECALL_NO_KW_STR_1",
- "PRECALL_NO_KW_TUPLE_1",
- "PRECALL_NO_KW_TYPE_1",
- "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST",
- "PRECALL_BOUND_METHOD",
- "PRECALL_PYFUNC",
- "RESUME_QUICK",
- "STORE_ATTR_ADAPTIVE",
- "STORE_ATTR_INSTANCE_VALUE",
- "STORE_ATTR_SLOT",
- "STORE_ATTR_WITH_HINT",
- "UNPACK_SEQUENCE_ADAPTIVE",
- "UNPACK_SEQUENCE_LIST",
- "UNPACK_SEQUENCE_TUPLE",
- "UNPACK_SEQUENCE_TWO_TUPLE",
- # Super instructions
- "LOAD_FAST__LOAD_FAST",
- "STORE_FAST__LOAD_FAST",
- "LOAD_FAST__LOAD_CONST",
- "LOAD_CONST__LOAD_FAST",
- "STORE_FAST__STORE_FAST",
+ opcode for family in _specializations.values() for opcode in family
]
_specialization_stats = [
"success",
diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index 8499d2855bdd4..a4e80805d3e5c 100644
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -641,7 +641,7 @@ def check_same_constant(const):
self.check_constant(f1, frozenset({0}))
self.assertTrue(f1(0))
- # Merging equal co_linetable and co_code is not a strict requirement
+ # Merging equal co_linetable is not a strict requirement
# for the Python semantics, it's a more an implementation detail.
@support.cpython_only
def test_merge_code_attrs(self):
@@ -650,7 +650,6 @@ def test_merge_code_attrs(self):
f2 = lambda a: a.b.c
self.assertIs(f1.__code__.co_linetable, f2.__code__.co_linetable)
- self.assertIs(f1.__code__.co_code, f2.__code__.co_code)
# Stripping unused constants is not a strict requirement for the
# Python semantics, it's a more an implementation detail.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst b/Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst
new file mode 100644
index 0000000000000..99fad382d13bb
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst
@@ -0,0 +1,2 @@
+Quicken bytecode in-place by storing it as part of the corresponding
+``PyCodeObject``.
diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h
index ee425f61bb113..272bcd6ea17b2 100644
--- a/Objects/clinic/codeobject.c.h
+++ b/Objects/clinic/codeobject.c.h
@@ -203,12 +203,12 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje
int co_stacksize = self->co_stacksize;
int co_flags = self->co_flags;
int co_firstlineno = self->co_firstlineno;
- PyBytesObject *co_code = (PyBytesObject *)self->co_code;
+ PyBytesObject *co_code = NULL;
PyObject *co_consts = self->co_consts;
PyObject *co_names = self->co_names;
- PyObject *co_varnames = self->co_varnames;
- PyObject *co_freevars = self->co_freevars;
- PyObject *co_cellvars = self->co_cellvars;
+ PyObject *co_varnames = NULL;
+ PyObject *co_freevars = NULL;
+ PyObject *co_cellvars = NULL;
PyObject *co_filename = self->co_filename;
PyObject *co_name = self->co_name;
PyObject *co_qualname = self->co_qualname;
@@ -456,4 +456,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n
exit:
return return_value;
}
-/*[clinic end generated code: output=9e8c4a19474ec520 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=b1b83a70ffc5b7cd input=a9049054013a1b77]*/
diff --git a/Objects/codeobject.c b/Objects/codeobject.c
index 5279f6ce17064..224493edb19ea 100644
--- a/Objects/codeobject.c
+++ b/Objects/codeobject.c
@@ -305,9 +305,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
co->co_qualname = con->qualname;
co->co_flags = con->flags;
- Py_INCREF(con->code);
- co->co_code = con->code;
- co->co_firstinstr = (_Py_CODEUNIT *)PyBytes_AS_STRING(con->code);
co->co_firstlineno = con->firstlineno;
Py_INCREF(con->linetable);
co->co_linetable = con->linetable;
@@ -341,16 +338,14 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
co->co_nplaincellvars = nplaincellvars;
co->co_ncellvars = ncellvars;
co->co_nfreevars = nfreevars;
- co->co_varnames = NULL;
- co->co_cellvars = NULL;
- co->co_freevars = NULL;
/* not set */
co->co_weakreflist = NULL;
co->co_extra = NULL;
co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
- co->co_quickened = NULL;
+ memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code),
+ PyBytes_GET_SIZE(con->code));
}
/* The caller is responsible for ensuring that the given data is valid. */
@@ -386,7 +381,8 @@ _PyCode_New(struct _PyCodeConstructor *con)
con->columntable = Py_None;
}
- PyCodeObject *co = PyObject_New(PyCodeObject, &PyCode_Type);
+ Py_ssize_t size = PyBytes_GET_SIZE(con->code) / sizeof(_Py_CODEUNIT);
+ PyCodeObject *co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size);
if (co == NULL) {
PyErr_NoMemory();
return NULL;
@@ -521,13 +517,6 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
goto error;
}
- Py_INCREF(varnames);
- co->co_varnames = varnames;
- Py_INCREF(cellvars);
- co->co_cellvars = cellvars;
- Py_INCREF(freevars);
- co->co_freevars = freevars;
-
error:
Py_XDECREF(localsplusnames);
Py_XDECREF(localspluskinds);
@@ -611,7 +600,7 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq)
if (addrq < 0) {
return co->co_firstlineno;
}
- assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code));
+ assert(addrq >= 0 && addrq < _PyCode_NBYTES(co));
PyCodeAddressRange bounds;
_PyCode_InitAddressRange(co, &bounds);
return _PyCode_CheckLineNumber(addrq, &bounds);
@@ -639,7 +628,7 @@ _PyCode_Addr2EndLine(PyCodeObject* co, int addrq)
return -1;
}
- assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code));
+ assert(addrq >= 0 && addrq < _PyCode_NBYTES(co));
PyCodeAddressRange bounds;
_PyCode_InitEndAddressRange(co, &bounds);
return _PyCode_CheckLineNumber(addrq, &bounds);
@@ -995,7 +984,7 @@ _source_offset_converter(int* value) {
static PyObject*
positionsiter_next(positionsiterator* pi)
{
- if (pi->pi_offset >= PyBytes_GET_SIZE(pi->pi_code->co_code)) {
+ if (pi->pi_offset >= _PyCode_NBYTES(pi->pi_code)) {
return NULL;
}
@@ -1151,46 +1140,39 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
PyObject *
_PyCode_GetVarnames(PyCodeObject *co)
{
- if (co->co_varnames == NULL) {
- // PyCodeObject owns this reference.
- co->co_varnames = get_localsplus_names(co, CO_FAST_LOCAL,
- co->co_nlocals);
- if (co->co_varnames == NULL) {
- return NULL;
- }
- }
- Py_INCREF(co->co_varnames);
- return co->co_varnames;
+ return get_localsplus_names(co, CO_FAST_LOCAL, co->co_nlocals);
}
PyObject *
_PyCode_GetCellvars(PyCodeObject *co)
{
- if (co->co_cellvars == NULL) {
- // PyCodeObject owns this reference.
- co->co_cellvars = get_localsplus_names(co, CO_FAST_CELL,
- co->co_ncellvars);
- if (co->co_cellvars == NULL) {
- return NULL;
- }
- }
- Py_INCREF(co->co_cellvars);
- return co->co_cellvars;
+ return get_localsplus_names(co, CO_FAST_CELL, co->co_ncellvars);
}
PyObject *
_PyCode_GetFreevars(PyCodeObject *co)
{
- if (co->co_freevars == NULL) {
- // PyCodeObject owns this reference.
- co->co_freevars = get_localsplus_names(co, CO_FAST_FREE,
- co->co_nfreevars);
- if (co->co_freevars == NULL) {
- return NULL;
+ return get_localsplus_names(co, CO_FAST_FREE, co->co_nfreevars);
+}
+
+PyObject *
+_PyCode_GetCode(PyCodeObject *co)
+{
+ PyObject *code = PyBytes_FromStringAndSize(NULL, _PyCode_NBYTES(co));
+ if (code == NULL) {
+ return NULL;
+ }
+ _Py_CODEUNIT *instructions = (_Py_CODEUNIT *)PyBytes_AS_STRING(code);
+ for (int i = 0; i < Py_SIZE(co); i++) {
+ _Py_CODEUNIT instruction = _PyCode_CODE(co)[i];
+ int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)];
+ int caches = _PyOpcode_Caches[opcode];
+ instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction));
+ while (caches--) {
+ instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0);
}
}
- Py_INCREF(co->co_freevars);
- return co->co_freevars;
+ return code;
}
@@ -1348,14 +1330,10 @@ code_dealloc(PyCodeObject *co)
PyMem_Free(co_extra);
}
- Py_XDECREF(co->co_code);
Py_XDECREF(co->co_consts);
Py_XDECREF(co->co_names);
Py_XDECREF(co->co_localsplusnames);
Py_XDECREF(co->co_localspluskinds);
- Py_XDECREF(co->co_varnames);
- Py_XDECREF(co->co_freevars);
- Py_XDECREF(co->co_cellvars);
Py_XDECREF(co->co_filename);
Py_XDECREF(co->co_name);
Py_XDECREF(co->co_qualname);
@@ -1363,10 +1341,10 @@ code_dealloc(PyCodeObject *co)
Py_XDECREF(co->co_endlinetable);
Py_XDECREF(co->co_columntable);
Py_XDECREF(co->co_exceptiontable);
- if (co->co_weakreflist != NULL)
+ if (co->co_weakreflist != NULL) {
PyObject_ClearWeakRefs((PyObject*)co);
- if (co->co_quickened) {
- PyMem_Free(co->co_quickened);
+ }
+ if (co->co_warmup == 0) {
_Py_QuickenedCount--;
}
PyObject_Free(co);
@@ -1420,8 +1398,21 @@ code_richcompare(PyObject *self, PyObject *other, int op)
if (!eq) goto unequal;
eq = co->co_firstlineno == cp->co_firstlineno;
if (!eq) goto unequal;
- eq = PyObject_RichCompareBool(co->co_code, cp->co_code, Py_EQ);
- if (eq <= 0) goto unequal;
+ PyObject *co_code = _PyCode_GetCode(co);
+ if (co_code == NULL) {
+ return NULL;
+ }
+ PyObject *cp_code = _PyCode_GetCode(cp);
+ if (cp_code == NULL) {
+ Py_DECREF(co_code);
+ return NULL;
+ }
+ eq = PyObject_RichCompareBool(co_code, cp_code, Py_EQ);
+ Py_DECREF(co_code);
+ Py_DECREF(cp_code);
+ if (eq <= 0) {
+ goto unequal;
+ }
/* compare constants */
consts1 = _PyCode_ConstantKey(co->co_consts);
@@ -1465,18 +1456,16 @@ code_richcompare(PyObject *self, PyObject *other, int op)
static Py_hash_t
code_hash(PyCodeObject *co)
{
- Py_hash_t h, h0, h1, h2, h3, h4;
+ Py_hash_t h, h0, h1, h2, h3;
h0 = PyObject_Hash(co->co_name);
if (h0 == -1) return -1;
- h1 = PyObject_Hash(co->co_code);
+ h1 = PyObject_Hash(co->co_consts);
if (h1 == -1) return -1;
- h2 = PyObject_Hash(co->co_consts);
+ h2 = PyObject_Hash(co->co_names);
if (h2 == -1) return -1;
- h3 = PyObject_Hash(co->co_names);
+ h3 = PyObject_Hash(co->co_localsplusnames);
if (h3 == -1) return -1;
- h4 = PyObject_Hash(co->co_localsplusnames);
- if (h4 == -1) return -1;
- h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^
+ h = h0 ^ h1 ^ h2 ^ h3 ^
co->co_argcount ^ co->co_posonlyargcount ^ co->co_kwonlyargcount ^
co->co_flags;
if (h == -1) h = -2;
@@ -1487,22 +1476,22 @@ code_hash(PyCodeObject *co)
#define OFF(x) offsetof(PyCodeObject, x)
static PyMemberDef code_memberlist[] = {
- {"co_argcount", T_INT, OFF(co_argcount), READONLY},
- {"co_posonlyargcount", T_INT, OFF(co_posonlyargcount), READONLY},
- {"co_kwonlyargcount", T_INT, OFF(co_kwonlyargcount), READONLY},
- {"co_stacksize",T_INT, OFF(co_stacksize), READONLY},
- {"co_flags", T_INT, OFF(co_flags), READONLY},
- {"co_code", T_OBJECT, OFF(co_code), READONLY},
- {"co_consts", T_OBJECT, OFF(co_consts), READONLY},
- {"co_names", T_OBJECT, OFF(co_names), READONLY},
- {"co_filename", T_OBJECT, OFF(co_filename), READONLY},
- {"co_name", T_OBJECT, OFF(co_name), READONLY},
- {"co_qualname", T_OBJECT, OFF(co_qualname), READONLY},
- {"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY},
- {"co_linetable", T_OBJECT, OFF(co_linetable), READONLY},
- {"co_endlinetable", T_OBJECT, OFF(co_endlinetable), READONLY},
- {"co_columntable", T_OBJECT, OFF(co_columntable), READONLY},
- {"co_exceptiontable", T_OBJECT, OFF(co_exceptiontable), READONLY},
+ {"co_argcount", T_INT, OFF(co_argcount), READONLY},
+ {"co_posonlyargcount", T_INT, OFF(co_posonlyargcount), READONLY},
+ {"co_kwonlyargcount", T_INT, OFF(co_kwonlyargcount), READONLY},
+ {"co_stacksize", T_INT, OFF(co_stacksize), READONLY},
+ {"co_flags", T_INT, OFF(co_flags), READONLY},
+ {"co_nlocals", T_INT, OFF(co_nlocals), READONLY},
+ {"co_consts", T_OBJECT, OFF(co_consts), READONLY},
+ {"co_names", T_OBJECT, OFF(co_names), READONLY},
+ {"co_filename", T_OBJECT, OFF(co_filename), READONLY},
+ {"co_name", T_OBJECT, OFF(co_name), READONLY},
+ {"co_qualname", T_OBJECT, OFF(co_qualname), READONLY},
+ {"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY},
+ {"co_linetable", T_OBJECT, OFF(co_linetable), READONLY},
+ {"co_endlinetable", T_OBJECT, OFF(co_endlinetable), READONLY},
+ {"co_columntable", T_OBJECT, OFF(co_columntable), READONLY},
+ {"co_exceptiontable", T_OBJECT, OFF(co_exceptiontable), READONLY},
{NULL} /* Sentinel */
};
@@ -1513,12 +1502,6 @@ code_getlnotab(PyCodeObject *code, void *closure)
return decode_linetable(code);
}
-static PyObject *
-code_getnlocals(PyCodeObject *code, void *closure)
-{
- return PyLong_FromLong(code->co_nlocals);
-}
-
static PyObject *
code_getvarnames(PyCodeObject *code, void *closure)
{
@@ -1538,23 +1521,26 @@ code_getfreevars(PyCodeObject *code, void *closure)
}
static PyObject *
-code_getquickened(PyCodeObject *code, void *closure)
+code_getcodeadaptive(PyCodeObject *code, void *closure)
{
- if (code->co_quickened == NULL) {
- Py_RETURN_NONE;
- }
- return PyBytes_FromStringAndSize((char *)code->co_firstinstr,
- PyBytes_Size(code->co_code));
+ return PyMemoryView_FromMemory(code->co_code_adaptive, _PyCode_NBYTES(code),
+ PyBUF_READ);
+}
+
+static PyObject *
+code_getcode(PyCodeObject *code, void *closure)
+{
+ return _PyCode_GetCode(code);
}
static PyGetSetDef code_getsetlist[] = {
- {"co_lnotab", (getter)code_getlnotab, NULL, NULL},
+ {"co_lnotab", (getter)code_getlnotab, NULL, NULL},
+ {"_co_code_adaptive", (getter)code_getcodeadaptive, NULL, NULL},
// The following old names are kept for backward compatibility.
- {"co_nlocals", (getter)code_getnlocals, NULL, NULL},
- {"co_varnames", (getter)code_getvarnames, NULL, NULL},
- {"co_cellvars", (getter)code_getcellvars, NULL, NULL},
- {"co_freevars", (getter)code_getfreevars, NULL, NULL},
- {"_co_quickened", (getter)code_getquickened, NULL, NULL},
+ {"co_varnames", (getter)code_getvarnames, NULL, NULL},
+ {"co_cellvars", (getter)code_getcellvars, NULL, NULL},
+ {"co_freevars", (getter)code_getfreevars, NULL, NULL},
+ {"co_code", (getter)code_getcode, NULL, NULL},
{0}
};
@@ -1562,7 +1548,7 @@ static PyGetSetDef code_getsetlist[] = {
static PyObject *
code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
{
- Py_ssize_t res = _PyObject_SIZE(Py_TYPE(co));
+ Py_ssize_t res = _PyObject_VAR_SIZE(Py_TYPE(co), Py_SIZE(co));
_PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra;
if (co_extra != NULL) {
@@ -1570,10 +1556,6 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
(co_extra->ce_size-1) * sizeof(co_extra->ce_extras[0]);
}
- if (co->co_quickened != NULL) {
- res += PyBytes_GET_SIZE(co->co_code);
- }
-
return PyLong_FromSsize_t(res);
}
@@ -1594,12 +1576,12 @@ code.replace
co_stacksize: int(c_default="self->co_stacksize") = -1
co_flags: int(c_default="self->co_flags") = -1
co_firstlineno: int(c_default="self->co_firstlineno") = -1
- co_code: PyBytesObject(c_default="(PyBytesObject *)self->co_code") = None
+ co_code: PyBytesObject(c_default="NULL") = None
co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = None
co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = None
- co_varnames: object(subclass_of="&PyTuple_Type", c_default="self->co_varnames") = None
- co_freevars: object(subclass_of="&PyTuple_Type", c_default="self->co_freevars") = None
- co_cellvars: object(subclass_of="&PyTuple_Type", c_default="self->co_cellvars") = None
+ co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = None
+ co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None
+ co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None
co_filename: unicode(c_default="self->co_filename") = None
co_name: unicode(c_default="self->co_name") = None
co_qualname: unicode(c_default="self->co_qualname") = None
@@ -1622,7 +1604,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
PyObject *co_name, PyObject *co_qualname,
PyBytesObject *co_linetable, PyObject *co_endlinetable,
PyObject *co_columntable, PyBytesObject *co_exceptiontable)
-/*[clinic end generated code: output=f046bf0be3bab91f input=a63d09f248f00794]*/
+/*[clinic end generated code: output=f046bf0be3bab91f input=78dbe204dbd06c2f]*/
{
#define CHECK_INT_ARG(ARG) \
if (ARG < 0) { \
@@ -1641,6 +1623,15 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
#undef CHECK_INT_ARG
+ PyObject *code = NULL;
+ if (co_code == NULL) {
+ code = _PyCode_GetCode(self);
+ if (code == NULL) {
+ return NULL;
+ }
+ co_code = (PyBytesObject *)code;
+ }
+
if (PySys_Audit("code.__new__", "OOOiiiiii",
co_code, co_filename, co_name, co_argcount,
co_posonlyargcount, co_kwonlyargcount, co_nlocals,
@@ -1694,6 +1685,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
(PyObject*)co_exceptiontable);
error:
+ Py_XDECREF(code);
Py_XDECREF(varnames);
Py_XDECREF(cellvars);
Py_XDECREF(freevars);
@@ -1737,8 +1729,8 @@ static struct PyMethodDef code_methods[] = {
PyTypeObject PyCode_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"code",
- sizeof(PyCodeObject),
- 0,
+ offsetof(PyCodeObject, co_code_adaptive),
+ sizeof(_Py_CODEUNIT),
(destructor)code_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
@@ -1913,15 +1905,12 @@ _PyCode_ConstantKey(PyObject *op)
void
_PyStaticCode_Dealloc(PyCodeObject *co)
{
- if (co->co_quickened) {
- PyMem_Free(co->co_quickened);
- co->co_quickened = NULL;
+ if (co->co_warmup == 0) {
_Py_QuickenedCount--;
}
co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
PyMem_Free(co->co_extra);
co->co_extra = NULL;
- co->co_firstinstr = (_Py_CODEUNIT *)PyBytes_AS_STRING(co->co_code);
if (co->co_weakreflist != NULL) {
PyObject_ClearWeakRefs((PyObject *)co);
co->co_weakreflist = NULL;
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
index 73b6c3d9f8ab7..7ccd300e2b454 100644
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -105,8 +105,9 @@ frame_getback(PyFrameObject *f, void *closure)
return res;
}
-/* Given the index of the effective opcode,
- scan back to construct the oparg with EXTENDED_ARG */
+// Given the index of the effective opcode, scan back to construct the oparg
+// with EXTENDED_ARG. This only works correctly with *unquickened* code,
+// obtained via a call to _PyCode_GetCode!
static unsigned int
get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i)
{
@@ -170,13 +171,17 @@ top_of_stack(int64_t stack)
static int64_t *
mark_stacks(PyCodeObject *code_obj, int len)
{
- const _Py_CODEUNIT *code =
- (const _Py_CODEUNIT *)PyBytes_AS_STRING(code_obj->co_code);
+ PyObject *co_code = _PyCode_GetCode(code_obj);
+ if (co_code == NULL) {
+ return NULL;
+ }
+ _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(co_code);
int64_t *stacks = PyMem_New(int64_t, len+1);
int i, j, opcode;
if (stacks == NULL) {
PyErr_NoMemory();
+ Py_DECREF(co_code);
return NULL;
}
for (int i = 1; i <= len; i++) {
@@ -304,6 +309,7 @@ mark_stacks(PyCodeObject *code_obj, int len)
}
}
}
+ Py_DECREF(co_code);
return stacks;
}
@@ -493,7 +499,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
/* PyCode_NewWithPosOnlyArgs limits co_code to be under INT_MAX so this
* should never overflow. */
- int len = (int)(PyBytes_GET_SIZE(f->f_frame->f_code->co_code) / sizeof(_Py_CODEUNIT));
+ int len = (int)Py_SIZE(f->f_frame->f_code);
int *lines = marklines(f->f_frame->f_code, len);
if (lines == NULL) {
return -1;
@@ -838,12 +844,23 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
static int
_PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg)
{
- const _Py_CODEUNIT *code =
- (const _Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code);
+ // This only works when opcode is a non-quickened form:
+ assert(_PyOpcode_Deopt[opcode] == opcode);
+ int check_oparg = 0;
for (int i = 0; i < frame->f_lasti; i++) {
- if (_Py_OPCODE(code[i]) == opcode && _Py_OPARG(code[i]) == oparg) {
+ _Py_CODEUNIT instruction = _PyCode_CODE(frame->f_code)[i];
+ int check_opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)];
+ check_oparg |= _Py_OPARG(instruction);
+ if (check_opcode == opcode && check_oparg == oparg) {
return 1;
}
+ if (check_opcode == EXTENDED_ARG) {
+ check_oparg <<= 8;
+ }
+ else {
+ check_oparg = 0;
+ }
+ i += _PyOpcode_Caches[check_opcode];
}
return 0;
}
@@ -862,7 +879,10 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) {
}
co = frame->f_code;
fast = _PyFrame_GetLocalsArray(frame);
- if (frame->f_lasti < 0 && _Py_OPCODE(co->co_firstinstr[0]) == COPY_FREE_VARS) {
+ // COPY_FREE_VARS has no quickened forms, so no need to use _PyOpcode_Deopt
+ // here:
+ if (frame->f_lasti < 0 && _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS)
+ {
/* Free vars have not been initialized -- Do that */
PyCodeObject *co = frame->f_code;
PyObject *closure = frame->f_func->func_closure;
@@ -872,6 +892,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) {
Py_INCREF(o);
frame->localsplus[offset + i] = o;
}
+ // COPY_FREE_VARS doesn't have inline CACHEs, either:
frame->f_lasti = 0;
}
for (int i = 0; i < co->co_nlocalsplus; i++) {
diff --git a/Objects/genobject.c b/Objects/genobject.c
index 6551b939c4590..3ad8dc1c45942 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -349,19 +349,17 @@ _PyGen_yf(PyGenObject *gen)
if (gen->gi_frame_valid) {
_PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe;
- PyObject *bytecode = gen->gi_code->co_code;
- unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
if (frame->f_lasti < 1) {
/* Return immediately if the frame didn't start yet. SEND
always come after LOAD_CONST: a code object should not start
with SEND */
- assert(code[0] != SEND);
+ assert(_Py_OPCODE(_PyCode_CODE(gen->gi_code)[0]) != SEND);
return NULL;
}
- int opcode = code[(frame->f_lasti+1)*sizeof(_Py_CODEUNIT)];
- int oparg = code[(frame->f_lasti+1)*sizeof(_Py_CODEUNIT)+1];
- if (opcode != RESUME || oparg < 2) {
+ _Py_CODEUNIT next = _PyCode_CODE(gen->gi_code)[frame->f_lasti + 1];
+ if (_PyOpcode_Deopt[_Py_OPCODE(next)] != RESUME || _Py_OPARG(next) < 2)
+ {
/* Not in a yield from */
return NULL;
}
@@ -485,14 +483,15 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
ret = _PyFrame_StackPop((_PyInterpreterFrame *)gen->gi_iframe);
assert(ret == yf);
Py_DECREF(ret);
+ // XXX: Performing this jump ourselves is awkward and problematic.
+ // See https://github.com/python/cpython/pull/31968.
/* Termination repetition of SEND loop */
assert(frame->f_lasti >= 0);
- PyObject *bytecode = gen->gi_code->co_code;
- unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
+ _Py_CODEUNIT *code = _PyCode_CODE(gen->gi_code);
/* Backup to SEND */
frame->f_lasti--;
- assert(code[frame->f_lasti*sizeof(_Py_CODEUNIT)] == SEND);
- int jump = code[frame->f_lasti*sizeof(_Py_CODEUNIT)+1];
+ assert(_Py_OPCODE(code[frame->f_lasti]) == SEND);
+ int jump = _Py_OPARG(code[frame->f_lasti]);
frame->f_lasti += jump;
if (_PyGen_FetchStopIterationValue(&val) == 0) {
ret = gen_send(gen, val);
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 7879515075613..4bed3ef49289a 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -8949,7 +8949,10 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co,
// "firstarg" is a cell here unless (very unlikely) super()
// was called from the C-API before the first MAKE_CELL op.
if (cframe->f_lasti >= 0) {
- assert(_Py_OPCODE(*co->co_firstinstr) == MAKE_CELL || _Py_OPCODE(*co->co_firstinstr) == COPY_FREE_VARS);
+ // MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need
+ // to use _PyOpcode_Deopt here:
+ assert(_Py_OPCODE(_PyCode_CODE(co)[0]) == MAKE_CELL ||
+ _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS);
assert(PyCell_Check(firstarg));
firstarg = PyCell_GET(firstarg);
}
diff --git a/Python/ceval.c b/Python/ceval.c
index 6f449e3172d08..7c6bfd46900e6 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1327,9 +1327,8 @@ eval_frame_handle_pending(PyThreadState *tstate)
/* Get opcode and oparg from original instructions, not quickened form. */
#define TRACING_NEXTOPARG() do { \
- _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code))[INSTR_OFFSET()]; \
- opcode = _Py_OPCODE(word); \
- oparg = _Py_OPARG(word); \
+ NEXTOPARG(); \
+ opcode = _PyOpcode_Deopt[opcode]; \
} while (0)
/* OpCode prediction macros
@@ -1650,9 +1649,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
PyCodeObject *co = frame->f_code; \
names = co->co_names; \
consts = co->co_consts; \
- first_instr = co->co_firstinstr; \
+ first_instr = _PyCode_CODE(co); \
} \
assert(frame->f_lasti >= -1); \
+ /* Jump back to the last instruction executed... */ \
next_instr = first_instr + frame->f_lasti + 1; \
stack_pointer = _PyFrame_GetStackPointer(frame); \
/* Set stackdepth to -1. \
@@ -1722,16 +1722,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
}
TARGET(RESUME) {
- int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code);
- if (err) {
- if (err < 0) {
- goto error;
- }
- /* Update first_instr and next_instr to point to newly quickened code */
- int nexti = INSTR_OFFSET();
- first_instr = frame->f_code->co_firstinstr;
- next_instr = first_instr + nexti;
- }
+ _PyCode_Warmup(frame->f_code);
JUMP_TO_INSTRUCTION(RESUME_QUICK);
}
@@ -4067,16 +4058,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
TARGET(JUMP_ABSOLUTE) {
PREDICTED(JUMP_ABSOLUTE);
- int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code);
- if (err) {
- if (err < 0) {
- goto error;
- }
- /* Update first_instr and next_instr to point to newly quickened code */
- int nexti = INSTR_OFFSET();
- first_instr = frame->f_code->co_firstinstr;
- next_instr = first_instr + nexti;
- }
+ _PyCode_Warmup(frame->f_code);
JUMP_TO_INSTRUCTION(JUMP_ABSOLUTE_QUICK);
}
@@ -5425,6 +5407,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
}
TARGET(EXTENDED_ARG) {
+ assert(oparg);
int oldoparg = oparg;
NEXTOPARG();
oparg |= oldoparg << 8;
@@ -6739,8 +6722,8 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
*/
initialize_trace_info(&tstate->trace_info, frame);
int entry_point = 0;
- _Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code);
- while (_Py_OPCODE(code[entry_point]) != RESUME) {
+ _Py_CODEUNIT *code = _PyCode_CODE(frame->f_code);
+ while (_PyOpcode_Deopt[_Py_OPCODE(code[entry_point])] != RESUME) {
entry_point++;
}
int lastline;
@@ -6759,7 +6742,9 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
/* Trace backward edges (except in 'yield from') or if line number has changed */
int trace = line != lastline ||
(frame->f_lasti < instr_prev &&
- _Py_OPCODE(frame->f_code->co_firstinstr[frame->f_lasti]) != SEND);
+ // SEND has no quickened forms, so no need to use _PyOpcode_Deopt
+ // here:
+ _Py_OPCODE(_PyCode_CODE(frame->f_code)[frame->f_lasti]) != SEND);
if (trace) {
result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None);
}
diff --git a/Python/compile.c b/Python/compile.c
index 950c44a749fc6..e24f425229b6a 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -134,9 +134,9 @@ static int
instr_size(struct instr *instruction)
{
int opcode = instruction->i_opcode;
- int oparg = instruction->i_oparg;
+ int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0;
int extended_args = (0xFFFFFF < oparg) + (0xFFFF < oparg) + (0xFF < oparg);
- int caches = _PyOpcode_InlineCacheEntries[opcode];
+ int caches = _PyOpcode_Caches[opcode];
return extended_args + 1 + caches;
}
@@ -144,8 +144,8 @@ static void
write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen)
{
int opcode = instruction->i_opcode;
- int oparg = instruction->i_oparg;
- int caches = _PyOpcode_InlineCacheEntries[opcode];
+ int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0;
+ int caches = _PyOpcode_Caches[opcode];
switch (ilen - caches) {
case 4:
*codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 24) & 0xFF);
diff --git a/Python/marshal.c b/Python/marshal.c
index 810244ba8ac78..e7cf6553bd142 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -544,13 +544,18 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
}
else if (PyCode_Check(v)) {
PyCodeObject *co = (PyCodeObject *)v;
+ PyObject *co_code = _PyCode_GetCode(co);
+ if (co_code == NULL) {
+ p->error = WFERR_NOMEMORY;
+ return;
+ }
W_TYPE(TYPE_CODE, p);
w_long(co->co_argcount, p);
w_long(co->co_posonlyargcount, p);
w_long(co->co_kwonlyargcount, p);
w_long(co->co_stacksize, p);
w_long(co->co_flags, p);
- w_object(co->co_code, p);
+ w_object(co_code, p);
w_object(co->co_consts, p);
w_object(co->co_names, p);
w_object(co->co_localsplusnames, p);
@@ -563,6 +568,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
w_object(co->co_endlinetable, p);
w_object(co->co_columntable, p);
w_object(co->co_exceptiontable, p);
+ Py_DECREF(co_code);
}
else if (PyObject_CheckBuffer(v)) {
/* Write unknown bytes-like objects as a bytes object */
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index 7c94999bfdfe4..2aa6471abf99a 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -3,49 +3,49 @@ static void *opcode_targets[256] = {
&&TARGET_POP_TOP,
&&TARGET_PUSH_NULL,
&&TARGET_BINARY_OP_ADAPTIVE,
- &&TARGET_BINARY_OP_ADD_INT,
&&TARGET_BINARY_OP_ADD_FLOAT,
+ &&TARGET_BINARY_OP_ADD_INT,
&&TARGET_BINARY_OP_ADD_UNICODE,
&&TARGET_BINARY_OP_INPLACE_ADD_UNICODE,
- &&TARGET_BINARY_OP_MULTIPLY_INT,
+ &&TARGET_BINARY_OP_MULTIPLY_FLOAT,
&&TARGET_NOP,
&&TARGET_UNARY_POSITIVE,
&&TARGET_UNARY_NEGATIVE,
&&TARGET_UNARY_NOT,
- &&TARGET_BINARY_OP_MULTIPLY_FLOAT,
- &&TARGET_BINARY_OP_SUBTRACT_INT,
- &&TARGET_UNARY_INVERT,
+ &&TARGET_BINARY_OP_MULTIPLY_INT,
&&TARGET_BINARY_OP_SUBTRACT_FLOAT,
- &&TARGET_COMPARE_OP_ADAPTIVE,
- &&TARGET_COMPARE_OP_FLOAT_JUMP,
- &&TARGET_COMPARE_OP_INT_JUMP,
- &&TARGET_COMPARE_OP_STR_JUMP,
+ &&TARGET_UNARY_INVERT,
+ &&TARGET_BINARY_OP_SUBTRACT_INT,
&&TARGET_BINARY_SUBSCR_ADAPTIVE,
+ &&TARGET_BINARY_SUBSCR_DICT,
&&TARGET_BINARY_SUBSCR_GETITEM,
&&TARGET_BINARY_SUBSCR_LIST_INT,
&&TARGET_BINARY_SUBSCR_TUPLE_INT,
+ &&TARGET_CALL_ADAPTIVE,
+ &&TARGET_CALL_PY_EXACT_ARGS,
+ &&TARGET_CALL_PY_WITH_DEFAULTS,
&&TARGET_BINARY_SUBSCR,
- &&TARGET_BINARY_SUBSCR_DICT,
- &&TARGET_STORE_SUBSCR_ADAPTIVE,
- &&TARGET_STORE_SUBSCR_LIST_INT,
- &&TARGET_STORE_SUBSCR_DICT,
+ &&TARGET_COMPARE_OP_ADAPTIVE,
+ &&TARGET_COMPARE_OP_FLOAT_JUMP,
+ &&TARGET_COMPARE_OP_INT_JUMP,
+ &&TARGET_COMPARE_OP_STR_JUMP,
&&TARGET_GET_LEN,
&&TARGET_MATCH_MAPPING,
&&TARGET_MATCH_SEQUENCE,
&&TARGET_MATCH_KEYS,
- &&TARGET_CALL_ADAPTIVE,
- &&TARGET_PUSH_EXC_INFO,
- &&TARGET_CALL_PY_EXACT_ARGS,
- &&TARGET_CALL_PY_WITH_DEFAULTS,
&&TARGET_JUMP_ABSOLUTE_QUICK,
+ &&TARGET_PUSH_EXC_INFO,
&&TARGET_LOAD_ATTR_ADAPTIVE,
&&TARGET_LOAD_ATTR_INSTANCE_VALUE,
- &&TARGET_LOAD_ATTR_WITH_HINT,
- &&TARGET_LOAD_ATTR_SLOT,
&&TARGET_LOAD_ATTR_MODULE,
+ &&TARGET_LOAD_ATTR_SLOT,
+ &&TARGET_LOAD_ATTR_WITH_HINT,
+ &&TARGET_LOAD_CONST__LOAD_FAST,
+ &&TARGET_LOAD_FAST__LOAD_CONST,
+ &&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_LOAD_GLOBAL_ADAPTIVE,
- &&TARGET_LOAD_GLOBAL_MODULE,
&&TARGET_LOAD_GLOBAL_BUILTIN,
+ &&TARGET_LOAD_GLOBAL_MODULE,
&&TARGET_LOAD_METHOD_ADAPTIVE,
&&TARGET_LOAD_METHOD_CLASS,
&&TARGET_WITH_EXCEPT_START,
@@ -61,26 +61,26 @@ static void *opcode_targets[256] = {
&&TARGET_PRECALL_ADAPTIVE,
&&TARGET_STORE_SUBSCR,
&&TARGET_DELETE_SUBSCR,
+ &&TARGET_PRECALL_BOUND_METHOD,
&&TARGET_PRECALL_BUILTIN_CLASS,
- &&TARGET_PRECALL_NO_KW_BUILTIN_O,
- &&TARGET_PRECALL_NO_KW_BUILTIN_FAST,
&&TARGET_PRECALL_BUILTIN_FAST_WITH_KEYWORDS,
- &&TARGET_PRECALL_NO_KW_LEN,
+ &&TARGET_PRECALL_NO_KW_BUILTIN_FAST,
+ &&TARGET_PRECALL_NO_KW_BUILTIN_O,
&&TARGET_PRECALL_NO_KW_ISINSTANCE,
&&TARGET_GET_ITER,
&&TARGET_GET_YIELD_FROM_ITER,
&&TARGET_PRINT_EXPR,
&&TARGET_LOAD_BUILD_CLASS,
+ &&TARGET_PRECALL_NO_KW_LEN,
&&TARGET_PRECALL_NO_KW_LIST_APPEND,
- &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O,
&&TARGET_LOAD_ASSERTION_ERROR,
&&TARGET_RETURN_GENERATOR,
+ &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST,
&&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
+ &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O,
&&TARGET_PRECALL_NO_KW_STR_1,
&&TARGET_PRECALL_NO_KW_TUPLE_1,
&&TARGET_PRECALL_NO_KW_TYPE_1,
- &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST,
- &&TARGET_PRECALL_BOUND_METHOD,
&&TARGET_LIST_TO_TUPLE,
&&TARGET_RETURN_VALUE,
&&TARGET_IMPORT_STAR,
@@ -157,23 +157,23 @@ static void *opcode_targets[256] = {
&&TARGET_FORMAT_VALUE,
&&TARGET_BUILD_CONST_KEY_MAP,
&&TARGET_BUILD_STRING,
- &&TARGET_UNPACK_SEQUENCE_ADAPTIVE,
- &&TARGET_UNPACK_SEQUENCE_LIST,
+ &&TARGET_STORE_FAST__LOAD_FAST,
+ &&TARGET_STORE_FAST__STORE_FAST,
&&TARGET_LOAD_METHOD,
- &&TARGET_UNPACK_SEQUENCE_TUPLE,
+ &&TARGET_STORE_SUBSCR_ADAPTIVE,
&&TARGET_LIST_EXTEND,
&&TARGET_SET_UPDATE,
&&TARGET_DICT_MERGE,
&&TARGET_DICT_UPDATE,
&&TARGET_PRECALL,
- &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
- &&TARGET_LOAD_FAST__LOAD_FAST,
- &&TARGET_STORE_FAST__LOAD_FAST,
- &&TARGET_LOAD_FAST__LOAD_CONST,
+ &&TARGET_STORE_SUBSCR_DICT,
+ &&TARGET_STORE_SUBSCR_LIST_INT,
+ &&TARGET_UNPACK_SEQUENCE_ADAPTIVE,
+ &&TARGET_UNPACK_SEQUENCE_LIST,
&&TARGET_CALL,
&&TARGET_KW_NAMES,
- &&TARGET_LOAD_CONST__LOAD_FAST,
- &&TARGET_STORE_FAST__STORE_FAST,
+ &&TARGET_UNPACK_SEQUENCE_TUPLE,
+ &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
diff --git a/Python/specialize.c b/Python/specialize.c
index d84adac352078..ce091a2c5f078 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -15,31 +15,6 @@
* ./adaptive.md
*/
-
-/* We layout the quickened data as a bi-directional array:
- * Instructions upwards, cache entries downwards.
- * first_instr is aligned to a SpecializedCacheEntry.
- * The nth instruction is located at first_instr[n]
- * The nth cache is located at ((SpecializedCacheEntry *)first_instr)[-1-n]
- * The first (index 0) cache entry is reserved for the count, to enable finding
- * the first instruction from the base pointer.
- * The cache_count argument must include space for the count.
- * We use the SpecializedCacheOrInstruction union to refer to the data
- * to avoid type punning.
-
- Layout of quickened data, each line 8 bytes for M cache entries and N instructions:
-
- <cache_count> <---- co->co_quickened
- <cache M-1>
- <cache M-2>
- ...
- <cache 0>
- <instr 0> <instr 1> <instr 2> <instr 3> <--- co->co_first_instr
- <instr 4> <instr 5> <instr 6> <instr 7>
- ...
- <instr N-1>
-*/
-
/* Map from opcode to adaptive opcode.
Values of zero are ignored. */
static uint8_t adaptive_opcodes[256] = {
@@ -275,26 +250,14 @@ _Py_PrintSpecializationStats(int to_file)
#define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
#endif
-static _Py_CODEUNIT *
-allocate(int instruction_count)
+// Insert adaptive instructions and superinstructions. This cannot fail.
+void
+_PyCode_Quicken(PyCodeObject *code)
{
- assert(instruction_count > 0);
- void *array = PyMem_Malloc(sizeof(_Py_CODEUNIT) * instruction_count);
- if (array == NULL) {
- PyErr_NoMemory();
- return NULL;
- }
_Py_QuickenedCount++;
- return (_Py_CODEUNIT *)array;
-}
-
-
-// Insert adaptive instructions and superinstructions.
-static void
-optimize(_Py_CODEUNIT *instructions, int len)
-{
int previous_opcode = -1;
- for(int i = 0; i < len; i++) {
+ _Py_CODEUNIT *instructions = _PyCode_CODE(code);
+ for (int i = 0; i < Py_SIZE(code); i++) {
int opcode = _Py_OPCODE(instructions[i]);
uint8_t adaptive_opcode = adaptive_opcodes[opcode];
if (adaptive_opcode) {
@@ -302,10 +265,10 @@ optimize(_Py_CODEUNIT *instructions, int len)
// Make sure the adaptive counter is zero:
assert(instructions[i + 1] == 0);
previous_opcode = -1;
- i += _PyOpcode_InlineCacheEntries[opcode];
+ i += _PyOpcode_Caches[opcode];
}
else {
- assert(!_PyOpcode_InlineCacheEntries[opcode]);
+ assert(!_PyOpcode_Caches[opcode]);
switch (opcode) {
case JUMP_ABSOLUTE:
_Py_SET_OPCODE(instructions[i], JUMP_ABSOLUTE_QUICK);
@@ -347,28 +310,6 @@ optimize(_Py_CODEUNIT *instructions, int len)
}
}
-int
-_Py_Quicken(PyCodeObject *code) {
- if (code->co_quickened) {
- return 0;
- }
- Py_ssize_t size = PyBytes_GET_SIZE(code->co_code);
- int instr_count = (int)(size/sizeof(_Py_CODEUNIT));
- if (instr_count > MAX_SIZE_TO_QUICKEN) {
- code->co_warmup = QUICKENING_WARMUP_COLDEST;
- return 0;
- }
- _Py_CODEUNIT *quickened = allocate(instr_count);
- if (quickened == NULL) {
- return -1;
- }
- memcpy(quickened, code->co_firstinstr, size);
- optimize(quickened, instr_count);
- code->co_quickened = quickened;
- code->co_firstinstr = quickened;
- return 0;
-}
-
static inline int
initial_counter_value(void) {
/* Starting value for the counter.
@@ -705,8 +646,7 @@ specialize_dict_access(
int
_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
- assert(_PyOpcode_InlineCacheEntries[LOAD_ATTR] ==
- INLINE_CACHE_ENTRIES_LOAD_ATTR);
+ assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR);
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
if (PyModule_CheckExact(owner)) {
int err = specialize_module_load_attr(owner, instr, name, LOAD_ATTR,
@@ -804,8 +744,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
int
_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
- assert(_PyOpcode_InlineCacheEntries[STORE_ATTR] ==
- INLINE_CACHE_ENTRIES_STORE_ATTR);
+ assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyTypeObject *type = Py_TYPE(owner);
if (PyModule_CheckExact(owner)) {
@@ -965,8 +904,7 @@ typedef enum {
int
_Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
- assert(_PyOpcode_InlineCacheEntries[LOAD_METHOD] ==
- INLINE_CACHE_ENTRIES_LOAD_METHOD);
+ assert(_PyOpcode_Caches[LOAD_METHOD] == INLINE_CACHE_ENTRIES_LOAD_METHOD);
_PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
PyTypeObject *owner_cls = Py_TYPE(owner);
@@ -1098,8 +1036,7 @@ _Py_Specialize_LoadGlobal(
PyObject *globals, PyObject *builtins,
_Py_CODEUNIT *instr, PyObject *name)
{
- assert(_PyOpcode_InlineCacheEntries[LOAD_GLOBAL] ==
- INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
+ assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
/* Use inline cache */
_PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1);
assert(PyUnicode_CheckExact(name));
@@ -1235,7 +1172,7 @@ int
_Py_Specialize_BinarySubscr(
PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
{
- assert(_PyOpcode_InlineCacheEntries[BINARY_SUBSCR] ==
+ assert(_PyOpcode_Caches[BINARY_SUBSCR] ==
INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
_PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1);
PyTypeObject *container_type = Py_TYPE(container);
@@ -1663,8 +1600,7 @@ int
_Py_Specialize_Precall(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
PyObject *kwnames, int oparg)
{
- assert(_PyOpcode_InlineCacheEntries[PRECALL] ==
- INLINE_CACHE_ENTRIES_PRECALL);
+ assert(_PyOpcode_Caches[PRECALL] == INLINE_CACHE_ENTRIES_PRECALL);
_PyPrecallCache *cache = (_PyPrecallCache *)(instr + 1);
int fail;
if (PyCFunction_CheckExact(callable)) {
@@ -1710,7 +1646,7 @@ int
_Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
PyObject *kwnames)
{
- assert(_PyOpcode_InlineCacheEntries[CALL] == INLINE_CACHE_ENTRIES_CALL);
+ assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL);
_PyCallCache *cache = (_PyCallCache *)(instr + 1);
int fail;
if (PyFunction_Check(callable)) {
@@ -1808,8 +1744,7 @@ void
_Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
int oparg)
{
- assert(_PyOpcode_InlineCacheEntries[BINARY_OP] ==
- INLINE_CACHE_ENTRIES_BINARY_OP);
+ assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP);
_PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1);
switch (oparg) {
case NB_ADD:
@@ -1936,8 +1871,7 @@ void
_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
int oparg)
{
- assert(_PyOpcode_InlineCacheEntries[COMPARE_OP] ==
- INLINE_CACHE_ENTRIES_COMPARE_OP);
+ assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
_PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1);
int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]);
if (next_opcode != POP_JUMP_IF_FALSE && next_opcode != POP_JUMP_IF_TRUE) {
@@ -2019,7 +1953,7 @@ unpack_sequence_fail_kind(PyObject *seq)
void
_Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg)
{
- assert(_PyOpcode_InlineCacheEntries[UNPACK_SEQUENCE] ==
+ assert(_PyOpcode_Caches[UNPACK_SEQUENCE] ==
INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
_PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)(instr + 1);
if (PyTuple_CheckExact(seq)) {
diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py
index 954fca81b51e9..d208258dbc54c 100644
--- a/Tools/scripts/deepfreeze.py
+++ b/Tools/scripts/deepfreeze.py
@@ -229,12 +229,8 @@ def generate_unicode(self, name: str, s: str) -> str:
def generate_code(self, name: str, code: types.CodeType) -> str:
# The ordering here matches PyCode_NewWithPosOnlyArgs()
# (but see below).
- co_code = self.generate(name + "_code", code.co_code)
co_consts = self.generate(name + "_consts", code.co_consts)
co_names = self.generate(name + "_names", code.co_names)
- co_varnames = self.generate(name + "_varnames", code.co_varnames)
- co_freevars = self.generate(name + "_freevars", code.co_freevars)
- co_cellvars = self.generate(name + "_cellvars", code.co_cellvars)
co_filename = self.generate(name + "_filename", code.co_filename)
co_name = self.generate(name + "_name", code.co_name)
co_qualname = self.generate(name + "_qualname", code.co_qualname)
@@ -249,14 +245,17 @@ def generate_code(self, name: str, code: types.CodeType) -> str:
# Derived values
nlocals, nplaincellvars, ncellvars, nfreevars = \
get_localsplus_counts(code, localsplusnames, localspluskinds)
- with self.block(f"static struct PyCodeObject {name} =", ";"):
- self.object_head("PyCode_Type")
+ co_code_adaptive = make_string_literal(code.co_code)
+ self.write("static")
+ with self.indent():
+ self.write(f"struct _PyCode_DEF({len(code.co_code)})")
+ with self.block(f"{name} =", ";"):
+ self.object_var_head("PyCode_Type", len(code.co_code) // 2)
# But the ordering here must match that in cpython/code.h
# (which is a pain because we tend to reorder those for perf)
# otherwise MSVC doesn't like it.
self.write(f".co_consts = {co_consts},")
self.write(f".co_names = {co_names},")
- self.write(f".co_firstinstr = (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval,")
self.write(f".co_exceptiontable = {co_exceptiontable},")
self.field(code, "co_flags")
self.write(".co_warmup = QUICKENING_INITIAL_WARMUP_VALUE,")
@@ -265,7 +264,11 @@ def generate_code(self, name: str, code: types.CodeType) -> str:
self.field(code, "co_kwonlyargcount")
self.field(code, "co_stacksize")
self.field(code, "co_firstlineno")
- self.write(f".co_code = {co_code},")
+ self.write(f".co_nlocalsplus = {len(localsplusnames)},")
+ self.field(code, "co_nlocals")
+ self.write(f".co_nplaincellvars = {nplaincellvars},")
+ self.write(f".co_ncellvars = {ncellvars},")
+ self.write(f".co_nfreevars = {nfreevars},")
self.write(f".co_localsplusnames = {co_localsplusnames},")
self.write(f".co_localspluskinds = {co_localspluskinds},")
self.write(f".co_filename = {co_filename},")
@@ -274,17 +277,11 @@ def generate_code(self, name: str, code: types.CodeType) -> str:
self.write(f".co_linetable = {co_linetable},")
self.write(f".co_endlinetable = {co_endlinetable},")
self.write(f".co_columntable = {co_columntable},")
- self.write(f".co_nlocalsplus = {len(localsplusnames)},")
- self.field(code, "co_nlocals")
- self.write(f".co_nplaincellvars = {nplaincellvars},")
- self.write(f".co_ncellvars = {ncellvars},")
- self.write(f".co_nfreevars = {nfreevars},")
- self.write(f".co_varnames = {co_varnames},")
- self.write(f".co_cellvars = {co_cellvars},")
- self.write(f".co_freevars = {co_freevars},")
- self.deallocs.append(f"_PyStaticCode_Dealloc(&{name});")
- self.interns.append(f"_PyStaticCode_InternStrings(&{name})")
- return f"& {name}.ob_base"
+ self.write(f".co_code_adaptive = {co_code_adaptive},")
+ name_as_code = f"(PyCodeObject *)&{name}"
+ self.deallocs.append(f"_PyStaticCode_Dealloc({name_as_code});")
+ self.interns.append(f"_PyStaticCode_InternStrings({name_as_code})")
+ return f"& {name}.ob_base.ob_base"
def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str:
if len(t) == 0:
@@ -450,13 +447,13 @@ def generate(args: list[str], output: TextIO) -> None:
code = compile(fd.read(), f"<frozen {modname}>", "exec")
printer.generate_file(modname, code)
with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"):
- for p in printer.deallocs:
- printer.write(p)
+ for p in printer.deallocs:
+ printer.write(p)
with printer.block(f"int\n_Py_Deepfreeze_Init(void)"):
- for p in printer.interns:
- with printer.block(f"if ({p} < 0)"):
- printer.write("return -1;")
- printer.write("return 0;")
+ for p in printer.interns:
+ with printer.block(f"if ({p} < 0)"):
+ printer.write("return -1;")
+ printer.write("return 0;")
if verbose:
print(f"Cache hits: {printer.hits}, misses: {printer.misses}")
diff --git a/Tools/scripts/generate_opcode_h.py b/Tools/scripts/generate_opcode_h.py
index 75a9c3f3bfadb..3b79dc6b7359f 100644
--- a/Tools/scripts/generate_opcode_h.py
+++ b/Tools/scripts/generate_opcode_h.py
@@ -28,7 +28,7 @@
#endif /* !Py_OPCODE_H */
"""
-DEFINE = "#define {:<31} {:>3}\n"
+DEFINE = "#define {:<38} {:>3}\n"
UINT32_MASK = (1<<32)-1
@@ -75,16 +75,27 @@ def main(opcode_py, outfile='Include/opcode.h'):
fobj.write(DEFINE.format(name, next_op))
used[next_op] = True
fobj.write(DEFINE.format('DO_TRACING', 255))
- fobj.write("\nextern const uint8_t _PyOpcode_InlineCacheEntries[256];\n")
+ fobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
+ fobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n")
fobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj)
write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj)
- fobj.write("\nconst uint8_t _PyOpcode_InlineCacheEntries[256] = {\n")
+ fobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n")
for i, entries in enumerate(opcode["_inline_cache_entries"]):
if entries:
fobj.write(f" [{opname[i]}] = {entries},\n")
fobj.write("};\n")
+ deoptcodes = {}
+ for basic in opmap:
+ deoptcodes[basic] = basic
+ for basic, family in opcode["_specializations"].items():
+ for specialized in family:
+ deoptcodes[specialized] = basic
+ fobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n")
+ for opt, deopt in sorted(deoptcodes.items()):
+ fobj.write(f" [{opt}] = {deopt},\n")
+ fobj.write("};\n")
fobj.write("#endif /* OPCODE_TABLES */\n")
fobj.write("\n")
More information about the Python-checkins
mailing list