https://github.com/python/cpython/commit/d27acd4461ee603bcf6f4a81ca6afccc9fc... commit: d27acd4461ee603bcf6f4a81ca6afccc9fc87331 branch: main author: Mark Shannon <mark@hotpy.org> committer: markshannon <mark@hotpy.org> date: 2023-10-31T10:09:54Z summary: GH-111485: Increment `next_instr` consistently at the start of the instruction. (GH-111486) files: M Include/internal/pycore_opcode_metadata.h M Lib/test/test_generated_cases.py M Lib/test/test_monitoring.py M Python/bytecodes.c M Python/ceval.c M Python/ceval_macros.h M Python/executor_cases.c.h M Python/generated_cases.c.h M Tools/cases_generator/analysis.py M Tools/cases_generator/instructions.py M Tools/cases_generator/stacking.py diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e2ed9bad7833a..27948749d58eb 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1356,8 +1356,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_GUARD_BOTH_UNICODE] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, @@ -1434,7 +1434,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG }, + [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_ZERO_SUPER_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, @@ -1495,7 +1495,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_ITER_JUMP_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [_IS_ITER_EXHAUSTED_LIST] = { true, INSTR_FMT_IX, 0 }, @@ -1533,7 +1533,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_CHECK_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -2047,6 +2047,7 @@ extern const uint8_t _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { [TO_BOOL] = 3, + [BINARY_OP_INPLACE_ADD_UNICODE] = 1, [BINARY_SUBSCR] = 1, [STORE_SUBSCR] = 1, [SEND] = 1, diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 790e6b1b1b91e..475d7492040b7 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -138,6 +138,9 @@ def test_inst_no_args(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); spam(); DISPATCH(); } @@ -152,6 +155,9 @@ def test_inst_one_pop(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *value; value = stack_pointer[-1]; spam(); @@ -169,6 +175,9 @@ def test_inst_one_push(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *res; spam(); STACK_GROW(1); @@ -186,6 +195,9 @@ def test_inst_one_push_one_pop(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -204,6 +216,9 @@ def test_binary_op(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *right; PyObject *left; PyObject *res; @@ -225,6 +240,9 @@ def test_overlap(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *right; PyObject *left; PyObject *result; @@ -249,6 +267,9 @@ def test_predictions_and_eval_breaker(self): """ output = """ TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); PREDICTED(OP1); static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); PyObject *arg; @@ -259,6 +280,9 @@ def test_predictions_and_eval_breaker(self): } TARGET(OP3) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP3); PyObject *arg; PyObject *res; arg = stack_pointer[-1]; @@ -278,6 +302,9 @@ def test_error_if_plain(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); if (cond) goto label; DISPATCH(); } @@ -292,6 +319,9 @@ def test_error_if_plain_with_comment(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); if (cond) goto label; DISPATCH(); } @@ -306,6 +336,9 @@ def test_error_if_pop(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *right; PyObject *left; PyObject *res; @@ -326,12 +359,14 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(OP); PyObject *value; value = stack_pointer[-1]; - uint16_t counter = read_u16(&next_instr[0].cache); - uint32_t extra = read_u32(&next_instr[1].cache); + uint16_t counter = read_u16(&this_instr[1].cache); + uint32_t extra = read_u32(&this_instr[2].cache); STACK_SHRINK(1); - next_instr += 3; DISPATCH(); } """ @@ -345,6 +380,9 @@ def test_suppress_dispatch(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); goto somewhere; } """ @@ -366,18 +404,24 @@ def test_macro_instruction(self): """ output = """ TARGET(OP1) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(OP1); PyObject *right; PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - uint16_t counter = read_u16(&next_instr[0].cache); + uint16_t counter = read_u16(&this_instr[1].cache); op1(left, right); - next_instr += 1; DISPATCH(); } TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP); PREDICTED(OP); + _Py_CODEUNIT *this_instr = next_instr - 6; static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); PyObject *right; PyObject *left; @@ -387,22 +431,24 @@ def test_macro_instruction(self): right = stack_pointer[-1]; left = stack_pointer[-2]; { - uint16_t counter = read_u16(&next_instr[0].cache); + uint16_t counter = read_u16(&this_instr[1].cache); op1(left, right); } // OP2 arg2 = stack_pointer[-3]; { - uint32_t extra = read_u32(&next_instr[3].cache); + uint32_t extra = read_u32(&this_instr[4].cache); res = op2(arg2, left, right); } STACK_SHRINK(2); stack_pointer[-1] = res; - next_instr += 5; DISPATCH(); } TARGET(OP3) { + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(OP3); PyObject *right; PyObject *left; PyObject *arg2; @@ -413,7 +459,6 @@ def test_macro_instruction(self): res = op3(arg2, left, right); STACK_SHRINK(2); stack_pointer[-1] = res; - next_instr += 5; DISPATCH(); } """ @@ -427,6 +472,9 @@ def test_array_input(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *above; PyObject **values; PyObject *below; @@ -449,6 +497,9 @@ def test_array_output(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *below; PyObject **values; PyObject *above; @@ -470,6 +521,9 @@ def test_array_input_output(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject **values; PyObject *above; values = stack_pointer - oparg; @@ -489,6 +543,9 @@ def test_array_error_if(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject **values; PyObject *extra; values = stack_pointer - oparg; @@ -509,6 +566,9 @@ def test_cond_effect(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); PyObject *cc; PyObject *input = NULL; PyObject *aa; @@ -541,6 +601,9 @@ def test_macro_cond_effect(self): """ output = """ TARGET(M) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); PyObject *right; PyObject *middle; PyObject *left; @@ -580,6 +643,9 @@ def test_macro_push_push(self): """ output = """ TARGET(M) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); PyObject *val1; PyObject *val2; // A @@ -609,6 +675,9 @@ def test_override_inst(self): """ output = """ TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); ham(); DISPATCH(); } @@ -627,6 +696,9 @@ def test_override_op(self): """ output = """ TARGET(M) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); ham(); DISPATCH(); } diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 3b8ecb765c23f..d12cd9637dd96 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1378,15 +1378,16 @@ def func(): x = 4 else: x = 6 + 7 self.check_events(func, recorders = JUMP_AND_BRANCH_RECORDERS, expected = [ ('branch', 'func', 2, 2), - ('branch', 'func', 3, 4), + ('branch', 'func', 3, 6), ('jump', 'func', 6, 2), ('branch', 'func', 2, 2), - ('branch', 'func', 3, 3), + ('branch', 'func', 3, 4), ('jump', 'func', 4, 2), - ('branch', 'func', 2, 2)]) + ('branch', 'func', 2, 7)]) self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [ ('line', 'get_events', 10), @@ -1394,17 +1395,18 @@ def func(): ('line', 'func', 2), ('branch', 'func', 2, 2), ('line', 'func', 3), - ('branch', 'func', 3, 4), + ('branch', 'func', 3, 6), ('line', 'func', 6), ('jump', 'func', 6, 2), ('line', 'func', 2), ('branch', 'func', 2, 2), ('line', 'func', 3), - ('branch', 'func', 3, 3), + ('branch', 'func', 3, 4), ('line', 'func', 4), ('jump', 'func', 4, 2), ('line', 'func', 2), - ('branch', 'func', 2, 2), + ('branch', 'func', 2, 7), + ('line', 'func', 7), ('line', 'get_events', 11)]) def test_except_star(self): @@ -1434,7 +1436,7 @@ def func(): ('line', 'meth', 1), ('jump', 'func', 5, 5), ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=122]'), + ('branch', 'func', '[offset=120]', '[offset=124]'), ('line', 'get_events', 11)]) self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ @@ -1450,7 +1452,7 @@ def func(): ('return', None), ('jump', 'func', 5, 5), ('jump', 'func', 5, '[offset=114]'), - ('branch', 'func', '[offset=120]', '[offset=122]'), + ('branch', 'func', '[offset=120]', '[offset=124]'), ('return', None), ('line', 'get_events', 11)]) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5b1d70b303060..d33ee3265d254 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -80,6 +80,7 @@ dummy_func( pop_1_error: // Dummy locals. PyObject *dummy; + _Py_CODEUNIT *this_instr; PyObject *attr; PyObject *attrs; PyObject *bottom; @@ -144,13 +145,13 @@ dummy_func( if (code_version != global_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); ERROR_IF(err, error); - next_instr--; + next_instr = this_instr; } else { if (oparg < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - next_instr[-1].op.code = RESUME_CHECK; + this_instr->op.code = RESUME_CHECK; } } @@ -172,7 +173,7 @@ dummy_func( if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } - next_instr--; + next_instr = this_instr; } else { if (oparg < 2) { @@ -180,12 +181,12 @@ dummy_func( } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, next_instr-1); + tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); ERROR_IF(err, error); - if (frame->instr_ptr != next_instr-1) { + if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ - next_instr = frame->instr_ptr; + next_instr = this_instr; DISPATCH(); } } @@ -261,11 +262,12 @@ dummy_func( macro(END_FOR) = POP_TOP + POP_TOP; inst(INSTRUMENTED_END_FOR, (receiver, value --)) { + TIER_ONE_ONLY /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + if (monitor_stop_iteration(tstate, frame, this_instr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -278,9 +280,10 @@ dummy_func( } inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) { + TIER_ONE_ONLY if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + if (monitor_stop_iteration(tstate, frame, this_instr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -310,14 +313,13 @@ dummy_func( inst(TO_BOOL, (unused/1, unused/2, value -- res)) { #if ENABLE_SPECIALIZATION - _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ int err = PyObject_IsTrue(value); DECREF_INPUTS(); @@ -491,10 +493,9 @@ dummy_func( // So the inputs are the same as for all BINARY_OP // specializations, but there is no output. // At the end we just skip over the STORE_FAST. - op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) { - _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; - assert(true_next.op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(true_next.op.arg); + op(_BINARY_OP_INPLACE_ADD_UNICODE, (unused/1, left, right --)) { + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. @@ -514,7 +515,8 @@ dummy_func( _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); ERROR_IF(*target_local == NULL, error); // The STORE_FAST is already done. - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_OP + 1); + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); } macro(BINARY_OP_INPLACE_ADD_UNICODE) = @@ -530,14 +532,13 @@ dummy_func( inst(BINARY_SUBSCR, (unused/1, container, sub -- res)) { #if ENABLE_SPECIALIZATION - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); DECREF_INPUTS(); @@ -656,8 +657,7 @@ dummy_func( STACK_SHRINK(2); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR; DISPATCH_INLINED(new_frame); } @@ -679,14 +679,13 @@ dummy_func( inst(STORE_SUBSCR, (unused/1, v, container, sub -- )) { #if ENABLE_SPECIALIZATION - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -754,7 +753,7 @@ dummy_func( case 0: if (do_raise(tstate, exc, cause)) { assert(oparg == 0); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } break; @@ -808,7 +807,7 @@ dummy_func( inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); + frame, this_instr, retval); if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); @@ -832,7 +831,7 @@ dummy_func( PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); + frame, this_instr, retval); if (err) goto error; Py_INCREF(retval); assert(EMPTY()); @@ -958,14 +957,13 @@ dummy_func( inst(SEND, (unused/1, receiver, v -- receiver, retval)) { #if ENABLE_SPECIALIZATION - _PySendCache *cache = (_PySendCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && @@ -979,8 +977,7 @@ dummy_func( gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } @@ -993,7 +990,7 @@ dummy_func( if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, this_instr); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -1018,8 +1015,7 @@ dummy_func( gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } @@ -1033,7 +1029,7 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, retval); + frame, this_instr, retval); if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -1093,7 +1089,7 @@ dummy_func( assert(exc && PyExceptionInstance_Check(exc)); Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } @@ -1105,7 +1101,7 @@ dummy_func( else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } } @@ -1120,7 +1116,7 @@ dummy_func( } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } } @@ -1184,14 +1180,13 @@ dummy_func( inst(UNPACK_SEQUENCE, (unused/1, seq -- unused[oparg])) { #if ENABLE_SPECIALIZATION - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); @@ -1247,15 +1242,14 @@ dummy_func( inst(STORE_ATTR, (unused/1, unused/3, v, owner --)) { #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr--; + next_instr = this_instr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); @@ -1369,15 +1363,14 @@ dummy_func( inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) { #if ENABLE_SPECIALIZATION - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) @@ -1692,11 +1685,10 @@ dummy_func( ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error); } - inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/9, unused, unused, unused -- unused, unused if (oparg & 1))) { - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; + inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) { // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } @@ -1706,24 +1698,24 @@ dummy_func( }; inst(LOAD_SUPER_ATTR, (unused/1, global_super, class, self -- attr, null if (oparg & 1))) { + TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - int load_method = oparg & 1; #if ENABLE_SPECIALIZATION - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, global_super, arg); + frame, this_instr, global_super, arg); ERROR_IF(err, error); } @@ -1736,12 +1728,12 @@ dummy_func( if (super == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, global_super, arg); + frame, this_instr, global_super, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, global_super, arg); + frame, this_instr, global_super, arg); if (err < 0) { Py_CLEAR(super); } @@ -1819,15 +1811,14 @@ dummy_func( inst(LOAD_ATTR, (unused/9, owner -- attr, self_or_null if (oparg & 1))) { #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { @@ -2018,8 +2009,7 @@ dummy_func( // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2046,8 +2036,7 @@ dummy_func( STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } @@ -2142,14 +2131,13 @@ dummy_func( inst(COMPARE_OP, (unused/1, left, right -- res)) { #if ENABLE_SPECIALIZATION - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); @@ -2277,24 +2265,23 @@ dummy_func( inst(JUMP_BACKWARD, (--)) { CHECK_EVAL_BREAKER(); - _Py_CODEUNIT *here = next_instr - 1; assert(oparg <= INSTR_OFFSET()); JUMPBY(1-oparg); #if ENABLE_SPECIALIZATION - here[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); - if (here[1].cache > tstate->interp->optimizer_backedge_threshold && + this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); + if (this_instr[1].cache > tstate->interp->optimizer_backedge_threshold && // Double-check that the opcode isn't instrumented or something: - here->op.code == JUMP_BACKWARD) + this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); + int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); ERROR_IF(optimized < 0, error); if (optimized) { // Rewind and enter the executor: - assert(here->op.code == ENTER_EXECUTOR); - next_instr = here; + assert(this_instr->op.code == ENTER_EXECUTOR); + next_instr = this_instr; } - here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); } #endif /* ENABLE_SPECIALIZATION */ } @@ -2331,7 +2318,7 @@ dummy_func( assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } @@ -2340,7 +2327,7 @@ dummy_func( assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } @@ -2456,14 +2443,13 @@ dummy_func( inst(FOR_ITER, (unused/1, iter -- iter, next)) { #if ENABLE_SPECIALIZATION - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2472,15 +2458,14 @@ dummy_func( if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -2488,32 +2473,31 @@ dummy_func( // Common case: no jump, leave it to the code generator } - inst(INSTRUMENTED_FOR_ITER, ( -- )) { - _Py_CODEUNIT *here = frame->instr_ptr; + inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { _Py_CODEUNIT *target; PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next != NULL) { PUSH(next); - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; + target = next_instr; } else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, here); + monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); /* Skip END_FOR */ - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + target = next_instr + oparg + 1; } - INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); } op(_ITER_CHECK_LIST, (iter -- iter)) { @@ -2532,7 +2516,6 @@ dummy_func( } Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -2588,7 +2571,6 @@ dummy_func( } Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -2640,7 +2622,6 @@ dummy_func( if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); // Jump over END_FOR instruction. JUMPBY(oparg + 1); DISPATCH(); @@ -2682,10 +2663,9 @@ dummy_func( gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - this_instr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); DISPATCH_INLINED(gen_frame); } @@ -2912,7 +2892,7 @@ dummy_func( unused/2 + _LOAD_ATTR_METHOD_LAZY_DICT; - inst(INSTRUMENTED_CALL, ( -- )) { + inst(INSTRUMENTED_CALL, (unused/3 -- )) { int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 2); @@ -2920,10 +2900,9 @@ dummy_func( &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, function, arg); + frame, this_instr, function, arg); ERROR_IF(err, error); - _PyCallCache *cache = (_PyCallCache *)next_instr; - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(CALL); } @@ -2959,14 +2938,13 @@ dummy_func( total_args++; } #if ENABLE_SPECIALIZATION - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_Call(callable, next_instr, total_args); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { args--; @@ -2996,8 +2974,7 @@ dummy_func( if (new_frame == NULL) { goto error; } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -3012,12 +2989,12 @@ dummy_func( if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + frame, this_instr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); + frame, this_instr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -3151,8 +3128,7 @@ dummy_func( } // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -3200,7 +3176,7 @@ dummy_func( * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - _PyCallCache *cache = (_PyCallCache *)next_instr; + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; DEOPT_IF(null != NULL); DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; @@ -3229,8 +3205,7 @@ dummy_func( for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3426,9 +3401,9 @@ dummy_func( Py_DECREF(self); Py_DECREF(callable); STACK_SHRINK(3); - // CALL + POP_TOP - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); - assert(next_instr[-1].op.code == POP_TOP); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); DISPATCH(); } @@ -3558,7 +3533,7 @@ dummy_func( : PEEK(total_args + 1); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr - 1, function, arg); + frame, this_instr, function, arg); ERROR_IF(err, error); GO_TO_INSTRUCTION(CALL_KW); } @@ -3600,7 +3575,7 @@ dummy_func( if (new_frame == NULL) { goto error; } - assert(next_instr - frame->instr_ptr == 1); + assert(next_instr - this_instr == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -3615,12 +3590,12 @@ dummy_func( if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + frame, this_instr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); + frame, this_instr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -3663,18 +3638,18 @@ dummy_func( PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, func, arg); + frame, this_instr, func, arg); if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, func, arg); + frame, this_instr, func, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, func, arg); + frame, this_instr, func, arg); if (err < 0) { Py_CLEAR(result); } @@ -3697,7 +3672,7 @@ dummy_func( if (new_frame == NULL) { goto error; } - assert(next_instr - frame->instr_ptr == 1); + assert(next_instr - this_instr == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -3818,14 +3793,13 @@ dummy_func( inst(BINARY_OP, (unused/1, lhs, rhs -- res)) { #if ENABLE_SPECIALIZATION - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); @@ -3842,12 +3816,11 @@ dummy_func( inst(INSTRUMENTED_INSTRUCTION, ( -- )) { int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, next_instr-1); + tstate, frame, this_instr); ERROR_IF(next_opcode < 0, error); - next_instr--; + next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; @@ -3855,43 +3828,38 @@ dummy_func( } inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) { - _Py_CODEUNIT *here = frame->instr_ptr; - INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) { - _Py_CODEUNIT *here = frame->instr_ptr; CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(this_instr, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); } inst(INSTRUMENTED_POP_JUMP_IF_TRUE, (unused/1 -- )) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_FALSE, (unused/1 -- )) { PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NONE, (unused/1 -- )) { PyObject *value = POP(); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsNone(value); int offset; if (flag) { @@ -3902,14 +3870,13 @@ dummy_func( offset = 0; } #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, (unused/1 -- )) { PyObject *value = POP(); - _Py_CODEUNIT *here = frame->instr_ptr; int offset; int nflag = Py_IsNone(value); if (nflag) { @@ -3920,9 +3887,9 @@ dummy_func( offset = oparg; } #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | !nflag; + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); } inst(EXTENDED_ARG, ( -- )) { @@ -3969,7 +3936,7 @@ dummy_func( op(_SAVE_RETURN_OFFSET, (--)) { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; diff --git a/Python/ceval.c b/Python/ceval.c index 6f8584c15b7bd..bada365011226 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -867,7 +867,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int monitor_raise(tstate, frame, next_instr-1); exception_unwind: { - /* We can't use frame->f_lasti here, as RERAISE may have set it */ + /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; int level, handler, lasti; if (get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti) == 0) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 544e8ef8fa8c0..b11240b1faed4 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -60,25 +60,21 @@ #endif #ifdef Py_STATS -#define INSTRUCTION_START(op) \ +#define INSTRUCTION_STATS(op) \ do { \ - frame->instr_ptr = next_instr++; \ OPCODE_EXE_INC(op); \ if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \ lastopcode = op; \ } while (0) #else -#define INSTRUCTION_START(op) \ - do { \ - frame->instr_ptr = next_instr++; \ - } while(0) +#define INSTRUCTION_STATS(op) ((void)0) #endif #if USE_COMPUTED_GOTOS -# define TARGET(op) TARGET_##op: INSTRUCTION_START(op); +# define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] #else -# define TARGET(op) case op: TARGET_##op: INSTRUCTION_START(op); +# define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode #endif diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9f37cd75b47ef..24de9eadf5904 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -119,14 +119,13 @@ PyObject *res; value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ int err = PyObject_IsTrue(value); Py_DECREF(value); @@ -380,14 +379,13 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); Py_DECREF(container); @@ -574,14 +572,13 @@ container = stack_pointer[-2]; v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -900,14 +897,13 @@ PyObject *seq; seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); @@ -989,15 +985,14 @@ owner = stack_pointer[-1]; v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr--; + next_instr = this_instr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); @@ -1133,15 +1128,14 @@ PyObject *res; PyObject *null = NULL; #if ENABLE_SPECIALIZATION - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) @@ -1624,15 +1618,14 @@ PyObject *self_or_null = NULL; owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { @@ -1887,14 +1880,13 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); @@ -3209,14 +3201,13 @@ rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); @@ -3275,7 +3266,7 @@ case _SAVE_RETURN_OFFSET: { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index fb4506e68765e..40c8cd8798c93 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4,11 +4,18 @@ // Do not edit! TARGET(NOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); DISPATCH(); } TARGET(RESUME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); + _Py_CODEUNIT *this_instr = next_instr - 1; static_assert(0 == 0, "incorrect cache size"); TIER_ONE_ONLY assert(frame == tstate->current_frame); @@ -20,18 +27,21 @@ if (code_version != global_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); if (err) goto error; - next_instr--; + next_instr = this_instr; } else { if (oparg < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - next_instr[-1].op.code = RESUME_CHECK; + this_instr->op.code = RESUME_CHECK; } DISPATCH(); } TARGET(RESUME_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); #if defined(__EMSCRIPTEN__) DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; @@ -44,13 +54,16 @@ } TARGET(INSTRUMENTED_RESUME) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; if (code_version != global_version) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } - next_instr--; + next_instr = this_instr; } else { if (oparg < 2) { @@ -58,12 +71,12 @@ } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, next_instr-1); + tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; - if (frame->instr_ptr != next_instr-1) { + if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ - next_instr = frame->instr_ptr; + next_instr = this_instr; DISPATCH(); } } @@ -71,6 +84,9 @@ } TARGET(LOAD_FAST_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); PyObject *value; value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; @@ -81,6 +97,9 @@ } TARGET(LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); PyObject *value; value = GETLOCAL(oparg); assert(value != NULL); @@ -91,6 +110,9 @@ } TARGET(LOAD_FAST_AND_CLEAR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); PyObject *value; value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value @@ -101,6 +123,9 @@ } TARGET(LOAD_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); PyObject *value1; PyObject *value2; uint32_t oparg1 = oparg >> 4; @@ -116,6 +141,9 @@ } TARGET(LOAD_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); PyObject *value; value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); @@ -125,6 +153,9 @@ } TARGET(STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); PyObject *value; value = stack_pointer[-1]; SETLOCAL(oparg, value); @@ -133,6 +164,9 @@ } TARGET(STORE_FAST_LOAD_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); PyObject *value1; PyObject *value2; value1 = stack_pointer[-1]; @@ -146,6 +180,9 @@ } TARGET(STORE_FAST_STORE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); PyObject *value1; PyObject *value2; value1 = stack_pointer[-1]; @@ -159,6 +196,9 @@ } TARGET(POP_TOP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); PyObject *value; value = stack_pointer[-1]; Py_DECREF(value); @@ -167,6 +207,9 @@ } TARGET(PUSH_NULL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); PyObject *res; res = NULL; STACK_GROW(1); @@ -175,6 +218,9 @@ } TARGET(END_FOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); PyObject *value; // POP_TOP value = stack_pointer[-1]; @@ -191,15 +237,19 @@ } TARGET(INSTRUMENTED_END_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); PyObject *value; PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; + TIER_ONE_ONLY /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + if (monitor_stop_iteration(tstate, frame, this_instr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -211,6 +261,9 @@ } TARGET(END_SEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); PyObject *value; PyObject *receiver; value = stack_pointer[-1]; @@ -222,13 +275,17 @@ } TARGET(INSTRUMENTED_END_SEND) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); PyObject *value; PyObject *receiver; value = stack_pointer[-1]; receiver = stack_pointer[-2]; + TIER_ONE_ONLY if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); - if (monitor_stop_iteration(tstate, frame, next_instr-1)) { + if (monitor_stop_iteration(tstate, frame, this_instr)) { goto error; } PyErr_SetRaisedException(NULL); @@ -240,6 +297,9 @@ } TARGET(UNARY_NEGATIVE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -251,6 +311,9 @@ } TARGET(UNARY_NOT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -261,40 +324,47 @@ } TARGET(TO_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); PREDICTED(TO_BOOL); + _Py_CODEUNIT *this_instr = next_instr - 4; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ int err = PyObject_IsTrue(value); Py_DECREF(value); if (err < 0) goto pop_1_error; res = err ? Py_True : Py_False; stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_BOOL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); PyObject *value; value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_INT) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -309,11 +379,13 @@ res = Py_True; } stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_LIST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -322,11 +394,13 @@ res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_NONE) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -335,11 +409,13 @@ STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_STR) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -355,15 +431,17 @@ res = Py_True; } stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(TO_BOOL_ALWAYS_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); PyObject *value; PyObject *res; value = stack_pointer[-1]; - uint32_t version = read_u32(&next_instr[1].cache); + uint32_t version = read_u32(&this_instr[2].cache); // This one is a bit weird, because we expect *some* failures: assert(version); DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); @@ -371,11 +449,13 @@ Py_DECREF(value); res = Py_True; stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(UNARY_INVERT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -387,6 +467,9 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); PyObject *right; PyObject *left; PyObject *res; @@ -407,11 +490,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); PyObject *right; PyObject *left; PyObject *res; @@ -432,11 +517,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); PyObject *right; PyObject *left; PyObject *res; @@ -457,11 +544,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -482,11 +571,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -507,11 +598,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -532,11 +625,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_UNICODE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); PyObject *right; PyObject *left; PyObject *res; @@ -557,11 +652,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); PyObject *right; PyObject *left; // _GUARD_BOTH_UNICODE @@ -573,9 +670,8 @@ } // _BINARY_OP_INPLACE_ADD_UNICODE { - _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; - assert(true_next.op.code == STORE_FAST); - PyObject **target_local = &GETLOCAL(true_next.op.arg); + assert(next_instr->op.code == STORE_FAST); + PyObject **target_local = &GETLOCAL(next_instr->op.arg); DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. @@ -595,14 +691,19 @@ _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (*target_local == NULL) goto pop_2_error; // The STORE_FAST is already done. - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_OP + 1); + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); } STACK_SHRINK(2); DISPATCH(); } TARGET(BINARY_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR); PREDICTED(BINARY_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; @@ -610,14 +711,13 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); Py_DECREF(container); @@ -625,11 +725,13 @@ if (res == NULL) goto pop_2_error; STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); PyObject *stop; PyObject *start; PyObject *container; @@ -655,6 +757,9 @@ } TARGET(STORE_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); PyObject *stop; PyObject *start; PyObject *container; @@ -680,6 +785,9 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); PyObject *sub; PyObject *list; PyObject *res; @@ -700,11 +808,13 @@ Py_DECREF(list); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_SUBSCR_STR_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); PyObject *sub; PyObject *str; PyObject *res; @@ -724,11 +834,13 @@ Py_DECREF(str); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_SUBSCR_TUPLE_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); PyObject *sub; PyObject *tuple; PyObject *res; @@ -749,11 +861,13 @@ Py_DECREF(tuple); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_DICT); PyObject *sub; PyObject *dict; PyObject *res; @@ -775,11 +889,13 @@ Py_DECREF(sub); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(BINARY_SUBSCR_GETITEM) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); PyObject *sub; PyObject *container; sub = stack_pointer[-1]; @@ -803,13 +919,15 @@ STACK_SHRINK(2); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; - SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_BINARY_SUBSCR; DISPATCH_INLINED(new_frame); } TARGET(LIST_APPEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); PyObject *v; PyObject *list; v = stack_pointer[-1]; @@ -820,6 +938,9 @@ } TARGET(SET_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); PyObject *v; PyObject *set; v = stack_pointer[-1]; @@ -832,7 +953,11 @@ } TARGET(STORE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED(STORE_SUBSCR); + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; @@ -841,14 +966,13 @@ container = stack_pointer[-2]; v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); @@ -857,11 +981,13 @@ Py_DECREF(sub); if (err) goto pop_3_error; STACK_SHRINK(3); - next_instr += 1; DISPATCH(); } TARGET(STORE_SUBSCR_LIST_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); PyObject *sub; PyObject *list; PyObject *value; @@ -885,11 +1011,13 @@ _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); STACK_SHRINK(3); - next_instr += 1; DISPATCH(); } TARGET(STORE_SUBSCR_DICT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); PyObject *sub; PyObject *dict; PyObject *value; @@ -902,11 +1030,13 @@ Py_DECREF(dict); if (err) goto pop_3_error; STACK_SHRINK(3); - next_instr += 1; DISPATCH(); } TARGET(DELETE_SUBSCR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); PyObject *sub; PyObject *container; sub = stack_pointer[-1]; @@ -921,6 +1051,9 @@ } TARGET(CALL_INTRINSIC_1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -933,6 +1066,9 @@ } TARGET(CALL_INTRINSIC_2) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); PyObject *value1; PyObject *value2; PyObject *res; @@ -949,6 +1085,9 @@ } TARGET(RAISE_VARARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; args = stack_pointer - oparg; PyObject *cause = NULL, *exc = NULL; @@ -962,7 +1101,7 @@ case 0: if (do_raise(tstate, exc, cause)) { assert(oparg == 0); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } break; @@ -975,6 +1114,9 @@ } TARGET(INTERPRETER_EXIT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); PyObject *retval; retval = stack_pointer[-1]; assert(frame == &entry_frame); @@ -987,6 +1129,9 @@ } TARGET(RETURN_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; STACK_SHRINK(1); @@ -1013,11 +1158,14 @@ } TARGET(INSTRUMENTED_RETURN_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); + frame, this_instr, retval); if (err) goto error; STACK_SHRINK(1); assert(EMPTY()); @@ -1034,6 +1182,9 @@ } TARGET(RETURN_CONST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_CONST); PyObject *value; PyObject *retval; // LOAD_CONST @@ -1068,10 +1219,13 @@ } TARGET(INSTRUMENTED_RETURN_CONST) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, next_instr-1, retval); + frame, this_instr, retval); if (err) goto error; Py_INCREF(retval); assert(EMPTY()); @@ -1088,6 +1242,9 @@ } TARGET(GET_AITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); PyObject *obj; PyObject *iter; obj = stack_pointer[-1]; @@ -1126,6 +1283,9 @@ } TARGET(GET_ANEXT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); PyObject *aiter; PyObject *awaitable; aiter = stack_pointer[-1]; @@ -1177,6 +1337,9 @@ } TARGET(GET_AWAITABLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; @@ -1208,7 +1371,11 @@ } TARGET(SEND) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); PREDICTED(SEND); + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *v; PyObject *receiver; @@ -1216,14 +1383,13 @@ v = stack_pointer[-1]; receiver = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PySendCache *cache = (_PySendCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && @@ -1237,8 +1403,7 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } @@ -1251,7 +1416,7 @@ if (retval == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) ) { - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, this_instr); } if (_PyGen_FetchStopIterationValue(&retval) == 0) { assert(retval != NULL); @@ -1263,11 +1428,13 @@ } Py_DECREF(v); stack_pointer[-1] = retval; - next_instr += 1; DISPATCH(); } TARGET(SEND_GEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); PyObject *v; PyObject *receiver; v = stack_pointer[-1]; @@ -1283,13 +1450,15 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); - assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); DISPATCH_INLINED(gen_frame); } TARGET(INSTRUMENTED_YIELD_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; assert(frame != &entry_frame); @@ -1300,7 +1469,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, next_instr-1, retval); + frame, this_instr, retval); if (err) goto error; tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -1316,6 +1485,9 @@ } TARGET(YIELD_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! @@ -1341,6 +1513,9 @@ } TARGET(POP_EXCEPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); PyObject *exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; @@ -1350,6 +1525,9 @@ } TARGET(RERAISE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); PyObject *exc; PyObject **values; exc = stack_pointer[-1]; @@ -1370,11 +1548,14 @@ assert(exc && PyExceptionInstance_Check(exc)); Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } TARGET(END_ASYNC_FOR) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); PyObject *exc; PyObject *awaitable; exc = stack_pointer[-1]; @@ -1387,7 +1568,7 @@ else { Py_INCREF(exc); _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } STACK_SHRINK(2); @@ -1395,6 +1576,9 @@ } TARGET(CLEANUP_THROW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); PyObject *exc_value; PyObject *last_sent_val; PyObject *sub_iter; @@ -1414,7 +1598,7 @@ } else { _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, next_instr-1); + monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } STACK_SHRINK(1); @@ -1424,6 +1608,9 @@ } TARGET(LOAD_ASSERTION_ERROR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); PyObject *value; value = Py_NewRef(PyExc_AssertionError); STACK_GROW(1); @@ -1432,6 +1619,9 @@ } TARGET(LOAD_BUILD_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); PyObject *bc; if (PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc) < 0) goto error; if (bc == NULL) { @@ -1445,6 +1635,9 @@ } TARGET(STORE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); PyObject *v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -1467,6 +1660,9 @@ } TARGET(DELETE_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -1487,19 +1683,22 @@ } TARGET(UNPACK_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED(UNPACK_SEQUENCE); + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); @@ -1507,11 +1706,13 @@ if (res == 0) goto pop_1_error; STACK_SHRINK(1); STACK_GROW(oparg); - next_instr += 1; DISPATCH(); } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1525,11 +1726,13 @@ Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); - next_instr += 1; DISPATCH(); } TARGET(UNPACK_SEQUENCE_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1544,11 +1747,13 @@ Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); - next_instr += 1; DISPATCH(); } TARGET(UNPACK_SEQUENCE_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1563,11 +1768,13 @@ Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); - next_instr += 1; DISPATCH(); } TARGET(UNPACK_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); PyObject *seq; seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); @@ -1580,22 +1787,25 @@ } TARGET(STORE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); PREDICTED(STORE_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 5; static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *v; owner = stack_pointer[-1]; v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr--; + next_instr = this_instr; _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); @@ -1603,11 +1813,13 @@ Py_DECREF(owner); if (err) goto pop_2_error; STACK_SHRINK(2); - next_instr += 4; DISPATCH(); } TARGET(DELETE_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); PyObject *owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -1619,6 +1831,9 @@ } TARGET(STORE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); PyObject *v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -1630,6 +1845,9 @@ } TARGET(DELETE_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; err = PyDict_DelItem(GLOBALS(), name); @@ -1645,6 +1863,9 @@ } TARGET(LOAD_LOCALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); PyObject *locals; locals = LOCALS(); if (locals == NULL) { @@ -1659,6 +1880,9 @@ } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); PyObject *mod_or_class_dict; PyObject *v; mod_or_class_dict = stack_pointer[-1]; @@ -1692,6 +1916,9 @@ } TARGET(LOAD_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); PyObject *v; PyObject *mod_or_class_dict = LOCALS(); if (mod_or_class_dict == NULL) { @@ -1729,20 +1956,23 @@ } TARGET(LOAD_GLOBAL) { + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED(LOAD_GLOBAL); + _Py_CODEUNIT *this_instr = next_instr - 5; static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; #if ENABLE_SPECIALIZATION - _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) @@ -1783,16 +2013,18 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); PyObject *res; PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { - uint16_t version = read_u16(&next_instr[1].cache); + uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1800,7 +2032,7 @@ } // _LOAD_GLOBAL_MODULE { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; @@ -1813,16 +2045,18 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_BUILTIN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); PyObject *res; PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { - uint16_t version = read_u16(&next_instr[1].cache); + uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1830,7 +2064,7 @@ } // _GUARD_BUILTINS_VERSION { - uint16_t version = read_u16(&next_instr[2].cache); + uint16_t version = read_u16(&this_instr[3].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); @@ -1838,7 +2072,7 @@ } // _LOAD_GLOBAL_BUILTINS { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; @@ -1851,11 +2085,13 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 4; DISPATCH(); } TARGET(DELETE_FAST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); PyObject *v = GETLOCAL(oparg); if (v == NULL) goto unbound_local_error; SETLOCAL(oparg, NULL); @@ -1863,6 +2099,9 @@ } TARGET(MAKE_CELL) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); // "initial" is probably NULL but not if it's an arg (or set // via PyFrame_LocalsToFast() before MAKE_CELL has run). PyObject *initial = GETLOCAL(oparg); @@ -1875,6 +2114,9 @@ } TARGET(DELETE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); // Can't use ERROR_IF here. @@ -1889,6 +2131,9 @@ } TARGET(LOAD_FROM_DICT_OR_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); PyObject *class_dict; PyObject *value; class_dict = stack_pointer[-1]; @@ -1915,6 +2160,9 @@ } TARGET(LOAD_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); PyObject *value; PyObject *cell = GETLOCAL(oparg); value = PyCell_GET(cell); @@ -1929,6 +2177,9 @@ } TARGET(STORE_DEREF) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); PyObject *v; v = stack_pointer[-1]; PyObject *cell = GETLOCAL(oparg); @@ -1940,6 +2191,9 @@ } TARGET(COPY_FREE_VARS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); assert(PyFunction_Check(frame->f_funcobj)); @@ -1954,6 +2208,9 @@ } TARGET(BUILD_STRING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); PyObject **pieces; PyObject *str; pieces = stack_pointer - oparg; @@ -1969,6 +2226,9 @@ } TARGET(BUILD_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); PyObject **values; PyObject *tup; values = stack_pointer - oparg; @@ -1981,6 +2241,9 @@ } TARGET(BUILD_LIST) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); PyObject **values; PyObject *list; values = stack_pointer - oparg; @@ -1993,6 +2256,9 @@ } TARGET(LIST_EXTEND) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); PyObject *iterable; PyObject *list; iterable = stack_pointer[-1]; @@ -2017,6 +2283,9 @@ } TARGET(SET_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); PyObject *iterable; PyObject *set; iterable = stack_pointer[-1]; @@ -2029,6 +2298,9 @@ } TARGET(BUILD_SET) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); PyObject **values; PyObject *set; values = stack_pointer - oparg; @@ -2053,6 +2325,9 @@ } TARGET(BUILD_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); PyObject **values; PyObject *map; values = stack_pointer - oparg*2; @@ -2071,6 +2346,9 @@ } TARGET(SETUP_ANNOTATIONS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); int err; PyObject *ann_dict; if (LOCALS() == NULL) { @@ -2112,6 +2390,9 @@ } TARGET(BUILD_CONST_KEY_MAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_CONST_KEY_MAP); PyObject *keys; PyObject **values; PyObject *map; @@ -2137,6 +2418,9 @@ } TARGET(DICT_UPDATE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); PyObject *update; PyObject *dict; update = stack_pointer[-1]; @@ -2156,6 +2440,9 @@ } TARGET(DICT_MERGE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); PyObject *update; PyObject *dict; PyObject *callable; @@ -2173,6 +2460,9 @@ } TARGET(MAP_ADD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); PyObject *value; PyObject *key; PyObject *dict; @@ -2188,15 +2478,21 @@ } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); } TARGET(LOAD_SUPER_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED(LOAD_SUPER_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self; PyObject *class; @@ -2206,24 +2502,24 @@ self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; + TIER_ONE_ONLY PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - int load_method = oparg & 1; #if ENABLE_SPECIALIZATION - _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, global_super, arg); + frame, this_instr, global_super, arg); if (err) goto pop_3_error; } @@ -2236,12 +2532,12 @@ if (super == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, global_super, arg); + frame, this_instr, global_super, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, global_super, arg); + frame, this_instr, global_super, arg); if (err < 0) { Py_CLEAR(super); } @@ -2259,11 +2555,13 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 1; DISPATCH(); } TARGET(LOAD_SUPER_ATTR_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); PyObject *self; PyObject *class; PyObject *global_super; @@ -2283,11 +2581,13 @@ if (attr == NULL) goto pop_3_error; STACK_SHRINK(2); stack_pointer[-1] = attr; - next_instr += 1; DISPATCH(); } TARGET(LOAD_SUPER_ATTR_METHOD) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); PyObject *self; PyObject *class; PyObject *global_super; @@ -2320,27 +2620,29 @@ STACK_SHRINK(1); stack_pointer[-2] = attr; stack_pointer[-1] = self_or_null; - next_instr += 1; DISPATCH(); } TARGET(LOAD_ATTR) { + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); PREDICTED(LOAD_ATTR); + _Py_CODEUNIT *this_instr = next_instr - 10; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyAttrCache *cache = (_PyAttrCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr--; + next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { @@ -2377,18 +2679,20 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); PyObject *owner; PyObject *attr; PyObject *null = NULL; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2402,7 +2706,7 @@ } // _LOAD_ATTR_INSTANCE_VALUE { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; DEOPT_IF(attr == NULL, LOAD_ATTR); @@ -2414,18 +2718,20 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); PyObject *owner; PyObject *attr; PyObject *null = NULL; // _CHECK_ATTR_MODULE owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -2433,7 +2739,7 @@ } // _LOAD_ATTR_MODULE { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); @@ -2448,18 +2754,20 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); PyObject *owner; PyObject *attr; PyObject *null = NULL; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -2475,7 +2783,7 @@ } // _LOAD_ATTR_WITH_HINT { - uint16_t hint = read_u16(&next_instr[3].cache); + uint16_t hint = read_u16(&this_instr[4].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); @@ -2499,25 +2807,27 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); PyObject *owner; PyObject *attr; PyObject *null = NULL; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); char *addr = (char *)owner + index; attr = *(PyObject **)addr; DEOPT_IF(attr == NULL, LOAD_ATTR); @@ -2529,25 +2839,27 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); PyObject *owner; PyObject *attr; PyObject *null = NULL; // _CHECK_ATTR_CLASS owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); assert(type_version != 0); DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_CLASS { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&this_instr[6].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = Py_NewRef(descr); @@ -2557,16 +2869,18 @@ STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t func_version = read_u32(&next_instr[3].cache); - PyObject *fget = read_obj(&next_instr[5].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *fget = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); @@ -2586,18 +2900,20 @@ // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); PyObject *owner; owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t func_version = read_u32(&next_instr[3].cache); - PyObject *getattribute = read_obj(&next_instr[5].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2619,19 +2935,21 @@ STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); - SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); - assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_LOAD_ATTR == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_LOAD_ATTR; DISPATCH_INLINED(new_frame); } TARGET(STORE_ATTR_INSTANCE_VALUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); PyObject *owner; PyObject *value; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2645,7 +2963,7 @@ // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); STAT_INC(STORE_ATTR, hit); PyDictValues *values = _PyDictOrValues_GetValues(dorv); @@ -2660,17 +2978,19 @@ Py_DECREF(owner); } STACK_SHRINK(2); - next_instr += 4; DISPATCH(); } TARGET(STORE_ATTR_WITH_HINT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); PyObject *owner; PyObject *value; owner = stack_pointer[-1]; value = stack_pointer[-2]; - uint32_t type_version = read_u32(&next_instr[1].cache); - uint16_t hint = read_u16(&next_instr[3].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); + uint16_t hint = read_u16(&this_instr[4].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2710,17 +3030,19 @@ dict->ma_version_tag = new_version; Py_DECREF(owner); STACK_SHRINK(2); - next_instr += 4; DISPATCH(); } TARGET(STORE_ATTR_SLOT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); PyObject *owner; PyObject *value; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); @@ -2728,7 +3050,7 @@ // _STORE_ATTR_SLOT value = stack_pointer[-2]; { - uint16_t index = read_u16(&next_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); char *addr = (char *)owner + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; @@ -2737,12 +3059,15 @@ Py_DECREF(owner); } STACK_SHRINK(2); - next_instr += 4; DISPATCH(); } TARGET(COMPARE_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); PREDICTED(COMPARE_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; @@ -2750,14 +3075,13 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); @@ -2772,11 +3096,13 @@ } STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(COMPARE_OP_FLOAT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -2795,11 +3121,13 @@ // It's always a bool, so we don't care about oparg & 16. STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(COMPARE_OP_INT) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); PyObject *right; PyObject *left; PyObject *res; @@ -2822,11 +3150,13 @@ // It's always a bool, so we don't care about oparg & 16. STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(COMPARE_OP_STR) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); PyObject *right; PyObject *left; PyObject *res; @@ -2846,11 +3176,13 @@ // It's always a bool, so we don't care about oparg & 16. STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(IS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); PyObject *right; PyObject *left; PyObject *b; @@ -2866,6 +3198,9 @@ } TARGET(CONTAINS_OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONTAINS_OP); PyObject *right; PyObject *left; PyObject *b; @@ -2882,6 +3217,9 @@ } TARGET(CHECK_EG_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); PyObject *match_type; PyObject *exc_value; PyObject *rest; @@ -2914,6 +3252,9 @@ } TARGET(CHECK_EXC_MATCH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); PyObject *right; PyObject *left; PyObject *b; @@ -2933,6 +3274,9 @@ } TARGET(IMPORT_NAME) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); PyObject *fromlist; PyObject *level; PyObject *res; @@ -2949,6 +3293,9 @@ } TARGET(IMPORT_FROM) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); PyObject *from; PyObject *res; from = stack_pointer[-1]; @@ -2961,36 +3308,44 @@ } TARGET(JUMP_FORWARD) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); JUMPBY(oparg); DISPATCH(); } TARGET(JUMP_BACKWARD) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD); CHECK_EVAL_BREAKER(); - _Py_CODEUNIT *here = next_instr - 1; assert(oparg <= INSTR_OFFSET()); JUMPBY(1-oparg); #if ENABLE_SPECIALIZATION - here[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); - if (here[1].cache > tstate->interp->optimizer_backedge_threshold && + this_instr[1].cache += (1 << OPTIMIZER_BITS_IN_COUNTER); + if (this_instr[1].cache > tstate->interp->optimizer_backedge_threshold && // Double-check that the opcode isn't instrumented or something: - here->op.code == JUMP_BACKWARD) + this_instr->op.code == JUMP_BACKWARD) { OPT_STAT_INC(attempts); - int optimized = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); + int optimized = _PyOptimizer_BackEdge(frame, this_instr, next_instr, stack_pointer); if (optimized < 0) goto error; if (optimized) { // Rewind and enter the executor: - assert(here->op.code == ENTER_EXECUTOR); - next_instr = here; + assert(this_instr->op.code == ENTER_EXECUTOR); + next_instr = this_instr; } - here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); + this_instr[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1); } #endif /* ENABLE_SPECIALIZATION */ DISPATCH(); } TARGET(ENTER_EXECUTOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); CHECK_EVAL_BREAKER(); PyCodeObject *code = _PyFrame_GetCode(frame); @@ -3009,34 +3364,41 @@ } TARGET(POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); PyObject *cond; cond = stack_pointer[-1]; assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); STACK_SHRINK(1); - next_instr += 1; DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); PyObject *cond; cond = stack_pointer[-1]; assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); STACK_SHRINK(1); - next_instr += 1; DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); PyObject *value; PyObject *b; PyObject *cond; @@ -3057,16 +3419,18 @@ assert(PyBool_Check(cond)); int flag = Py_IsTrue(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } STACK_SHRINK(1); - next_instr += 1; DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); PyObject *value; PyObject *b; PyObject *cond; @@ -3087,16 +3451,18 @@ assert(PyBool_Check(cond)); int flag = Py_IsFalse(cond); #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); } STACK_SHRINK(1); - next_instr += 1; DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -3107,6 +3473,9 @@ } TARGET(GET_LEN) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); PyObject *obj; PyObject *len_o; obj = stack_pointer[-1]; @@ -3121,6 +3490,9 @@ } TARGET(MATCH_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); PyObject *names; PyObject *type; PyObject *subject; @@ -3148,6 +3520,9 @@ } TARGET(MATCH_MAPPING) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); PyObject *subject; PyObject *res; subject = stack_pointer[-1]; @@ -3159,6 +3534,9 @@ } TARGET(MATCH_SEQUENCE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); PyObject *subject; PyObject *res; subject = stack_pointer[-1]; @@ -3170,6 +3548,9 @@ } TARGET(MATCH_KEYS) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); PyObject *keys; PyObject *subject; PyObject *values_or_none; @@ -3184,6 +3565,9 @@ } TARGET(GET_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; @@ -3196,6 +3580,9 @@ } TARGET(GET_YIELD_FROM_ITER) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); PyObject *iterable; PyObject *iter; iterable = stack_pointer[-1]; @@ -3228,20 +3615,23 @@ } TARGET(FOR_ITER) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); PREDICTED(FOR_ITER); + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; iter = stack_pointer[-1]; #if ENABLE_SPECIALIZATION - _PyForIterCache *cache = (_PyForIterCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -3250,15 +3640,14 @@ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, next_instr-1); + monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3266,40 +3655,44 @@ // Common case: no jump, leave it to the code generator STACK_GROW(1); stack_pointer[-1] = next; - next_instr += 1; DISPATCH(); } TARGET(INSTRUMENTED_FOR_ITER) { - _Py_CODEUNIT *here = frame->instr_ptr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); _Py_CODEUNIT *target; PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next != NULL) { PUSH(next); - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER; + target = next_instr; } else { if (_PyErr_Occurred(tstate)) { if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { goto error; } - monitor_raise(tstate, frame, here); + monitor_raise(tstate, frame, this_instr); _PyErr_Clear(tstate); } /* iterator ended normally */ - assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR || - next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR); + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); STACK_SHRINK(1); Py_DECREF(iter); /* Skip END_FOR */ - target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; + target = next_instr + oparg + 1; } - INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); + INSTRUMENTED_JUMP(this_instr, target, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(FOR_ITER_LIST) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); PyObject *iter; PyObject *next; // _ITER_CHECK_LIST @@ -3320,7 +3713,6 @@ } Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3337,11 +3729,13 @@ } STACK_GROW(1); stack_pointer[-1] = next; - next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_TUPLE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); PyObject *iter; PyObject *next; // _ITER_CHECK_TUPLE @@ -3362,7 +3756,6 @@ } Py_DECREF(iter); STACK_SHRINK(1); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); DISPATCH(); @@ -3379,11 +3772,13 @@ } STACK_GROW(1); stack_pointer[-1] = next; - next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_RANGE) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); PyObject *iter; PyObject *next; // _ITER_CHECK_RANGE @@ -3400,7 +3795,6 @@ if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); // Jump over END_FOR instruction. JUMPBY(oparg + 1); DISPATCH(); @@ -3419,11 +3813,13 @@ } STACK_GROW(1); stack_pointer[-1] = next; - next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_GEN) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); PyObject *iter; iter = stack_pointer[-1]; DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); @@ -3436,15 +3832,17 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_FOR_ITER == next_instr - this_instr); frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg); DISPATCH_INLINED(gen_frame); } TARGET(BEFORE_ASYNC_WITH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BEFORE_ASYNC_WITH); PyObject *mgr; PyObject *exit; PyObject *res; @@ -3485,6 +3883,9 @@ } TARGET(BEFORE_WITH) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BEFORE_WITH); PyObject *mgr; PyObject *exit; PyObject *res; @@ -3528,6 +3929,9 @@ } TARGET(WITH_EXCEPT_START) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); PyObject *val; PyObject *lasti; PyObject *exit_func; @@ -3566,6 +3970,9 @@ } TARGET(PUSH_EXC_INFO) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); PyObject *new_exc; PyObject *prev_exc; new_exc = stack_pointer[-1]; @@ -3585,13 +3992,16 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); PyObject *owner; PyObject *attr; PyObject *self; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -3604,14 +4014,14 @@ } // _GUARD_KEYS_VERSION { - uint32_t keys_version = read_u32(&next_instr[3].cache); + uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&this_instr[6].cache); assert(oparg & 1); /* Cached method object */ STAT_INC(LOAD_ATTR, hit); @@ -3623,25 +4033,27 @@ STACK_GROW(1); stack_pointer[-2] = attr; stack_pointer[-1] = self; - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); PyObject *owner; PyObject *attr; PyObject *self; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_NO_DICT { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&this_instr[6].cache); assert(oparg & 1); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3653,17 +4065,19 @@ STACK_GROW(1); stack_pointer[-2] = attr; stack_pointer[-1] = self; - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); PyObject *owner; PyObject *attr; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -3676,14 +4090,14 @@ } // _GUARD_KEYS_VERSION { - uint32_t keys_version = read_u32(&next_instr[3].cache); + uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3691,24 +4105,26 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); PyObject *owner; PyObject *attr; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); assert(Py_TYPE(owner)->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -3717,18 +4133,20 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; - next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); PyObject *owner; PyObject *attr; PyObject *self; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { - uint32_t type_version = read_u32(&next_instr[1].cache); + uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -3743,7 +4161,7 @@ } // _LOAD_ATTR_METHOD_LAZY_DICT { - PyObject *descr = read_obj(&next_instr[5].cache); + PyObject *descr = read_obj(&this_instr[6].cache); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -3754,11 +4172,13 @@ STACK_GROW(1); stack_pointer[-2] = attr; stack_pointer[-1] = self; - next_instr += 9; DISPATCH(); } TARGET(INSTRUMENTED_CALL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 2); @@ -3766,15 +4186,18 @@ &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, function, arg); + frame, this_instr, function, arg); if (err) goto error; - _PyCallCache *cache = (_PyCallCache *)next_instr; - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(CALL); } TARGET(CALL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); PREDICTED(CALL); + _Py_CODEUNIT *this_instr = next_instr - 4; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; @@ -3790,14 +4213,13 @@ total_args++; } #if ENABLE_SPECIALIZATION - _PyCallCache *cache = (_PyCallCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_Call(callable, next_instr, total_args); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { args--; @@ -3827,8 +4249,7 @@ if (new_frame == NULL) { goto error; } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } @@ -3843,12 +4264,12 @@ if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + frame, this_instr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); + frame, this_instr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -3863,12 +4284,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); PyObject *null; PyObject *callable; PyObject *self; @@ -3900,7 +4323,7 @@ self_or_null = self; callable = func; { - uint32_t func_version = read_u32(&next_instr[1].cache); + uint32_t func_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -3930,10 +4353,9 @@ } } // _SAVE_RETURN_OFFSET - next_instr += 3; { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; @@ -3964,6 +4386,9 @@ } TARGET(CALL_PY_EXACT_ARGS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); PyObject *self_or_null; PyObject *callable; PyObject **args; @@ -3976,7 +4401,7 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - uint32_t func_version = read_u32(&next_instr[1].cache); + uint32_t func_version = read_u32(&this_instr[2].cache); DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); @@ -4006,10 +4431,9 @@ } } // _SAVE_RETURN_OFFSET - next_instr += 3; { #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - frame->instr_ptr); + frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; @@ -4040,13 +4464,16 @@ } TARGET(CALL_PY_WITH_DEFAULTS) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); PyObject **args; PyObject *self_or_null; PyObject *callable; args = stack_pointer - oparg; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&next_instr[1].cache); + uint32_t func_version = read_u32(&this_instr[2].cache); DEOPT_IF(tstate->interp->eval_frame, CALL); int argcount = oparg; if (self_or_null != NULL) { @@ -4076,13 +4503,15 @@ } // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). STACK_SHRINK(oparg + 2); - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; DISPATCH_INLINED(new_frame); } TARGET(CALL_TYPE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4101,11 +4530,13 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(CALL_STR_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4125,12 +4556,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_TUPLE_1) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4150,12 +4583,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_ALLOC_AND_ENTER_INIT) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); PyObject **args; PyObject *null; PyObject *callable; @@ -4167,7 +4602,7 @@ * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) * 3. Pushes the frame for ``__init__`` to the frame stack * */ - _PyCallCache *cache = (_PyCallCache *)next_instr; + _PyCallCache *cache = (_PyCallCache *)&this_instr[1]; DEOPT_IF(null != NULL, CALL); DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; @@ -4196,8 +4631,7 @@ for (int i = 0; i < oparg; i++) { init_frame->localsplus[i+1] = args[i]; } - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - frame->instr_ptr); + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; STACK_SHRINK(oparg+2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4214,6 +4648,9 @@ } TARGET(EXIT_INIT_CHECK) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); PyObject *should_be_none; should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); @@ -4228,6 +4665,9 @@ } TARGET(CALL_BUILTIN_CLASS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4254,12 +4694,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BUILTIN_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4294,12 +4736,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BUILTIN_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4338,12 +4782,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4376,12 +4822,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_LEN) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4413,11 +4861,13 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(CALL_ISINSTANCE) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4451,11 +4901,13 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; DISPATCH(); } TARGET(CALL_LIST_APPEND) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); PyObject **args; PyObject *self; PyObject *callable; @@ -4474,13 +4926,16 @@ Py_DECREF(self); Py_DECREF(callable); STACK_SHRINK(3); - // CALL + POP_TOP - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); - assert(next_instr[-1].op.code == POP_TOP); + // Skip POP_TOP + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_O) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4518,12 +4973,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4559,12 +5016,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4601,12 +5060,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4641,12 +5102,14 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_CALL_KW) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); int is_meth = PEEK(oparg + 2) != NULL; int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 3); @@ -4654,13 +5117,17 @@ : PEEK(total_args + 1); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr - 1, function, arg); + frame, this_instr, function, arg); if (err) goto error; GO_TO_INSTRUCTION(CALL_KW); } TARGET(CALL_KW) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_KW); PREDICTED(CALL_KW); + _Py_CODEUNIT *this_instr = next_instr - 1; PyObject *kwnames; PyObject **args; PyObject *self_or_null; @@ -4706,7 +5173,7 @@ if (new_frame == NULL) { goto error; } - assert(next_instr - frame->instr_ptr == 1); + assert(next_instr - this_instr == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -4721,12 +5188,12 @@ if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, callable, arg); + frame, this_instr, callable, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, callable, arg); + frame, this_instr, callable, arg); if (err < 0) { Py_CLEAR(res); } @@ -4747,11 +5214,18 @@ } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } TARGET(CALL_FUNCTION_EX) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); PREDICTED(CALL_FUNCTION_EX); + _Py_CODEUNIT *this_instr = next_instr - 1; PyObject *kwargs = NULL; PyObject *callargs; PyObject *func; @@ -4781,18 +5255,18 @@ PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, - frame, next_instr-1, func, arg); + frame, this_instr, func, arg); if (err) goto error; result = PyObject_Call(func, callargs, kwargs); if (result == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, - frame, next_instr-1, func, arg); + frame, this_instr, func, arg); } else { int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, - frame, next_instr-1, func, arg); + frame, this_instr, func, arg); if (err < 0) { Py_CLEAR(result); } @@ -4815,7 +5289,7 @@ if (new_frame == NULL) { goto error; } - assert(next_instr - frame->instr_ptr == 1); + assert(next_instr - this_instr == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -4834,6 +5308,9 @@ } TARGET(MAKE_FUNCTION) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; @@ -4854,6 +5331,9 @@ } TARGET(SET_FUNCTION_ATTRIBUTE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); PyObject *func; PyObject *attr; func = stack_pointer[-1]; @@ -4888,6 +5368,9 @@ } TARGET(RETURN_GENERATOR) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4913,6 +5396,9 @@ } TARGET(BUILD_SLICE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); PyObject *step = NULL; PyObject *stop; PyObject *start; @@ -4932,6 +5418,9 @@ } TARGET(CONVERT_VALUE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); PyObject *value; PyObject *result; value = stack_pointer[-1]; @@ -4946,6 +5435,9 @@ } TARGET(FORMAT_SIMPLE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -4964,6 +5456,9 @@ } TARGET(FORMAT_WITH_SPEC) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); PyObject *fmt_spec; PyObject *value; PyObject *res; @@ -4979,6 +5474,9 @@ } TARGET(COPY) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); PyObject *bottom; PyObject *top; bottom = stack_pointer[-1 - (oparg-1)]; @@ -4990,7 +5488,11 @@ } TARGET(BINARY_OP) { + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); + _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *rhs; PyObject *lhs; @@ -4998,14 +5500,13 @@ rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - next_instr--; + if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); @@ -5016,11 +5517,13 @@ if (res == NULL) goto pop_2_error; STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 1; DISPATCH(); } TARGET(SWAP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); PyObject *top; PyObject *bottom; top = stack_pointer[-1]; @@ -5032,13 +5535,15 @@ } TARGET(INSTRUMENTED_INSTRUCTION) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, next_instr-1); + tstate, frame, this_instr); if (next_opcode < 0) goto error; - next_instr--; + next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - INCREMENT_ADAPTIVE_COUNTER(cache->counter); + INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; @@ -5046,49 +5551,57 @@ } TARGET(INSTRUMENTED_JUMP_FORWARD) { - _Py_CODEUNIT *here = frame->instr_ptr; - INSTRUMENTED_JUMP(here, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - _Py_CODEUNIT *here = frame->instr_ptr; + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); CHECK_EVAL_BREAKER(); - INSTRUMENTED_JUMP(here, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); + INSTRUMENTED_JUMP(this_instr, next_instr + 1 - oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsTrue(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - next_instr += 1; + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); PyObject *cond = POP(); assert(PyBool_Check(cond)); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsFalse(cond); int offset = flag * oparg; #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - next_instr += 1; + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); PyObject *value = POP(); - _Py_CODEUNIT *here = frame->instr_ptr; int flag = Py_IsNone(value); int offset; if (flag) { @@ -5099,16 +5612,17 @@ offset = 0; } #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | flag; + this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - next_instr += 1; + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); PyObject *value = POP(); - _Py_CODEUNIT *here = frame->instr_ptr; int offset; int nflag = Py_IsNone(value); if (nflag) { @@ -5119,14 +5633,16 @@ offset = oparg; } #if ENABLE_SPECIALIZATION - next_instr->cache = (next_instr->cache << 1) | !nflag; + this_instr[1].cache = (this_instr[1].cache << 1) | !nflag; #endif - INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - next_instr += 1; + INSTRUMENTED_JUMP(this_instr, next_instr + offset, PY_MONITORING_EVENT_BRANCH); DISPATCH(); } TARGET(EXTENDED_ARG) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -5135,11 +5651,17 @@ } TARGET(CACHE) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); assert(0 && "Executing a cache."); Py_UNREACHABLE(); } TARGET(RESERVED) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); } diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 53f715db6f24a..180bd01302ec1 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -432,7 +432,6 @@ def report_non_viable_uops(self, jsonfile: str) -> None: "LOAD_FAST_LOAD_FAST", "LOAD_CONST_LOAD_FAST", "STORE_FAST_STORE_FAST", - "_BINARY_OP_INPLACE_ADD_UNICODE", "POP_JUMP_IF_TRUE", "POP_JUMP_IF_FALSE", "_ITER_JUMP_LIST", diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index c6b551675e3e7..e9ed2a867ae2b 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -61,6 +61,7 @@ class Instruction: # Computed by constructor always_exits: str # If the block always exits, its last line; else "" has_deopt: bool + needs_this_instr: bool cache_offset: int cache_effects: list[parsing.CacheEffect] input_effects: list[StackEffect] @@ -87,6 +88,7 @@ def __init__(self, inst: parsing.InstDef): effect for effect in inst.inputs if isinstance(effect, parsing.CacheEffect) ] self.cache_offset = sum(c.size for c in self.cache_effects) + self.needs_this_instr = variable_used(self.inst, "this_instr") or any(c.name != UNUSED for c in self.cache_effects) self.input_effects = [ effect for effect in inst.inputs if isinstance(effect, StackEffect) ] @@ -164,7 +166,8 @@ def write_body( func = f"read_u{bits}" if tier == TIER_ONE: out.emit( - f"{typ}{ceffect.name} = {func}(&next_instr[{active.offset}].cache);" + f"{typ}{ceffect.name} = " + f"{func}(&this_instr[{active.offset + 1}].cache);" ) else: out.emit(f"{typ}{ceffect.name} = ({typ.strip()})operand;") diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 69a6c10d6199c..123e38c524f49 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -365,8 +365,17 @@ def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None: ] out.emit("") with out.block(f"TARGET({mac.name})"): + needs_this = any(part.instr.needs_this_instr for part in parts) + if needs_this and not mac.predicted: + out.emit(f"_Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;") + else: + out.emit(f"frame->instr_ptr = next_instr;") + out.emit(f"next_instr += {mac.cache_offset+1};") + out.emit(f"INSTRUCTION_STATS({mac.name});") if mac.predicted: out.emit(f"PREDICTED({mac.name});") + if needs_this: + out.emit(f"_Py_CODEUNIT *this_instr = next_instr - {mac.cache_offset+1};") out.static_assert_family_size(mac.name, mac.family, mac.cache_offset) try: next_instr_is_set = write_components( @@ -375,8 +384,6 @@ def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None: except AssertionError as err: raise AssertionError(f"Error writing macro {mac.name}") from err if not parts[-1].instr.always_exits: - if not next_instr_is_set and mac.cache_offset: - out.emit(f"next_instr += {mac.cache_offset};") if parts[-1].instr.check_eval_breaker: out.emit("CHECK_EVAL_BREAKER();") out.emit("DISPATCH();") @@ -450,8 +457,6 @@ def write_components( if mgr.instr.name == "_SAVE_RETURN_OFFSET": next_instr_is_set = True - if cache_offset: - out.emit(f"next_instr += {cache_offset};") if tier == TIER_ONE: assert_no_pokes(managers)