https://github.com/python/cpython/commit/b5ee0258bf5bb60a5a5a65c64717853e06b... commit: b5ee0258bf5bb60a5a5a65c64717853e06b64808 branch: main author: mpage <mpage@meta.com> committer: mpage <mpage@cs.stanford.edu> date: 2025-01-14T11:56:11-08:00 summary: gh-115999: Specialize `LOAD_ATTR` for instance and class receivers in free-threaded builds (#128164) Finish specialization for LOAD_ATTR in the free-threaded build by adding support for class and instance receivers. files: M Include/cpython/pystats.h M Include/internal/pycore_dict.h M Include/internal/pycore_opcode_metadata.h M Include/internal/pycore_uop_metadata.h M Lib/test/test_capi/test_type.py M Lib/test/test_descr.py M Lib/test/test_generated_cases.py M Lib/test/test_opcache.py M Objects/dictobject.c M Python/bytecodes.c M Python/executor_cases.c.h M Python/generated_cases.c.h M Python/optimizer_bytecodes.c M Python/optimizer_cases.c.h M Python/specialize.c M Tools/cases_generator/analyzer.py M Tools/cases_generator/generators_common.py M Tools/cases_generator/stack.py diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 29ef0c0e4d4e72..ee8885cda7b60d 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -31,7 +31,7 @@ #define PYSTATS_MAX_UOP_ID 512 -#define SPECIALIZATION_FAILURE_KINDS 36 +#define SPECIALIZATION_FAILURE_KINDS 37 /* Stats for determining who is calling PyEval_EvalFrame */ #define EVAL_CALL_TOTAL 0 diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 71927006d1cd48..74ac8f2148174c 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -114,6 +114,16 @@ extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); + +/* Look up a string key in an all unicode dict keys, assign the keys object a version, and + * store it in version. + * + * Returns DKIX_ERROR if key is not a string or if the keys object is not all + * strings. + * + * Returns DKIX_EMPTY if the key is not present. + */ +extern Py_ssize_t _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 5365e2a5c6b8cd..0c0a6145bdbb27 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1540,7 +1540,7 @@ int _PyOpcode_max_stack_effect(int opcode, int oparg, int *effect) { return 0; } case LOAD_ATTR_WITH_HINT: { - *effect = Py_MAX(0, (oparg & 1)); + *effect = Py_MAX(1, (oparg & 1)); return 0; } case LOAD_BUILD_CLASS: { diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 721fa94da19615..5670fe26f72071 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -862,7 +862,7 @@ int _PyUop_num_popped(int opcode, int oparg) case _CHECK_ATTR_WITH_HINT: return 0; case _LOAD_ATTR_WITH_HINT: - return 1; + return 2; case _LOAD_ATTR_SLOT_0: return 1; case _LOAD_ATTR_SLOT_1: diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py index 54c83e09f892a0..92d056e802eeed 100644 --- a/Lib/test/test_capi/test_type.py +++ b/Lib/test/test_capi/test_type.py @@ -1,4 +1,4 @@ -from test.support import import_helper +from test.support import import_helper, Py_GIL_DISABLED, refleak_helper import unittest _testcapi = import_helper.import_module('_testcapi') @@ -37,6 +37,9 @@ class D(A, C): pass # as well type_freeze(D) + @unittest.skipIf( + Py_GIL_DISABLED and refleak_helper.hunting_for_refleaks(), + "Specialization failure triggers gh-127773") def test_freeze_meta(self): """test PyType_Freeze() with overridden MRO""" type_freeze = _testcapi.type_freeze diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 51f97bb51f7bd2..a7ebc9e8be0294 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -7,6 +7,7 @@ import random import string import sys +import textwrap import types import unittest import warnings @@ -15,6 +16,7 @@ from copy import deepcopy from contextlib import redirect_stdout from test import support +from test.support.script_helper import assert_python_ok try: import _testcapi @@ -5222,6 +5224,7 @@ def test_type_lookup_mro_reference(self): # Issue #14199: _PyType_Lookup() has to keep a strong reference to # the type MRO because it may be modified during the lookup, if # __bases__ is set during the lookup for example. + code = textwrap.dedent(""" class MyKey(object): def __hash__(self): return hash('mykey') @@ -5237,12 +5240,29 @@ class Base2(object): mykey = 'from Base2' mykey2 = 'from Base2' - with self.assertWarnsRegex(RuntimeWarning, 'X'): - X = type('X', (Base,), {MyKey(): 5}) - # mykey is read from Base - self.assertEqual(X.mykey, 'from Base') - # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__ - self.assertEqual(X.mykey2, 'from Base2') + X = type('X', (Base,), {MyKey(): 5}) + + bases_before = ",".join([c.__name__ for c in X.__bases__]) + print(f"before={bases_before}") + + # mykey is initially read from Base, however, the lookup will be perfomed + # again if specialization fails. The second lookup will use the new + # mro set by __eq__. + print(X.mykey) + + bases_after = ",".join([c.__name__ for c in X.__bases__]) + print(f"after={bases_after}") + + # mykey2 is read from Base2 because MyKey.__eq__ has set __bases_ + print(f"mykey2={X.mykey2}") + """) + _, out, err = assert_python_ok("-c", code) + err = err.decode() + self.assertRegex(err, "RuntimeWarning: .*X") + out = out.decode() + self.assertRegex(out, "before=Base") + self.assertRegex(out, "after=Base2") + self.assertRegex(out, "mykey2=from Base2") class PicklingTests(unittest.TestCase): diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 4a1c99edacb71a..7a50a29bb0126c 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1639,12 +1639,16 @@ def test_escaping_call_next_to_cmacro(self): """ self.run_cases_test(input, output) - def test_pop_dead_inputs_all_live(self): + def test_pystackref_frompyobject_new_next_to_cmacro(self): input = """ - inst(OP, (a, b --)) { - POP_DEAD_INPUTS(); - HAM(a, b); - INPUTS_DEAD(); + inst(OP, (-- out1, out2)) { + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); } """ output = """ @@ -1652,22 +1656,28 @@ def test_pop_dead_inputs_all_live(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - _PyStackRef a; - _PyStackRef b; - b = stack_pointer[-1]; - a = stack_pointer[-2]; - HAM(a, b); - stack_pointer += -2; + _PyStackRef out1; + _PyStackRef out2; + PyObject *obj = SPAM(); + #ifdef Py_GIL_DISABLED + out1 = PyStackRef_FromPyObjectNew(obj); + #else + out1 = PyStackRef_FromPyObjectNew(obj); + #endif + out2 = PyStackRef_FromPyObjectNew(obj); + stack_pointer[0] = out1; + stack_pointer[1] = out2; + stack_pointer += 2; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } """ self.run_cases_test(input, output) - def test_pop_dead_inputs_some_live(self): + def test_pop_input(self): input = """ - inst(OP, (a, b, c --)) { - POP_DEAD_INPUTS(); + inst(OP, (a, b --)) { + POP_INPUT(b); HAM(a); INPUTS_DEAD(); } @@ -1678,8 +1688,10 @@ def test_pop_dead_inputs_some_live(self): next_instr += 1; INSTRUCTION_STATS(OP); _PyStackRef a; - a = stack_pointer[-3]; - stack_pointer += -2; + _PyStackRef b; + b = stack_pointer[-1]; + a = stack_pointer[-2]; + stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); HAM(a); stack_pointer += -1; @@ -1689,29 +1701,23 @@ def test_pop_dead_inputs_some_live(self): """ self.run_cases_test(input, output) - def test_pop_dead_inputs_with_output(self): + def test_pop_input_with_empty_stack(self): input = """ - inst(OP, (a, b -- c)) { - POP_DEAD_INPUTS(); - c = SPAM(); + inst(OP, (--)) { + POP_INPUT(foo); } """ - output = """ - TARGET(OP) { - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(OP); - _PyStackRef c; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - c = SPAM(); - stack_pointer[0] = c; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - DISPATCH(); + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") + + def test_pop_input_with_non_tos(self): + input = """ + inst(OP, (a, b --)) { + POP_INPUT(a); } """ - self.run_cases_test(input, output) + with self.assertRaises(SyntaxError): + self.run_cases_test(input, "") def test_no_escaping_calls_in_branching_macros(self): diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index c7cd4c2e8a3146..b80ccbf17f1ee6 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -564,6 +564,16 @@ def instantiate(): instantiate() +def make_deferred_ref_count_obj(): + """Create an object that uses deferred reference counting. + + Only objects that use deferred refence counting may be stored in inline + caches in free-threaded builds. This constructs a new class named Foo, + which uses deferred reference counting. + """ + return type("Foo", (object,), {}) + + @threading_helper.requires_working_threading() class TestRacesDoNotCrash(TestBase): # Careful with these. Bigger numbers have a higher chance of catching bugs, @@ -714,11 +724,11 @@ def write(items): opname = "FOR_ITER_LIST" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_class(self): def get_items(): class C: - a = object() + a = make_deferred_ref_count_obj() items = [] for _ in range(self.ITEMS): @@ -739,12 +749,45 @@ def write(items): del item.a except AttributeError: pass - item.a = object() + item.a = make_deferred_ref_count_obj() opname = "LOAD_ATTR_CLASS" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft + def test_load_attr_class_with_metaclass_check(self): + def get_items(): + class Meta(type): + pass + + class C(metaclass=Meta): + a = make_deferred_ref_count_obj() + + items = [] + for _ in range(self.ITEMS): + item = C + items.append(item) + return items + + def read(items): + for item in items: + try: + item.a + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.a + except AttributeError: + pass + item.a = make_deferred_ref_count_obj() + + opname = "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK" + self.assert_races_do_not_crash(opname, get_items, read, write) + + @requires_specialization_ft def test_load_attr_getattribute_overridden(self): def get_items(): class C: @@ -774,7 +817,7 @@ def write(items): opname = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_instance_value(self): def get_items(): class C: @@ -798,7 +841,7 @@ def write(items): opname = "LOAD_ATTR_INSTANCE_VALUE" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_method_lazy_dict(self): def get_items(): class C(Exception): @@ -828,7 +871,7 @@ def write(items): opname = "LOAD_ATTR_METHOD_LAZY_DICT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_method_no_dict(self): def get_items(): class C: @@ -859,7 +902,7 @@ def write(items): opname = "LOAD_ATTR_METHOD_NO_DICT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_method_with_values(self): def get_items(): class C: @@ -914,7 +957,7 @@ def write(items): opname = "LOAD_ATTR_MODULE" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_load_attr_property(self): def get_items(): class C: @@ -944,7 +987,34 @@ def write(items): opname = "LOAD_ATTR_PROPERTY" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft + def test_load_attr_slot(self): + def get_items(): + class C: + __slots__ = ["a", "b"] + + items = [] + for i in range(self.ITEMS): + item = C() + item.a = i + item.b = i + self.ITEMS + items.append(item) + return items + + def read(items): + for item in items: + item.a + item.b + + def write(items): + for item in items: + item.a = 100 + item.b = 200 + + opname = "LOAD_ATTR_SLOT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + @requires_specialization_ft def test_load_attr_with_hint(self): def get_items(): class C: diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 2a054c3f2ae0ff..82789d5e56f523 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1129,6 +1129,24 @@ dictkeys_generic_lookup(PyDictObject *mp, PyDictKeysObject* dk, PyObject *key, P return do_lookup(mp, dk, key, hash, compare_generic); } +static bool +check_keys_unicode(PyDictKeysObject *dk, PyObject *key) +{ + return PyUnicode_CheckExact(key) && (dk->dk_kind != DICT_KEYS_GENERAL); +} + +static Py_ssize_t +hash_unicode_key(PyObject *key) +{ + assert(PyUnicode_CheckExact(key)); + Py_hash_t hash = unicode_get_hash(key); + if (hash == -1) { + hash = PyUnicode_Type.tp_hash(key); + assert(hash != -1); + } + return hash; +} + #ifdef Py_GIL_DISABLED static Py_ssize_t unicodekeys_lookup_unicode_threadsafe(PyDictKeysObject* dk, PyObject *key, @@ -1167,21 +1185,28 @@ unicodekeys_lookup_split(PyDictKeysObject* dk, PyObject *key, Py_hash_t hash) Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dk, PyObject *key) { - DictKeysKind kind = dk->dk_kind; - if (!PyUnicode_CheckExact(key) || kind == DICT_KEYS_GENERAL) { + if (!check_keys_unicode(dk, key)) { return DKIX_ERROR; } - Py_hash_t hash = unicode_get_hash(key); - if (hash == -1) { - hash = PyUnicode_Type.tp_hash(key); - if (hash == -1) { - PyErr_Clear(); - return DKIX_ERROR; - } - } + Py_hash_t hash = hash_unicode_key(key); return unicodekeys_lookup_unicode(dk, key, hash); } +Py_ssize_t +_PyDictKeys_StringLookupAndVersion(PyDictKeysObject *dk, PyObject *key, uint32_t *version) +{ + if (!check_keys_unicode(dk, key)) { + return DKIX_ERROR; + } + Py_ssize_t ix; + Py_hash_t hash = hash_unicode_key(key); + LOCK_KEYS(dk); + ix = unicodekeys_lookup_unicode(dk, key, hash); + *version = _PyDictKeys_GetVersionForCurrentState(_PyInterpreterState_GET(), dk); + UNLOCK_KEYS(dk); + return ix; +} + /* Like _PyDictKeys_StringLookup() but only works on split keys. Note * that in free-threaded builds this locks the keys object as required. */ @@ -1926,6 +1951,16 @@ build_indices_unicode(PyDictKeysObject *keys, PyDictUnicodeEntry *ep, Py_ssize_t } } +static void +invalidate_and_clear_inline_values(PyDictValues *values) +{ + assert(values->embedded); + FT_ATOMIC_STORE_UINT8(values->valid, 0); + for (int i = 0; i < values->capacity; i++) { + FT_ATOMIC_STORE_PTR_RELEASE(values->values[i], NULL); + } +} + /* Restructure the table by allocating a new table and reinserting all items again. When entries have been deleted, the new table may @@ -2017,7 +2052,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp, if (oldvalues->embedded) { assert(oldvalues->embedded == 1); assert(oldvalues->valid == 1); - FT_ATOMIC_STORE_UINT8(oldvalues->valid, 0); + invalidate_and_clear_inline_values(oldvalues); } else { free_values(oldvalues, IS_DICT_SHARED(mp)); @@ -7007,7 +7042,13 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr #ifdef Py_GIL_DISABLED PyObject *value = _Py_atomic_load_ptr_acquire(&values->values[ix]); - if (value == NULL || _Py_TryIncrefCompare(&values->values[ix], value)) { + if (value == NULL) { + if (FT_ATOMIC_LOAD_UINT8(values->valid)) { + *attr = NULL; + return true; + } + } + else if (_Py_TryIncrefCompare(&values->values[ix], value)) { *attr = value; return true; } @@ -7345,7 +7386,7 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) } mp->ma_values = values; - FT_ATOMIC_STORE_UINT8(_PyObject_InlineValues(obj)->valid, 0); + invalidate_and_clear_inline_values(_PyObject_InlineValues(obj)); assert(_PyObject_InlineValuesConsistencyCheck(obj)); ASSERT_CONSISTENT(mp); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cec530fefffefb..a906ded365650c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2190,18 +2190,23 @@ dummy_func( PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)); } split op(_LOAD_ATTR_INSTANCE_VALUE, (offset/1, owner -- attr, null if (oparg & 1))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); DEOPT_IF(attr_o == NULL); + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + DEOPT_IF(true); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); DECREF_INPUTS(); } @@ -2227,9 +2232,8 @@ dummy_func( assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(mod_keys->dk_nentries)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(mod_keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - DEAD(mod_keys); // Clear mod_keys from stack in case we need to deopt - POP_DEAD_INPUTS(); + POP_INPUT(mod_keys); DEOPT_IF(attr_o == NULL); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); @@ -2251,30 +2255,50 @@ dummy_func( _LOAD_ATTR_MODULE_FROM_KEYS + unused/5; - op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) { + op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict: PyDictObject *)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - EXIT_IF(dict == NULL); - assert(PyDict_CheckExact((PyObject *)dict)); + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + EXIT_IF(dict_o == NULL); + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; } - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict: PyDictObject * -- attr, null if (oparg & 1))) { PyObject *attr_o; + if (!LOCK_OBJECT(dict)) { + POP_INPUT(dict); + DEOPT_IF(true); + } - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys)); + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL); + if (attr_o == NULL) { + UNLOCK_OBJECT(dict); + POP_INPUT(dict); + DEOPT_IF(true); + } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); + DEAD(dict); null = PyStackRef_NULL; DECREF_INPUTS(); } @@ -2289,12 +2313,17 @@ dummy_func( split op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); DEOPT_IF(attr_o == NULL); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + DEOPT_IF(!increfed); + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); DECREF_INPUTS(); } @@ -2309,7 +2338,7 @@ dummy_func( EXIT_IF(!PyType_Check(owner_o)); assert(type_version != 0); - EXIT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version); + EXIT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version); } split op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr, null if (oparg & 1))) { @@ -2363,7 +2392,7 @@ dummy_func( DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -3281,13 +3310,15 @@ dummy_func( op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid)); } op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version); } split op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { @@ -3357,7 +3388,7 @@ dummy_func( op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) { char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1aa80f398d7470..cda01bb768c269 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2652,7 +2652,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner_o)->valid) { + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -2668,15 +2668,23 @@ uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; @@ -2691,15 +2699,23 @@ uint16_t offset = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; @@ -2778,55 +2794,88 @@ case _CHECK_ATTR_WITH_HINT: { _PyStackRef owner; + PyDictObject *dict; owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + if (dict_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(PyDict_CheckExact((PyObject *)dict)); + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; + stack_pointer[0].bits = (uintptr_t)dict; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_WITH_HINT: { + PyDictObject *dict; _PyStackRef owner; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; + dict = (PyDictObject *)stack_pointer[-1].bits; + owner = stack_pointer[-2]; uint16_t hint = (uint16_t)CURRENT_OPERAND0(); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (!LOCK_OBJECT(dict)) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } if (hint >= (size_t)dict->ma_keys->dk_nentries) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (!DK_IS_UNICODE(dict->ma_keys)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } attr_o = ep->me_value; if (attr_o == NULL) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + UNLOCK_OBJECT(dict); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); null = PyStackRef_NULL; PyStackRef_CLOSE(owner); - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); + stack_pointer[-2] = attr; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } @@ -2839,15 +2888,23 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; break; @@ -2861,15 +2918,23 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND0(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); PyStackRef_CLOSE(owner); stack_pointer[-1] = attr; stack_pointer[0] = null; @@ -2890,7 +2955,7 @@ JUMP_TO_JUMP_TARGET(); } assert(type_version != 0); - if (((PyTypeObject *)owner_o)->tp_version_tag != type_version) { + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3924,7 +3989,8 @@ owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner_o)->valid) { + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -3937,7 +4003,8 @@ uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - if (owner_heap_type->ht_cached_keys->dk_version != keys_version) { + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } @@ -4022,7 +4089,7 @@ owner = stack_pointer[-1]; uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ if (dict != NULL) { UOP_STAT_INC(uopcode, miss); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 810beb61d0db5e..81408380d6b2b8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5345,7 +5345,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS @@ -5380,7 +5380,7 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); DEOPT_IF(!PyType_Check(owner_o), LOAD_ATTR); assert(type_version != 0); - DEOPT_IF(((PyTypeObject *)owner_o)->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version, LOAD_ATTR); } // _GUARD_TYPE_VERSION { @@ -5421,7 +5421,7 @@ DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version, LOAD_ATTR); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); @@ -5463,19 +5463,24 @@ PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid), LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = *value_ptr; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + if (!_Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr)) { + DEOPT_IF(true, LOAD_ATTR); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectSteal(attr_o); PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ @@ -5507,7 +5512,7 @@ { uint16_t dictoffset = read_u16(&this_instr[4].cache); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = *(PyObject **)ptr; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); } @@ -5586,14 +5591,16 @@ { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -5716,14 +5723,16 @@ { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - DEOPT_IF(!_PyObject_InlineValues(owner_o)->valid, LOAD_ATTR); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + DEOPT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&this_instr[4].cache); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version, LOAD_ATTR); } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { @@ -5824,12 +5833,17 @@ { uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - char *addr = (char *)owner_o + index; - PyObject *attr_o = *(PyObject **)addr; + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); DEOPT_IF(attr_o == NULL, LOAD_ATTR); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + DEOPT_IF(!increfed, LOAD_ATTR); + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif STAT_INC(LOAD_ATTR, hit); null = PyStackRef_NULL; - attr = PyStackRef_FromPyObjectNew(attr_o); PyStackRef_CLOSE(owner); } /* Skip 5 cache entries */ @@ -5846,6 +5860,7 @@ INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; + PyDictObject *dict; _PyStackRef attr; _PyStackRef null = PyStackRef_NULL; /* Skip 1 cache entry */ @@ -5861,26 +5876,40 @@ { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(dict == NULL, LOAD_ATTR); - assert(PyDict_CheckExact((PyObject *)dict)); + PyDictObject *dict_o = _PyObject_GetManagedDict(owner_o); + DEOPT_IF(dict_o == NULL, LOAD_ATTR); + assert(PyDict_CheckExact((PyObject *)dict_o)); + dict = dict_o; } // _LOAD_ATTR_WITH_HINT { uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject *attr_o; - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + if (!LOCK_OBJECT(dict)) { + DEOPT_IF(true, LOAD_ATTR); + } + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - DEOPT_IF(!DK_IS_UNICODE(dict->ma_keys), LOAD_ATTR); + if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } attr_o = ep->me_value; - DEOPT_IF(attr_o == NULL, LOAD_ATTR); + if (attr_o == NULL) { + UNLOCK_OBJECT(dict); + DEOPT_IF(true, LOAD_ATTR); + } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(attr_o); - attr = PyStackRef_FromPyObjectSteal(attr_o); + attr = PyStackRef_FromPyObjectNew(attr_o); + UNLOCK_OBJECT(dict); null = PyStackRef_NULL; PyStackRef_CLOSE(owner); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 788adecca8af80..4d96ada5acf00f 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -582,11 +582,17 @@ dummy_func(void) { } } - op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) { + op(_CHECK_ATTR_WITH_HINT, (owner -- owner, dict)) { + dict = sym_new_not_null(ctx); + (void)owner; + } + + op(_LOAD_ATTR_WITH_HINT, (hint/1, owner, dict -- attr, null if (oparg & 1))) { attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)hint; (void)owner; + (void)dict; } op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 1a7cc6becfefb6..aff4493fdc4dd7 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1250,22 +1250,33 @@ } case _CHECK_ATTR_WITH_HINT: { + _Py_UopsSymbol *owner; + _Py_UopsSymbol *dict; + owner = stack_pointer[-1]; + dict = sym_new_not_null(ctx); + (void)owner; + stack_pointer[0] = dict; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } case _LOAD_ATTR_WITH_HINT: { + _Py_UopsSymbol *dict; _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *null = NULL; - owner = stack_pointer[-1]; + dict = stack_pointer[-1]; + owner = stack_pointer[-2]; uint16_t hint = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); null = sym_new_null(ctx); (void)hint; (void)owner; - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = null; - stack_pointer += (oparg & 1); + (void)dict; + stack_pointer[-2] = attr; + if (oparg & 1) stack_pointer[-1] = null; + stack_pointer += -1 + (oparg & 1); assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/specialize.c b/Python/specialize.c index 897005c4f1078d..8d9f19c8895187 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -3,6 +3,7 @@ #include "opcode.h" #include "pycore_code.h" +#include "pycore_critical_section.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // DICT_KEYS_UNICODE #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState() @@ -537,6 +538,7 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, PyObject *consts, #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ 33 #define SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN 34 #define SPEC_FAIL_ATTR_SPLIT_DICT 35 +#define SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED 36 /* Binary subscr and store subscr */ @@ -729,11 +731,8 @@ unspecialize(_Py_CODEUNIT *instr) } static int function_kind(PyCodeObject *code); -#ifndef Py_GIL_DISABLED static bool function_check_args(PyObject *o, int expected_argcount, int opcode); static uint32_t function_get_version(PyObject *o, int opcode); -static uint32_t type_get_version(PyTypeObject *t, int opcode); -#endif static int specialize_module_load_attr_lock_held(PyDictObject *dict, _Py_CODEUNIT *instr, PyObject *name) @@ -879,10 +878,11 @@ descriptor_is_class(PyObject *descriptor, PyObject *name) (descriptor == _PyType_Lookup(&PyBaseObject_Type, name))); } -#ifndef Py_GIL_DISABLED static DescriptorClassification -analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { +analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) { bool has_getattr = false; + bool have_ga_version = false; + unsigned int ga_version; getattrofunc getattro_slot = type->tp_getattro; if (getattro_slot == PyObject_GenericGetAttr) { /* Normal attribute lookup; */ @@ -892,24 +892,27 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { getattro_slot == _Py_slot_tp_getattro) { /* One or both of __getattribute__ or __getattr__ may have been overridden See typeobject.c for why these functions are special. */ - PyObject *getattribute = _PyType_LookupRef(type, &_Py_ID(__getattribute__)); + PyObject *getattribute = _PyType_LookupRefAndVersion(type, + &_Py_ID(__getattribute__), &ga_version); + have_ga_version = true; PyInterpreterState *interp = _PyInterpreterState_GET(); bool has_custom_getattribute = getattribute != NULL && getattribute != interp->callable_cache.object__getattribute__; - PyObject *getattr = _PyType_LookupRef(type, &_Py_ID(__getattr__)); + PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)); has_getattr = getattr != NULL; - Py_XDECREF(getattr); if (has_custom_getattribute) { if (getattro_slot == _Py_slot_tp_getattro && !has_getattr && Py_IS_TYPE(getattribute, &PyFunction_Type)) { *descr = getattribute; + *tp_version = ga_version; return GETATTRIBUTE_IS_PYTHON_FUNCTION; } /* Potentially both __getattr__ and __getattribute__ are set. Too complicated */ Py_DECREF(getattribute); *descr = NULL; + *tp_version = ga_version; return GETSET_OVERRIDDEN; } /* Potentially has __getattr__ but no custom __getattribute__. @@ -923,16 +926,18 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr) { } else { *descr = NULL; + *tp_version = FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag); return GETSET_OVERRIDDEN; } - PyObject *descriptor = _PyType_LookupRef(type, name); + unsigned int descr_version; + PyObject *descriptor = _PyType_LookupRefAndVersion(type, name, &descr_version); *descr = descriptor; + *tp_version = have_ga_version ? ga_version : descr_version; if (descriptor_is_class(descriptor, name)) { return DUNDER_CLASS; } return classify_descriptor(descriptor, has_getattr); } -#endif //!Py_GIL_DISABLED static DescriptorClassification analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, unsigned int *tp_version) @@ -952,12 +957,13 @@ analyze_descriptor_store(PyTypeObject *type, PyObject *name, PyObject **descr, u static int specialize_dict_access_inline( PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type, - DescriptorClassification kind, PyObject *name, unsigned int tp_version, + PyObject *name, unsigned int tp_version, int base_op, int values_op) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(PyUnicode_CheckExact(name)); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(owner); Py_ssize_t index = _PyDictKeys_StringLookupSplit(keys, name); assert (index != DKIX_ERROR); if (index == DKIX_EMPTY) { @@ -965,6 +971,7 @@ specialize_dict_access_inline( return 0; } assert(index >= 0); + assert(_PyObject_InlineValues(owner)->valid); char *value_addr = (char *)&_PyObject_InlineValues(owner)->values[index]; Py_ssize_t offset = value_addr - (char *)owner; if (offset != (uint16_t)offset) { @@ -980,10 +987,13 @@ specialize_dict_access_inline( static int specialize_dict_access_hint( PyDictObject *dict, _Py_CODEUNIT *instr, PyTypeObject *type, - DescriptorClassification kind, PyObject *name, unsigned int tp_version, + PyObject *name, unsigned int tp_version, int base_op, int hint_op) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict); + // We found an instance with a __dict__. if (_PyDict_HasSplitTable(dict)) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_SPLIT_DICT); @@ -1027,7 +1037,7 @@ specialize_dict_access( PyDictObject *dict = _PyObject_GetManagedDict(owner); if (dict == NULL) { // managed dict, not materialized, inline values valid - res = specialize_dict_access_inline(owner, instr, type, kind, name, + res = specialize_dict_access_inline(owner, instr, type, name, tp_version, base_op, values_op); } else { @@ -1047,16 +1057,19 @@ specialize_dict_access( int res; Py_BEGIN_CRITICAL_SECTION(dict); // materialized managed dict - res = specialize_dict_access_hint(dict, instr, type, kind, name, + res = specialize_dict_access_hint(dict, instr, type, name, tp_version, base_op, hint_op); Py_END_CRITICAL_SECTION(); return res; } } -#ifndef Py_GIL_DISABLED -static int specialize_attr_loadclassattr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, - PyObject* descr, DescriptorClassification kind, bool is_method); +static int +specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, + PyObject *name, PyObject *descr, + unsigned int tp_version, + DescriptorClassification kind, bool is_method, + uint32_t shared_keys_version); static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name); /* Returns true if instances of obj's class are @@ -1065,7 +1078,7 @@ static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyOb * For other objects, we check their actual dictionary. */ static bool -instance_has_key(PyObject *obj, PyObject* name) +instance_has_key(PyObject *obj, PyObject *name, uint32_t *shared_keys_version) { PyTypeObject *cls = Py_TYPE(obj); if ((cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { @@ -1073,36 +1086,38 @@ instance_has_key(PyObject *obj, PyObject* name) } if (cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictKeysObject *keys = ((PyHeapTypeObject *)cls)->ht_cached_keys; - Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); + Py_ssize_t index = + _PyDictKeys_StringLookupAndVersion(keys, name, shared_keys_version); return index >= 0; } PyDictObject *dict = _PyObject_GetManagedDict(obj); if (dict == NULL || !PyDict_CheckExact(dict)) { return false; } + bool result; + Py_BEGIN_CRITICAL_SECTION(dict); if (dict->ma_values) { - return false; + result = false; } - Py_ssize_t index = _PyDict_LookupIndex(dict, name); - if (index < 0) { - return false; + else { + result = (_PyDict_LookupIndex(dict, name) >= 0); } - return true; + Py_END_CRITICAL_SECTION(); + return result; } static int -specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name) +do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name, + bool shadow, uint32_t shared_keys_version, + DescriptorClassification kind, PyObject *descr, unsigned int tp_version) { _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); - bool shadow = instance_has_key(owner, name); - PyObject *descr = NULL; - DescriptorClassification kind = analyze_descriptor_load(type, name, &descr); - Py_XDECREF(descr); // turn strong ref into a borrowed ref - assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); - if (type_get_version(type, LOAD_ATTR) == 0) { + if (tp_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } + uint8_t oparg = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.arg); switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -1112,9 +1127,10 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (shadow) { goto try_instance; } - int oparg = instr->op.arg; if (oparg & 1) { - if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, true)) { + if (specialize_attr_loadclassattr(owner, instr, name, descr, + tp_version, kind, true, + shared_keys_version)) { return 0; } else { @@ -1140,7 +1156,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (!function_check_args(fget, 1, LOAD_ATTR)) { return -1; } - if (instr->op.arg & 1) { + if (oparg & 1) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); return -1; } @@ -1149,8 +1165,14 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); return -1; } - assert(type->tp_version_tag != 0); - write_u32(lm_cache->type_version, type->tp_version_tag); + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(fget)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return -1; + } + #endif + assert(tp_version != 0); + write_u32(lm_cache->type_version, tp_version); /* borrowed */ write_obj(lm_cache->descr, fget); specialize(instr, LOAD_ATTR_PROPERTY); @@ -1176,7 +1198,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na assert(dmem->type == Py_T_OBJECT_EX || dmem->type == _Py_T_OBJECT); assert(offset > 0); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); + write_u32(cache->version, tp_version); specialize(instr, LOAD_ATTR_SLOT); return 0; } @@ -1185,7 +1207,7 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na Py_ssize_t offset = offsetof(PyObject, ob_type); assert(offset == (uint16_t)offset); cache->index = (uint16_t)offset; - write_u32(cache->version, type->tp_version_tag); + write_u32(cache->version, tp_version); specialize(instr, LOAD_ATTR_SLOT); return 0; } @@ -1200,13 +1222,18 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na return -1; case GETATTRIBUTE_IS_PYTHON_FUNCTION: { + #ifndef Py_GIL_DISABLED + // In free-threaded builds it's possible for tp_getattro to change + // after the call to analyze_descriptor. That is fine: the version + // guard will fail. assert(type->tp_getattro == _Py_slot_tp_getattro); + #endif assert(Py_IS_TYPE(descr, &PyFunction_Type)); _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1); if (!function_check_args(descr, 2, LOAD_ATTR)) { return -1; } - if (instr->op.arg & 1) { + if (oparg & 1) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); return -1; } @@ -1219,10 +1246,16 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); return -1; } + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return -1; + } + #endif write_u32(lm_cache->keys_version, version); /* borrowed */ write_obj(lm_cache->descr, descr); - write_u32(lm_cache->type_version, type->tp_version_tag); + write_u32(lm_cache->type_version, tp_version); specialize(instr, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); return 0; } @@ -1237,8 +1270,10 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na if (shadow) { goto try_instance; } - if ((instr->op.arg & 1) == 0) { - if (specialize_attr_loadclassattr(owner, instr, name, descr, kind, false)) { + if ((oparg & 1) == 0) { + if (specialize_attr_loadclassattr(owner, instr, name, descr, + tp_version, kind, false, + shared_keys_version)) { return 0; } } @@ -1252,14 +1287,28 @@ specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* na } Py_UNREACHABLE(); try_instance: - if (specialize_dict_access(owner, instr, type, kind, name, type->tp_version_tag, + if (specialize_dict_access(owner, instr, type, kind, name, tp_version, LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT)) { return 0; } return -1; } -#endif // Py_GIL_DISABLED + +static int +specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* name) +{ + // 0 is not a valid version + uint32_t shared_keys_version = 0; + bool shadow = instance_has_key(owner, name, &shared_keys_version); + PyObject *descr = NULL; + unsigned int tp_version = 0; + PyTypeObject *type = Py_TYPE(owner); + DescriptorClassification kind = analyze_descriptor_load(type, name, &descr, &tp_version); + int result = do_specialize_instance_load_attr(owner, instr, name, shadow, shared_keys_version, kind, descr, tp_version); + Py_XDECREF(descr); + return result; +} void _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *name) @@ -1281,20 +1330,10 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam fail = specialize_module_load_attr(owner, instr, name); } else if (PyType_Check(owner)) { - #ifdef Py_GIL_DISABLED - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); - fail = true; - #else fail = specialize_class_load_attr(owner, instr, name); - #endif } else { - #ifdef Py_GIL_DISABLED - SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); - fail = true; - #else fail = specialize_instance_load_attr(owner, instr, name); - #endif } if (fail) { @@ -1402,8 +1441,6 @@ _Py_Specialize_StoreAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *na return; } -#ifndef Py_GIL_DISABLED - #ifdef Py_STATS static int load_attr_fail_kind(DescriptorClassification kind) @@ -1452,8 +1489,10 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_OVERRIDDEN); return -1; } - PyObject *metadescriptor = _PyType_Lookup(Py_TYPE(cls), name); + unsigned int meta_version = 0; + PyObject *metadescriptor = _PyType_LookupRefAndVersion(Py_TYPE(cls), name, &meta_version); DescriptorClassification metakind = classify_descriptor(metadescriptor, false); + Py_XDECREF(metadescriptor); switch (metakind) { case METHOD: case NON_DESCRIPTOR: @@ -1468,38 +1507,52 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, } PyObject *descr = NULL; DescriptorClassification kind = 0; - kind = analyze_descriptor_load(cls, name, &descr); - Py_XDECREF(descr); // turn strong ref into a borrowed ref - if (type_get_version(cls, LOAD_ATTR) == 0) { + unsigned int tp_version = 0; + kind = analyze_descriptor_load(cls, name, &descr, &tp_version); + if (tp_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + Py_XDECREF(descr); return -1; } bool metaclass_check = false; if ((Py_TYPE(cls)->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) == 0) { metaclass_check = true; - if (type_get_version(Py_TYPE(cls), LOAD_ATTR) == 0) { + if (meta_version == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); + Py_XDECREF(descr); return -1; } } switch (kind) { case METHOD: case NON_DESCRIPTOR: - write_u32(cache->type_version, cls->tp_version_tag); + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + Py_XDECREF(descr); + return -1; + } + #endif + write_u32(cache->type_version, tp_version); write_obj(cache->descr, descr); if (metaclass_check) { - write_u32(cache->keys_version, Py_TYPE(cls)->tp_version_tag); + write_u32(cache->keys_version, meta_version); specialize(instr, LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); } else { specialize(instr, LOAD_ATTR_CLASS); } + Py_XDECREF(descr); return 0; #ifdef Py_STATS case ABSENT: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR); + Py_XDECREF(descr); return -1; #endif default: SPECIALIZATION_FAIL(LOAD_ATTR, load_attr_fail_kind(kind)); + Py_XDECREF(descr); return -1; } } @@ -1508,29 +1561,41 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, // can cause a significant drop in cache hits. A possible test is // python.exe -m test_typing test_re test_dis test_zlib. static int -specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, -PyObject *descr, DescriptorClassification kind, bool is_method) +specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, + PyObject *name, PyObject *descr, + unsigned int tp_version, + DescriptorClassification kind, bool is_method, + uint32_t shared_keys_version) { _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); PyTypeObject *owner_cls = Py_TYPE(owner); assert(descr != NULL); assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR)); - if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) { - PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; - assert(_PyDictKeys_StringLookup(keys, name) < 0); - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( - _PyInterpreterState_GET(), keys); - if (keys_version == 0) { + + #ifdef Py_GIL_DISABLED + if (!_PyObject_HasDeferredRefcount(descr)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_DESCR_NOT_DEFERRED); + return 0; + } + #endif + + unsigned long tp_flags = PyType_GetFlags(owner_cls); + if (tp_flags & Py_TPFLAGS_INLINE_VALUES) { + #ifndef Py_GIL_DISABLED + assert(_PyDictKeys_StringLookup( + ((PyHeapTypeObject *)owner_cls)->ht_cached_keys, name) < 0); + #endif + if (shared_keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; } - write_u32(cache->keys_version, keys_version); + write_u32(cache->keys_version, shared_keys_version); specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); } else { Py_ssize_t dictoffset; - if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + if (tp_flags & Py_TPFLAGS_MANAGED_DICT) { dictoffset = MANAGED_DICT_OFFSET; } else { @@ -1576,13 +1641,11 @@ PyObject *descr, DescriptorClassification kind, bool is_method) * PyType_Modified usages in typeobject.c). The MCACHE has been * working since Python 2.6 and it's battle-tested. */ - write_u32(cache->type_version, owner_cls->tp_version_tag); + write_u32(cache->type_version, tp_version); write_obj(cache->descr, descr); return 1; } -#endif // Py_GIL_DISABLED - static void specialize_load_global_lock_held( @@ -1729,7 +1792,6 @@ function_kind(PyCodeObject *code) { return SIMPLE_FUNCTION; } -#ifndef Py_GIL_DISABLED /* Returning false indicates a failure. */ static bool function_check_args(PyObject *o, int expected_argcount, int opcode) @@ -1763,19 +1825,6 @@ function_get_version(PyObject *o, int opcode) return version; } -/* Returning 0 indicates a failure. */ -static uint32_t -type_get_version(PyTypeObject *t, int opcode) -{ - uint32_t version = t->tp_version_tag; - if (version == 0) { - SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS); - return 0; - } - return version; -} -#endif // Py_GIL_DISABLED - void _Py_Specialize_BinarySubscr( _PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 99896f32fd2b08..4013b503502df6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -386,7 +386,7 @@ def find_assignment_target(node: parser.InstDef, idx: int) -> list[lexer.Token]: """Find the tokens that make up the left-hand side of an assignment""" offset = 0 for tkn in reversed(node.block.tokens[: idx]): - if tkn.kind in {"SEMI", "LBRACE", "RBRACE"}: + if tkn.kind in {"SEMI", "LBRACE", "RBRACE", "CMACRO"}: return node.block.tokens[idx - offset : idx] offset += 1 return [] diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 69d84183f1c7e6..f54afbb880d2fa 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -126,7 +126,7 @@ def __init__(self, out: CWriter): "PyStackRef_AsPyObjectSteal": self.stackref_steal, "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, - "POP_DEAD_INPUTS": self.pop_dead_inputs, + "POP_INPUT": self.pop_input, } self.out = out @@ -423,7 +423,7 @@ def save_stack( self.emit_save(storage) return True - def pop_dead_inputs( + def pop_input( self, tkn: Token, tkn_iter: TokenIterator, @@ -432,9 +432,18 @@ def pop_dead_inputs( inst: Instruction | None, ) -> bool: next(tkn_iter) + name_tkn = next(tkn_iter) + name = name_tkn.text next(tkn_iter) next(tkn_iter) - storage.pop_dead_inputs(self.out) + if not storage.inputs: + raise analysis_error("stack is empty", tkn) + tos = storage.inputs[-1] + if tos.name != name: + raise analysis_error(f"'{name} is not top of stack", name_tkn) + tos.defined = False + storage.clear_dead_inputs() + storage.flush(self.out) return True def emit_reload(self, storage: Storage) -> None: diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 9471fe0e56f7d8..286f47d0cfb11b 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -512,10 +512,6 @@ def flush(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = self._push_defined_outputs() self.stack.flush(out, cast_type, extract_bits) - def pop_dead_inputs(self, out: CWriter, cast_type: str = "uintptr_t", extract_bits: bool = True) -> None: - self.clear_dead_inputs() - self.stack.flush(out, cast_type, extract_bits) - def save(self, out: CWriter) -> None: assert self.spilled >= 0 if self.spilled == 0: