Skip to content

Commit 2a4d4d5

Browse files
colesburyebonnal
colesbury
authored and
ebonnal
committed
pythongh-115999: Add free-threaded specialization for STORE_SUBSCR (python#127169)
The specialization only depends on the type, so no special thread-safety considerations there. STORE_SUBSCR_LIST_INT needs to lock the list before modifying it. `_PyDict_SetItem_Take2` already internally locks the dictionary using a critical section.
1 parent 6079b6e commit 2a4d4d5

File tree

5 files changed

+109
-70
lines changed

5 files changed

+109
-70
lines changed

‎Python/bytecodes.c

+8-3
Original file line numberDiff line numberDiff line change
@@ -910,15 +910,15 @@ dummy_func(
910910
};
911911

912912
specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) {
913-
#if ENABLE_SPECIALIZATION
913+
#if ENABLE_SPECIALIZATION_FT
914914
if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
915915
next_instr = this_instr;
916916
_Py_Specialize_StoreSubscr(container, sub, next_instr);
917917
DISPATCH_SAME_OPARG();
918918
}
919919
OPCODE_DEFERRED_INC(STORE_SUBSCR);
920920
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
921-
#endif /* ENABLE_SPECIALIZATION */
921+
#endif /* ENABLE_SPECIALIZATION_FT */
922922
}
923923

924924
op(_STORE_SUBSCR, (v, container, sub -- )) {
@@ -940,13 +940,18 @@ dummy_func(
940940
// Ensure nonnegative, zero-or-one-digit ints.
941941
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
942942
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
943+
DEOPT_IF(!LOCK_OBJECT(list));
943944
// Ensure index < len(list)
944-
DEOPT_IF(index >= PyList_GET_SIZE(list));
945+
if (index >= PyList_GET_SIZE(list)) {
946+
UNLOCK_OBJECT(list);
947+
DEOPT_IF(true);
948+
}
945949
STAT_INC(STORE_SUBSCR, hit);
946950

947951
PyObject *old_value = PyList_GET_ITEM(list, index);
948952
PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value));
949953
assert(old_value != NULL);
954+
UNLOCK_OBJECT(list); // unlock before decrefs!
950955
Py_DECREF(old_value);
951956
PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free);
952957
DEAD(sub_st);

‎Python/ceval_macros.h

+23
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,29 @@ GETITEM(PyObject *v, Py_ssize_t i) {
284284
}
285285

286286

287+
// Try to lock an object in the free threading build, if it's not already
288+
// locked. Use with a DEOPT_IF() to deopt if the object is already locked.
289+
// These are no-ops in the default GIL build. The general pattern is:
290+
//
291+
// DEOPT_IF(!LOCK_OBJECT(op));
292+
// if (/* condition fails */) {
293+
// UNLOCK_OBJECT(op);
294+
// DEOPT_IF(true);
295+
// }
296+
// ...
297+
// UNLOCK_OBJECT(op);
298+
//
299+
// NOTE: The object must be unlocked on every exit code path and you should
300+
// avoid any potentially escaping calls (like PyStackRef_CLOSE) while the
301+
// object is locked.
302+
#ifdef Py_GIL_DISABLED
303+
# define LOCK_OBJECT(op) PyMutex_LockFast(&(_PyObject_CAST(op))->ob_mutex._bits)
304+
# define UNLOCK_OBJECT(op) PyMutex_Unlock(&(_PyObject_CAST(op))->ob_mutex)
305+
#else
306+
# define LOCK_OBJECT(op) (1)
307+
# define UNLOCK_OBJECT(op) ((void)0)
308+
#endif
309+
287310
#define GLOBALS() frame->f_globals
288311
#define BUILTINS() frame->f_builtins
289312
#define LOCALS() frame->f_locals

‎Python/executor_cases.c.h

+10-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/generated_cases.c.h

+8-3
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/specialize.c

+60-62
Original file line numberDiff line numberDiff line change
@@ -1814,112 +1814,110 @@ _Py_Specialize_BinarySubscr(
18141814
cache->counter = adaptive_counter_cooldown();
18151815
}
18161816

1817-
void
1818-
_Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr)
1819-
{
1820-
PyObject *container = PyStackRef_AsPyObjectBorrow(container_st);
1821-
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
18221817

1823-
assert(ENABLE_SPECIALIZATION);
1824-
_PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1);
1825-
PyTypeObject *container_type = Py_TYPE(container);
1826-
if (container_type == &PyList_Type) {
1827-
if (PyLong_CheckExact(sub)) {
1828-
if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)
1829-
&& ((PyLongObject *)sub)->long_value.ob_digit[0] < (size_t)PyList_GET_SIZE(container))
1830-
{
1831-
instr->op.code = STORE_SUBSCR_LIST_INT;
1832-
goto success;
1833-
}
1834-
else {
1835-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1836-
goto fail;
1837-
}
1838-
}
1839-
else if (PySlice_Check(sub)) {
1840-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE);
1841-
goto fail;
1842-
}
1843-
else {
1844-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1845-
goto fail;
1846-
}
1847-
}
1848-
if (container_type == &PyDict_Type) {
1849-
instr->op.code = STORE_SUBSCR_DICT;
1850-
goto success;
1851-
}
18521818
#ifdef Py_STATS
1819+
static int
1820+
store_subscr_fail_kind(PyObject *container_type)
1821+
{
18531822
PyMappingMethods *as_mapping = container_type->tp_as_mapping;
18541823
if (as_mapping && (as_mapping->mp_ass_subscript
18551824
== PyDict_Type.tp_as_mapping->mp_ass_subscript)) {
1856-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE);
1857-
goto fail;
1825+
return SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE;
18581826
}
18591827
if (PyObject_CheckBuffer(container)) {
18601828
if (PyLong_CheckExact(sub) && (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub))) {
1861-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1829+
return SPEC_FAIL_OUT_OF_RANGE;
18621830
}
18631831
else if (strcmp(container_type->tp_name, "array.array") == 0) {
18641832
if (PyLong_CheckExact(sub)) {
1865-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_INT);
1833+
return SPEC_FAIL_SUBSCR_ARRAY_INT;
18661834
}
18671835
else if (PySlice_Check(sub)) {
1868-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_SLICE);
1836+
return SPEC_FAIL_SUBSCR_ARRAY_SLICE;
18691837
}
18701838
else {
1871-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1839+
return SPEC_FAIL_OTHER;
18721840
}
18731841
}
18741842
else if (PyByteArray_CheckExact(container)) {
18751843
if (PyLong_CheckExact(sub)) {
1876-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_INT);
1844+
return SPEC_FAIL_SUBSCR_BYTEARRAY_INT;
18771845
}
18781846
else if (PySlice_Check(sub)) {
1879-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE);
1847+
return SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE;
18801848
}
18811849
else {
1882-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1850+
return SPEC_FAIL_OTHER;
18831851
}
18841852
}
18851853
else {
18861854
if (PyLong_CheckExact(sub)) {
1887-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_INT);
1855+
return SPEC_FAIL_SUBSCR_BUFFER_INT;
18881856
}
18891857
else if (PySlice_Check(sub)) {
1890-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_SLICE);
1858+
return SPEC_FAIL_SUBSCR_BUFFER_SLICE;
18911859
}
18921860
else {
1893-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1861+
return SPEC_FAIL_OTHER;
18941862
}
18951863
}
1896-
goto fail;
1864+
return SPEC_FAIL_OTHER;
18971865
}
18981866
PyObject *descriptor = _PyType_Lookup(container_type, &_Py_ID(__setitem__));
18991867
if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) {
19001868
PyFunctionObject *func = (PyFunctionObject *)descriptor;
19011869
PyCodeObject *code = (PyCodeObject *)func->func_code;
19021870
int kind = function_kind(code);
19031871
if (kind == SIMPLE_FUNCTION) {
1904-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_SIMPLE);
1872+
return SPEC_FAIL_SUBSCR_PY_SIMPLE;
19051873
}
19061874
else {
1907-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_OTHER);
1875+
return SPEC_FAIL_SUBSCR_PY_OTHER;
19081876
}
1909-
goto fail;
19101877
}
1911-
#endif // Py_STATS
1912-
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1913-
fail:
1914-
STAT_INC(STORE_SUBSCR, failure);
1915-
assert(!PyErr_Occurred());
1916-
instr->op.code = STORE_SUBSCR;
1917-
cache->counter = adaptive_counter_backoff(cache->counter);
1918-
return;
1919-
success:
1920-
STAT_INC(STORE_SUBSCR, success);
1921-
assert(!PyErr_Occurred());
1922-
cache->counter = adaptive_counter_cooldown();
1878+
return SPEC_FAIL_OTHER;
1879+
}
1880+
#endif
1881+
1882+
void
1883+
_Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr)
1884+
{
1885+
PyObject *container = PyStackRef_AsPyObjectBorrow(container_st);
1886+
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
1887+
1888+
assert(ENABLE_SPECIALIZATION_FT);
1889+
PyTypeObject *container_type = Py_TYPE(container);
1890+
if (container_type == &PyList_Type) {
1891+
if (PyLong_CheckExact(sub)) {
1892+
if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)
1893+
&& ((PyLongObject *)sub)->long_value.ob_digit[0] < (size_t)PyList_GET_SIZE(container))
1894+
{
1895+
specialize(instr, STORE_SUBSCR_LIST_INT);
1896+
return;
1897+
}
1898+
else {
1899+
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1900+
unspecialize(instr);
1901+
return;
1902+
}
1903+
}
1904+
else if (PySlice_Check(sub)) {
1905+
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE);
1906+
unspecialize(instr);
1907+
return;
1908+
}
1909+
else {
1910+
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1911+
unspecialize(instr);
1912+
return;
1913+
}
1914+
}
1915+
if (container_type == &PyDict_Type) {
1916+
specialize(instr, STORE_SUBSCR_DICT);
1917+
return;
1918+
}
1919+
SPECIALIZATION_FAIL(STORE_SUBSCR, store_subscr_fail_kind(container_type));
1920+
unspecialize(instr);
19231921
}
19241922

19251923
/* Returns a borrowed reference.

0 commit comments

Comments
 (0)