From e5059b293f5bf8253178126e264d57bb09d61bca Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 11 Jan 2023 11:37:36 +0000 Subject: [PATCH 1/9] Add COMPARE_AND_BRANCH instruction --- Include/internal/pycore_code.h | 2 + Include/internal/pycore_opcode.h | 23 ++-- Include/opcode.h | 19 +-- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 14 ++- Lib/test/test_compile.py | 14 +-- Lib/test/test_dis.py | 8 +- Objects/frameobject.c | 8 ++ Python/bytecodes.c | 109 +++++++++------- Python/compile.c | 3 + Python/generated_cases.c.h | 179 +++++++++++++-------------- Python/opcode_metadata.h | 9 +- Python/opcode_targets.h | 14 +-- Python/specialize.c | 43 ++++--- 14 files changed, 244 insertions(+), 204 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index b53657ac08a041..582ffa615a1d7e 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -230,6 +230,8 @@ extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT * int oparg, PyObject **locals); extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg); +extern void _Py_Specialize_CompareAndBranch(PyObject *lhs, PyObject *rhs, + _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg); diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 5806d69f98a956..05c0485b0641d8 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -50,6 +50,7 @@ const uint8_t _PyOpcode_Caches[256] = { [COMPARE_OP] = 1, [LOAD_GLOBAL] = 5, [BINARY_OP] = 1, + [COMPARE_AND_BRANCH] = 1, [CALL] = 4, }; @@ -102,10 +103,11 @@ const uint8_t _PyOpcode_Deopt[256] = { [CHECK_EG_MATCH] = CHECK_EG_MATCH, [CHECK_EXC_MATCH] = CHECK_EXC_MATCH, [CLEANUP_THROW] = CLEANUP_THROW, + [COMPARE_AND_BRANCH] = COMPARE_AND_BRANCH, + [COMPARE_AND_BRANCH_FLOAT] = COMPARE_AND_BRANCH, + [COMPARE_AND_BRANCH_INT] = COMPARE_AND_BRANCH, + [COMPARE_AND_BRANCH_STR] = COMPARE_AND_BRANCH, [COMPARE_OP] = COMPARE_OP, - [COMPARE_OP_FLOAT_JUMP] = COMPARE_OP, - [COMPARE_OP_INT_JUMP] = COMPARE_OP, - [COMPARE_OP_STR_JUMP] = COMPARE_OP, [CONTAINS_OP] = CONTAINS_OP, [COPY] = COPY, [COPY_FREE_VARS] = COPY_FREE_VARS, @@ -274,7 +276,7 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", + [COMPARE_AND_BRANCH_FLOAT] = "COMPARE_AND_BRANCH_FLOAT", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -282,8 +284,8 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", - [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", - [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", + [COMPARE_AND_BRANCH_INT] = "COMPARE_AND_BRANCH_INT", + [COMPARE_AND_BRANCH_STR] = "COMPARE_AND_BRANCH_STR", [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [STORE_SUBSCR] = "STORE_SUBSCR", @@ -367,9 +369,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", + [COMPARE_AND_BRANCH] = "COMPARE_AND_BRANCH", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -379,14 +381,14 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [160] = "<160>", [161] = "<161>", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", @@ -493,7 +495,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 160: \ case 161: \ case 166: \ case 167: \ diff --git a/Include/opcode.h b/Include/opcode.h index fcc46a3ede9d44..e057a44d1fd8a8 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -95,6 +95,7 @@ extern "C" { #define STORE_DEREF 138 #define DELETE_DEREF 139 #define JUMP_BACKWARD 140 +#define COMPARE_AND_BRANCH 141 #define CALL_FUNCTION_EX 142 #define EXTENDED_ARG 144 #define LIST_APPEND 145 @@ -153,9 +154,9 @@ extern "C" { #define CALL_NO_KW_STR_1 45 #define CALL_NO_KW_TUPLE_1 46 #define CALL_NO_KW_TYPE_1 47 -#define COMPARE_OP_FLOAT_JUMP 48 -#define COMPARE_OP_INT_JUMP 56 -#define COMPARE_OP_STR_JUMP 57 +#define COMPARE_AND_BRANCH_FLOAT 48 +#define COMPARE_AND_BRANCH_INT 56 +#define COMPARE_AND_BRANCH_STR 57 #define FOR_ITER_LIST 58 #define FOR_ITER_TUPLE 59 #define FOR_ITER_RANGE 62 @@ -179,12 +180,12 @@ extern "C" { #define STORE_ATTR_SLOT 87 #define STORE_ATTR_WITH_HINT 113 #define STORE_FAST__LOAD_FAST 121 -#define STORE_FAST__STORE_FAST 141 -#define STORE_SUBSCR_DICT 143 -#define STORE_SUBSCR_LIST_INT 153 -#define UNPACK_SEQUENCE_LIST 154 -#define UNPACK_SEQUENCE_TUPLE 158 -#define UNPACK_SEQUENCE_TWO_TUPLE 159 +#define STORE_FAST__STORE_FAST 143 +#define STORE_SUBSCR_DICT 153 +#define STORE_SUBSCR_LIST_INT 154 +#define UNPACK_SEQUENCE_LIST 158 +#define UNPACK_SEQUENCE_TUPLE 159 +#define UNPACK_SEQUENCE_TWO_TUPLE 160 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index cbde9132342266..5a8cbb98ed3f56 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -429,6 +429,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a1 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) # Python 3.12a1 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) # Python 3.12a1 3515 (Embed jump mask in COMPARE_OP oparg) +# Python 3.12a1 3516 (Add COMAPRE_AND_BRANCH instruction) # Python 3.13 will start with 3550 @@ -441,7 +442,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3515).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3516).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 414faa35ee89d7..82556b8d2b4551 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -189,7 +189,8 @@ def pseudo_op(name, op, real_ops): def_op('DELETE_DEREF', 139) hasfree.append(139) jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) - +def_op('COMPARE_AND_BRANCH', 141) # Comparison and jump +hascompare.append(141) def_op('CALL_FUNCTION_EX', 142) # Flags def_op('EXTENDED_ARG', 144) @@ -309,10 +310,10 @@ def pseudo_op(name, op, real_ops): "CALL_NO_KW_TUPLE_1", "CALL_NO_KW_TYPE_1", ], - "COMPARE_OP": [ - "COMPARE_OP_FLOAT_JUMP", - "COMPARE_OP_INT_JUMP", - "COMPARE_OP_STR_JUMP", + "COMPARE_AND_BRANCH": [ + "COMPARE_AND_BRANCH_FLOAT", + "COMPARE_AND_BRANCH_INT", + "COMPARE_AND_BRANCH_STR", ], "FOR_ITER": [ "FOR_ITER_LIST", @@ -392,6 +393,9 @@ def pseudo_op(name, op, real_ops): "COMPARE_OP": { "counter": 1, }, + "COMPARE_AND_BRANCH": { + "counter": 1, + }, "BINARY_SUBSCR": { "counter": 1, "type_version": 2, diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index f74d2ed2c420b8..1606c9cfda3239 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1121,11 +1121,11 @@ def aug(): check_op_count(aug, "BUILD_SLICE", 0) def test_compare_positions(self): - for opname, op in [ - ("COMPARE_OP", "<"), - ("COMPARE_OP", "<="), - ("COMPARE_OP", ">"), - ("COMPARE_OP", ">="), + for opname_prefix, op in [ + ("COMPARE_", "<"), + ("COMPARE_", "<="), + ("COMPARE_", ">"), + ("COMPARE_", ">="), ("CONTAINS_OP", "in"), ("CONTAINS_OP", "not in"), ("IS_OP", "is"), @@ -1140,7 +1140,7 @@ def test_compare_positions(self): actual_positions = [ instruction.positions for instruction in dis.get_instructions(code) - if instruction.opname == opname + if instruction.opname.startswith(opname_prefix) ] with self.subTest(source): self.assertEqual(actual_positions, expected_positions) @@ -1270,7 +1270,7 @@ def test_multiline_boolean_expression(self): self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_FALSE', line=2, end_line=2, column=15, end_column=16, occurrence=2) # compare d and 0 - self.assertOpcodeSourcePositionIs(compiled_code, 'COMPARE_OP', + self.assertOpcodeSourcePositionIs(compiled_code, 'COMPARE_AND_BRANCH', line=4, end_line=4, column=8, end_column=13, occurrence=1) # jump if comparison it True self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 994bb370a5757f..cb7e22ff7b3051 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1561,12 +1561,12 @@ def _prepare_test_cases(): Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=60, starts_line=5, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=62, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=13, argval='<', argrepr='<', offset=64, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=64, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=72, argrepr='to 72', offset=68, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='JUMP_BACKWARD', opcode=140, arg=22, argval=28, argrepr='to 28', offset=70, starts_line=6, is_jump_target=False, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=72, starts_line=7, is_jump_target=True, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=74, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=68, argval='>', argrepr='>', offset=76, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=68, argval='>', argrepr='>', offset=76, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=84, argrepr='to 84', offset=80, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='JUMP_BACKWARD', opcode=140, arg=28, argval=28, argrepr='to 28', offset=82, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=84, starts_line=8, is_jump_target=True, positions=None), @@ -1588,12 +1588,12 @@ def _prepare_test_cases(): Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=154, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=156, starts_line=14, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=158, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=75, argval='>', argrepr='>', offset=160, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=75, argval='>', argrepr='>', offset=160, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=168, argrepr='to 168', offset=164, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='JUMP_BACKWARD', opcode=140, arg=26, argval=116, argrepr='to 116', offset=166, starts_line=15, is_jump_target=False, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=168, starts_line=16, is_jump_target=True, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=170, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=13, argval='<', argrepr='<', offset=172, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_AND_BRANCH', opcode=141, arg=13, argval='<', argrepr='<', offset=172, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=180, argrepr='to 180', offset=176, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='JUMP_FORWARD', opcode=110, arg=16, argval=212, argrepr='to 212', offset=178, starts_line=17, is_jump_target=False, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=180, starts_line=11, is_jump_target=True, positions=None), diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 39ccca70f9cbf3..6bc04bc8e848fc 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -357,6 +357,14 @@ mark_stacks(PyCodeObject *code_obj, int len) assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack); stacks[j] = next_stack; break; + case COMPARE_AND_BRANCH: + next_stack = pop_value(pop_value(next_stack)); + i++; + j = get_arg(code, i) + i + 1; + assert(j < len); + assert(stacks[j] == UNINITIALIZED || stacks[j] == next_stack); + stacks[j] = next_stack; + break; case GET_ITER: case GET_AITER: next_stack = push_value(pop_value(next_stack), Iterator); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index faa6df6a0a65ee..9544ed8a395eec 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -90,11 +90,6 @@ static size_t jump; // Dummy variables for cache effects static uint16_t invert, counter, index, hint; static uint32_t type_version; -// Dummy opcode names for 'op' opcodes -#define _COMPARE_OP_FLOAT 1003 -#define _COMPARE_OP_INT 1004 -#define _COMPARE_OP_STR 1005 -#define _JUMP_IF 1006 static PyObject * dummy_func( @@ -1829,13 +1824,6 @@ dummy_func( Py_DECREF(owner); } - family(compare_op) = { - COMPARE_OP, - _COMPARE_OP_FLOAT, - _COMPARE_OP_INT, - _COMPARE_OP_STR, - }; - inst(COMPARE_OP, (unused/1, left, right -- res)) { _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1853,40 +1841,67 @@ dummy_func( ERROR_IF(res == NULL, error); } - // The result is an int disguised as an object pointer. - op(_COMPARE_OP_FLOAT, (unused/1, left, right -- jump: size_t)) { + family(compare_and_branch) = { + COMPARE_AND_BRANCH, + COMPARE_AND_BRANCH_FLOAT, + COMPARE_AND_BRANCH_INT, + COMPARE_AND_BRANCH_STR, + }; + + inst(COMPARE_AND_BRANCH, (unused/2, left, right -- )) { + _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { + assert(cframe.use_tracing == 0); + next_instr--; + _Py_Specialize_CompareAndBranch(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(COMPARE_AND_BRANCH, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); + assert((oparg >> 4) <= Py_GE); + PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); + Py_DECREF(left); + Py_DECREF(right); + ERROR_IF(cond == NULL, error); + assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || + _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); + bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; + int offset = _Py_OPARG(next_instr[1]); + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); + if (err < 0) { + goto error; + } + if (jump_on_true == (err != 0)) { + JUMPBY(offset); + } + } + + inst(COMPARE_AND_BRANCH_FLOAT, (unused/2, left, right -- )) { assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_AND_BRANCH); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_AND_BRANCH); + STAT_INC(COMPARE_AND_BRANCH, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - jump = sign_ish & oparg; - } - // The input is an int disguised as an object pointer! - op(_JUMP_IF, (jump: size_t --)) { - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - if (jump) { - JUMPBY(oparg); + if (sign_ish & oparg) { + int offset = _Py_OPARG(next_instr[1]); + JUMPBY(offset); } } - // We're praying that the compiler optimizes the flags manipuations. - super(COMPARE_OP_FLOAT_JUMP) = _COMPARE_OP_FLOAT + _JUMP_IF; - // Similar to COMPARE_OP_FLOAT - op(_COMPARE_OP_INT, (unused/1, left, right -- jump: size_t)) { + // Similar to COMPARE_AND_BRANCH_FLOAT + inst(COMPARE_AND_BRANCH_INT, (unused/2, left, right -- )) { assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP); - STAT_INC(COMPARE_OP, hit); + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_AND_BRANCH); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_AND_BRANCH); + DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_AND_BRANCH); + DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_AND_BRANCH); + STAT_INC(COMPARE_AND_BRANCH, hit); assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; @@ -1894,17 +1909,18 @@ dummy_func( int sign_ish = COMPARISON_BIT(ileft, iright); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - jump = sign_ish & oparg; + if (sign_ish & oparg) { + int offset = _Py_OPARG(next_instr[1]); + JUMPBY(offset); + } } - super(COMPARE_OP_INT_JUMP) = _COMPARE_OP_INT + _JUMP_IF; - // Similar to COMPARE_OP_FLOAT, but for ==, != only - op(_COMPARE_OP_STR, (unused/1, left, right -- jump: size_t)) { + // Similar to COMPARE_AND_BRANCH_FLOAT, but for ==, != only + inst(COMPARE_AND_BRANCH_STR, (unused/2, left, right -- )) { assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_AND_BRANCH); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_AND_BRANCH); + STAT_INC(COMPARE_AND_BRANCH, hit); int res = _PyUnicode_Equal(left, right); assert((oparg >>4) == Py_EQ || (oparg >>4) == Py_NE); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -1912,11 +1928,12 @@ dummy_func( assert(res == 0 || res == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - jump = (res + COMPARISON_NOT_EQUALS) & oparg; + if ((res + COMPARISON_NOT_EQUALS) & oparg) { + int offset = _Py_OPARG(next_instr[1]); + JUMPBY(offset); + } } - super(COMPARE_OP_STR_JUMP) = _COMPARE_OP_STR + _JUMP_IF; - inst(IS_OP, (left, right -- b)) { int res = Py_Is(left, right) ^ oparg; DECREF_INPUTS(); diff --git a/Python/compile.c b/Python/compile.c index c0177fbf3e3374..f98892ec5fc6ac 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1195,6 +1195,9 @@ stack_effect(int opcode, int oparg, int jump) case POP_JUMP_IF_TRUE: return -1; + case COMPARE_AND_BRANCH: + return -2; + case LOAD_GLOBAL: return (oparg & 1) + 1; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9874ddf206d83a..67a93868b4bd21 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2077,7 +2077,6 @@ } TARGET(COMPARE_OP) { - PREDICTED(COMPARE_OP); PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *res; @@ -2101,112 +2100,108 @@ DISPATCH(); } - TARGET(COMPARE_OP_FLOAT_JUMP) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - size_t jump; + TARGET(COMPARE_AND_BRANCH) { + PREDICTED(COMPARE_AND_BRANCH); + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - jump = sign_ish & oparg; - _tmp_2 = (PyObject *)jump; + next_instr--; + _Py_Specialize_CompareAndBranch(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); } - JUMPBY(1); - NEXTOPARG(); - JUMPBY(1); - { - size_t jump = (size_t)_tmp_2; - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - if (jump) { - JUMPBY(oparg); - } + STAT_INC(COMPARE_AND_BRANCH, deferred); + DECREMENT_ADAPTIVE_COUNTER(cache->counter); + assert((oparg >> 4) <= Py_GE); + PyObject *cond = PyObject_RichCompare(left, right, oparg>>4); + Py_DECREF(left); + Py_DECREF(right); + if (cond == NULL) goto pop_2_error; + assert(_Py_OPCODE(next_instr[1]) == POP_JUMP_IF_FALSE || + _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE); + bool jump_on_true = _Py_OPCODE(next_instr[1]) == POP_JUMP_IF_TRUE; + int offset = _Py_OPARG(next_instr[1]); + int err = PyObject_IsTrue(cond); + Py_DECREF(cond); + if (err < 0) { + goto error; + } + if (jump_on_true == (err != 0)) { + JUMPBY(offset); } STACK_SHRINK(2); + JUMPBY(2); DISPATCH(); } - TARGET(COMPARE_OP_INT_JUMP) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - size_t jump; - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_OP); - DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); - Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; - Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - jump = sign_ish & oparg; - _tmp_2 = (PyObject *)jump; - } - JUMPBY(1); - NEXTOPARG(); - JUMPBY(1); - { - size_t jump = (size_t)_tmp_2; - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - if (jump) { - JUMPBY(oparg); - } + TARGET(COMPARE_AND_BRANCH_FLOAT) { + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + assert(cframe.use_tracing == 0); + DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_AND_BRANCH); + DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_AND_BRANCH); + STAT_INC(COMPARE_AND_BRANCH, hit); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + if (sign_ish & oparg) { + int offset = _Py_OPARG(next_instr[1]); + JUMPBY(offset); } STACK_SHRINK(2); + JUMPBY(2); DISPATCH(); } - TARGET(COMPARE_OP_STR_JUMP) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); - { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - size_t jump; - assert(cframe.use_tracing == 0); - // Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false) - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); - STAT_INC(COMPARE_OP, hit); - int res = _PyUnicode_Equal(left, right); - assert((oparg >>4) == Py_EQ || (oparg >>4) == Py_NE); - _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(res == 0 || res == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - jump = (res + COMPARISON_NOT_EQUALS) & oparg; - _tmp_2 = (PyObject *)jump; + TARGET(COMPARE_AND_BRANCH_INT) { + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + assert(cframe.use_tracing == 0); + DEOPT_IF(!PyLong_CheckExact(left), COMPARE_AND_BRANCH); + DEOPT_IF(!PyLong_CheckExact(right), COMPARE_AND_BRANCH); + DEOPT_IF((size_t)(Py_SIZE(left) + 1) > 2, COMPARE_AND_BRANCH); + DEOPT_IF((size_t)(Py_SIZE(right) + 1) > 2, COMPARE_AND_BRANCH); + STAT_INC(COMPARE_AND_BRANCH, hit); + assert(Py_ABS(Py_SIZE(left)) <= 1 && Py_ABS(Py_SIZE(right)) <= 1); + Py_ssize_t ileft = Py_SIZE(left) * ((PyLongObject *)left)->ob_digit[0]; + Py_ssize_t iright = Py_SIZE(right) * ((PyLongObject *)right)->ob_digit[0]; + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + if (sign_ish & oparg) { + int offset = _Py_OPARG(next_instr[1]); + JUMPBY(offset); } - JUMPBY(1); - NEXTOPARG(); - JUMPBY(1); - { - size_t jump = (size_t)_tmp_2; - assert(opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_TRUE); - if (jump) { - JUMPBY(oparg); - } + STACK_SHRINK(2); + JUMPBY(2); + DISPATCH(); + } + + TARGET(COMPARE_AND_BRANCH_STR) { + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + assert(cframe.use_tracing == 0); + DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_AND_BRANCH); + DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_AND_BRANCH); + STAT_INC(COMPARE_AND_BRANCH, hit); + int res = _PyUnicode_Equal(left, right); + assert((oparg >>4) == Py_EQ || (oparg >>4) == Py_NE); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(res == 0 || res == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + if ((res + COMPARISON_NOT_EQUALS) & oparg) { + int offset = _Py_OPARG(next_instr[1]); + JUMPBY(offset); } STACK_SHRINK(2); + JUMPBY(2); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 3316ea66b4ef04..dd0de7904cc4c1 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -2,7 +2,7 @@ // from Python/bytecodes.c // Do not edit! enum Direction { DIR_NONE, DIR_READ, DIR_WRITE }; -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC000, INSTR_FMT_IBCIB, INSTR_FMT_IBIB }; +enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC0, INSTR_FMT_IBC000, INSTR_FMT_IBIB }; static const struct { short n_popped; short n_pushed; @@ -113,9 +113,10 @@ static const struct { [STORE_ATTR_WITH_HINT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [STORE_ATTR_SLOT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [COMPARE_OP] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, - [COMPARE_OP_FLOAT_JUMP] = { 3, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBCIB }, - [COMPARE_OP_INT_JUMP] = { 3, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBCIB }, - [COMPARE_OP_STR_JUMP] = { 3, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBCIB }, + [COMPARE_AND_BRANCH] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, + [COMPARE_AND_BRANCH_FLOAT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, + [COMPARE_AND_BRANCH_INT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, + [COMPARE_AND_BRANCH_STR] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, [IS_OP] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CONTAINS_OP] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CHECK_EG_MATCH] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 83b3af7c0b1da8..f1c3f3e0c4ee17 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -47,7 +47,7 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_STR_1, &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_COMPARE_OP_FLOAT_JUMP, + &&TARGET_COMPARE_AND_BRANCH_FLOAT, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,8 +55,8 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, - &&TARGET_COMPARE_OP_INT_JUMP, - &&TARGET_COMPARE_OP_STR_JUMP, + &&TARGET_COMPARE_AND_BRANCH_INT, + &&TARGET_COMPARE_AND_BRANCH_STR, &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_TUPLE, &&TARGET_STORE_SUBSCR, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_COMPARE_AND_BRANCH, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,15 +152,15 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, diff --git a/Python/specialize.c b/Python/specialize.c index 38bb151e74710d..df7aea8a66a133 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -311,6 +311,7 @@ _PyCode_Quicken(PyCodeObject *code) if (opcode == POP_JUMP_IF_FALSE) { mask = mask ^ 0xf; } + instructions[i - 1 - INLINE_CACHE_ENTRIES_COMPARE_OP].opcode = COMPARE_AND_BRANCH; instructions[i - 1 - INLINE_CACHE_ENTRIES_COMPARE_OP].oparg = (oparg & 0xf0) | mask; break; } @@ -2003,52 +2004,58 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, { assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); + SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); + STAT_INC(COMPARE_OP, failure); + _py_set_opcode(instr, COMPARE_OP); + cache->counter = adaptive_counter_backoff(cache->counter); + return; +} + +void +_Py_Specialize_CompareAndBranch(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, + int oparg) +{ + assert(_PyOpcode_Caches[COMPARE_AND_BRANCH] == INLINE_CACHE_ENTRIES_COMPARE_OP); + _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]); - if (next_opcode != POP_JUMP_IF_FALSE && next_opcode != POP_JUMP_IF_TRUE) { - if (next_opcode == EXTENDED_ARG) { - SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_EXTENDED_ARG); - goto failure; - } - SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP); - goto failure; - } + assert(next_opcode == POP_JUMP_IF_FALSE || next_opcode == POP_JUMP_IF_TRUE); if (Py_TYPE(lhs) != Py_TYPE(rhs)) { - SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); + SPECIALIZATION_FAIL(COMPARE_AND_BRANCH, compare_op_fail_kind(lhs, rhs)); goto failure; } if (PyFloat_CheckExact(lhs)) { - _py_set_opcode(instr, COMPARE_OP_FLOAT_JUMP); + _py_set_opcode(instr, COMPARE_AND_BRANCH_FLOAT); goto success; } if (PyLong_CheckExact(lhs)) { if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) { - _py_set_opcode(instr, COMPARE_OP_INT_JUMP); + _py_set_opcode(instr, COMPARE_AND_BRANCH_INT); goto success; } else { - SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_BIG_INT); + SPECIALIZATION_FAIL(COMPARE_AND_BRANCH, SPEC_FAIL_COMPARE_OP_BIG_INT); goto failure; } } if (PyUnicode_CheckExact(lhs)) { int cmp = oparg >> 4; if (cmp != Py_EQ && cmp != Py_NE) { - SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_STRING); + SPECIALIZATION_FAIL(COMPARE_AND_BRANCH, SPEC_FAIL_COMPARE_OP_STRING); goto failure; } else { - _py_set_opcode(instr, COMPARE_OP_STR_JUMP); + _py_set_opcode(instr, COMPARE_AND_BRANCH_STR); goto success; } } - SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); + SPECIALIZATION_FAIL(COMPARE_AND_BRANCH, compare_op_fail_kind(lhs, rhs)); failure: - STAT_INC(COMPARE_OP, failure); - _py_set_opcode(instr, COMPARE_OP); + STAT_INC(COMPARE_AND_BRANCH, failure); + _py_set_opcode(instr, COMPARE_AND_BRANCH); cache->counter = adaptive_counter_backoff(cache->counter); return; success: - STAT_INC(COMPARE_OP, success); + STAT_INC(COMPARE_AND_BRANCH, success); cache->counter = adaptive_counter_cooldown(); } From 482777c0143f0528553073096b6198c8dc0d7fb4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 11 Jan 2023 15:46:19 +0000 Subject: [PATCH 2/9] Update summarize stats script to handle both COMPAREs. --- Python/specialize.c | 48 ++++++++++++++++---------------- Tools/scripts/summarize_stats.py | 2 ++ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index df7aea8a66a133..107bdb9e01d368 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -432,19 +432,19 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 /* COMPARE_OP */ -#define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 -#define SPEC_FAIL_COMPARE_OP_STRING 13 -#define SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP 14 -#define SPEC_FAIL_COMPARE_OP_BIG_INT 15 -#define SPEC_FAIL_COMPARE_OP_BYTES 16 -#define SPEC_FAIL_COMPARE_OP_TUPLE 17 -#define SPEC_FAIL_COMPARE_OP_LIST 18 -#define SPEC_FAIL_COMPARE_OP_SET 19 -#define SPEC_FAIL_COMPARE_OP_BOOL 20 -#define SPEC_FAIL_COMPARE_OP_BASEOBJECT 21 -#define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 22 -#define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 23 -#define SPEC_FAIL_COMPARE_OP_EXTENDED_ARG 24 +#define SPEC_FAIL_COMPARE_DIFFERENT_TYPES 12 +#define SPEC_FAIL_COMPARE_STRING 13 +#define SPEC_FAIL_COMPARE_NOT_FOLLOWED_BY_COND_JUMP 14 +#define SPEC_FAIL_COMPARE_BIG_INT 15 +#define SPEC_FAIL_COMPARE_BYTES 16 +#define SPEC_FAIL_COMPARE_TUPLE 17 +#define SPEC_FAIL_COMPARE_LIST 18 +#define SPEC_FAIL_COMPARE_SET 19 +#define SPEC_FAIL_COMPARE_BOOL 20 +#define SPEC_FAIL_COMPARE_BASEOBJECT 21 +#define SPEC_FAIL_COMPARE_FLOAT_LONG 22 +#define SPEC_FAIL_COMPARE_LONG_FLOAT 23 +#define SPEC_FAIL_COMPARE_EXTENDED_ARG 24 /* FOR_ITER */ #define SPEC_FAIL_FOR_ITER_GENERATOR 10 @@ -1969,30 +1969,30 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs) { if (Py_TYPE(lhs) != Py_TYPE(rhs)) { if (PyFloat_CheckExact(lhs) && PyLong_CheckExact(rhs)) { - return SPEC_FAIL_COMPARE_OP_FLOAT_LONG; + return SPEC_FAIL_COMPARE_FLOAT_LONG; } if (PyLong_CheckExact(lhs) && PyFloat_CheckExact(rhs)) { - return SPEC_FAIL_COMPARE_OP_LONG_FLOAT; + return SPEC_FAIL_COMPARE_LONG_FLOAT; } - return SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES; + return SPEC_FAIL_COMPARE_DIFFERENT_TYPES; } if (PyBytes_CheckExact(lhs)) { - return SPEC_FAIL_COMPARE_OP_BYTES; + return SPEC_FAIL_COMPARE_BYTES; } if (PyTuple_CheckExact(lhs)) { - return SPEC_FAIL_COMPARE_OP_TUPLE; + return SPEC_FAIL_COMPARE_TUPLE; } if (PyList_CheckExact(lhs)) { - return SPEC_FAIL_COMPARE_OP_LIST; + return SPEC_FAIL_COMPARE_LIST; } if (PySet_CheckExact(lhs) || PyFrozenSet_CheckExact(lhs)) { - return SPEC_FAIL_COMPARE_OP_SET; + return SPEC_FAIL_COMPARE_SET; } if (PyBool_Check(lhs)) { - return SPEC_FAIL_COMPARE_OP_BOOL; + return SPEC_FAIL_COMPARE_BOOL; } if (Py_TYPE(lhs)->tp_richcompare == PyBaseObject_Type.tp_richcompare) { - return SPEC_FAIL_COMPARE_OP_BASEOBJECT; + return SPEC_FAIL_COMPARE_BASEOBJECT; } return SPEC_FAIL_OTHER; } @@ -2033,14 +2033,14 @@ _Py_Specialize_CompareAndBranch(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *inst goto success; } else { - SPECIALIZATION_FAIL(COMPARE_AND_BRANCH, SPEC_FAIL_COMPARE_OP_BIG_INT); + SPECIALIZATION_FAIL(COMPARE_AND_BRANCH, SPEC_FAIL_COMPARE_BIG_INT); goto failure; } } if (PyUnicode_CheckExact(lhs)) { int cmp = oparg >> 4; if (cmp != Py_EQ && cmp != Py_NE) { - SPECIALIZATION_FAIL(COMPARE_AND_BRANCH, SPEC_FAIL_COMPARE_OP_STRING); + SPECIALIZATION_FAIL(COMPARE_AND_BRANCH, SPEC_FAIL_COMPARE_STRING); goto failure; } else { diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 1c8d10f7027727..4284b22eb80354 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -228,6 +228,8 @@ def kind_to_text(kind, defines, opname): return pretty(defines[kind][0]) if opname.endswith("ATTR"): opname = "ATTR" + if opname in ("COMPARE_OP", "COMPARE_AND_BRANCH"): + opname = "COMPARE" if opname.endswith("SUBSCR"): opname = "SUBSCR" for name in defines[kind]: From 358393d998abe714e69751e96e55c8e41a6be5bd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 12 Jan 2023 13:20:01 +0000 Subject: [PATCH 3/9] Remove attempted specialization of COMPARE_OP. --- Include/internal/pycore_code.h | 2 -- Python/bytecodes.c | 8 -------- Python/generated_cases.c.h | 8 -------- Python/specialize.c | 14 +------------- 4 files changed, 1 insertion(+), 31 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 582ffa615a1d7e..a287250acc1912 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -228,8 +228,6 @@ extern void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames); extern void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg, PyObject **locals); -extern void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, - _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_CompareAndBranch(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg); extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9544ed8a395eec..18f9eabd842cb0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1825,15 +1825,7 @@ dummy_func( } inst(COMPARE_OP, (unused/1, left, right -- res)) { - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); - next_instr--; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); Py_DECREF(left); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 67a93868b4bd21..e9819dfda997bb 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2080,15 +2080,7 @@ PyObject *right = PEEK(1); PyObject *left = PEEK(2); PyObject *res; - _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; - if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { - assert(cframe.use_tracing == 0); - next_instr--; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(cache->counter); assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); Py_DECREF(left); diff --git a/Python/specialize.c b/Python/specialize.c index 107bdb9e01d368..bd0ec29dc28f10 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -126,6 +126,7 @@ print_spec_stats(FILE *out, OpcodeStats *stats) /* Mark some opcodes as specializable for stats, * even though we don't specialize them yet. */ fprintf(out, "opcode[%d].specializable : 1\n", BINARY_SLICE); + fprintf(out, "opcode[%d].specializable : 1\n", COMPARE_OP); fprintf(out, "opcode[%d].specializable : 1\n", STORE_SLICE); for (int i = 0; i < 256; i++) { if (_PyOpcode_Caches[i]) { @@ -1998,19 +1999,6 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs) } #endif -void -_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, - int oparg) -{ - assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); - _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); - SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); - STAT_INC(COMPARE_OP, failure); - _py_set_opcode(instr, COMPARE_OP); - cache->counter = adaptive_counter_backoff(cache->counter); - return; -} - void _Py_Specialize_CompareAndBranch(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg) From c5c17647252c08b6304ca7790846fc00f165cdbd Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 12 Jan 2023 13:47:00 +0000 Subject: [PATCH 4/9] Add news --- .../2023-01-12-13-46-49.gh-issue-100982.mJ234s.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-01-12-13-46-49.gh-issue-100982.mJ234s.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-12-13-46-49.gh-issue-100982.mJ234s.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-12-13-46-49.gh-issue-100982.mJ234s.rst new file mode 100644 index 00000000000000..4f43e783cd6a19 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-01-12-13-46-49.gh-issue-100982.mJ234s.rst @@ -0,0 +1,4 @@ +Adds a new :opcode:`COMPARE_AND_BRANCH` instruction. This is a bit more +efficient when performing a comparison immediately followed by a branch, and +restores the design intent of PEP 659 that specializations are local to a +single instruction. From 944147c3e387d233b40105ab21c810ca42d4f859 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 12 Jan 2023 15:44:45 +0000 Subject: [PATCH 5/9] Document COMPARE_OP instruction. --- Doc/library/dis.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 334b9df4fc1f1a..38d02abfb570a5 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -954,6 +954,15 @@ iterations of the loop. ``cmp_op[opname]``. +.. opcode:: COMPARE_OP (opname) + + Compares the top two values on the stack, popping them, then branches. + The direction and offset of the jump is embedded as a ``POP_JUMP_IF_TRUE`` + or ``POP_JUMP_IF_FALSE`` instruction immediately following the cache. + + .. versionadded:: 3.12 + + .. opcode:: IS_OP (invert) Performs ``is`` comparison, or ``is not`` if ``invert`` is 1. From ef3fe457469d8497020af00d339999cfa59b4444 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 12 Jan 2023 16:12:54 +0000 Subject: [PATCH 6/9] Document the correct instruction --- Doc/library/dis.rst | 2 +- Lib/opcode.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 38d02abfb570a5..ce07cd076934fa 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -954,7 +954,7 @@ iterations of the loop. ``cmp_op[opname]``. -.. opcode:: COMPARE_OP (opname) +.. opcode:: COMPARE_AND_BRANCH (opname) Compares the top two values on the stack, popping them, then branches. The direction and offset of the jump is embedded as a ``POP_JUMP_IF_TRUE`` diff --git a/Lib/opcode.py b/Lib/opcode.py index 82556b8d2b4551..e2c420419faf6b 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -189,8 +189,9 @@ def pseudo_op(name, op, real_ops): def_op('DELETE_DEREF', 139) hasfree.append(139) jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) -def_op('COMPARE_AND_BRANCH', 141) # Comparison and jump +jrel_op('COMPARE_AND_BRANCH', 141) # Comparison and jump hascompare.append(141) + def_op('CALL_FUNCTION_EX', 142) # Flags def_op('EXTENDED_ARG', 144) From 447f63e5a59b9f97b236c9b3f260e386e42fbb69 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 12 Jan 2023 19:24:51 +0000 Subject: [PATCH 7/9] Regenerate file --- Include/internal/pycore_opcode.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 05c0485b0641d8..61ea4269a9b670 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -22,7 +22,7 @@ static const uint32_t _PyOpcode_RelativeJump[9] = { 0U, 536870912U, 135118848U, - 4163U, + 12355U, 0U, 0U, 0U, @@ -33,7 +33,7 @@ static const uint32_t _PyOpcode_Jump[9] = { 0U, 536870912U, 135118848U, - 4163U, + 12355U, 0U, 0U, 0U, From 7d3350885163d11068eb3268bb12e87e2ea9c052 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 12 Jan 2023 21:41:29 +0000 Subject: [PATCH 8/9] Revert change to COMPARE_AND_BRANCH classification. --- Include/internal/pycore_opcode.h | 4 ++-- Lib/opcode.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 61ea4269a9b670..05c0485b0641d8 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -22,7 +22,7 @@ static const uint32_t _PyOpcode_RelativeJump[9] = { 0U, 536870912U, 135118848U, - 12355U, + 4163U, 0U, 0U, 0U, @@ -33,7 +33,7 @@ static const uint32_t _PyOpcode_Jump[9] = { 0U, 536870912U, 135118848U, - 12355U, + 4163U, 0U, 0U, 0U, diff --git a/Lib/opcode.py b/Lib/opcode.py index e2c420419faf6b..46051b28a0f159 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -189,7 +189,7 @@ def pseudo_op(name, op, real_ops): def_op('DELETE_DEREF', 139) hasfree.append(139) jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) -jrel_op('COMPARE_AND_BRANCH', 141) # Comparison and jump +def_op('COMPARE_AND_BRANCH', 141) # Comparison and jump hascompare.append(141) def_op('CALL_FUNCTION_EX', 142) # Flags From 776ba5475652c7dee9b0a2acd06dc833675106b4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 16 Jan 2023 10:52:22 +0000 Subject: [PATCH 9/9] Avoid unused variable warning when NDEBUG is defined. --- Python/specialize.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/specialize.c b/Python/specialize.c index bd0ec29dc28f10..500fd714b38b3a 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2005,8 +2005,10 @@ _Py_Specialize_CompareAndBranch(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *inst { assert(_PyOpcode_Caches[COMPARE_AND_BRANCH] == INLINE_CACHE_ENTRIES_COMPARE_OP); _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); +#ifndef NDEBUG int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]); assert(next_opcode == POP_JUMP_IF_FALSE || next_opcode == POP_JUMP_IF_TRUE); +#endif if (Py_TYPE(lhs) != Py_TYPE(rhs)) { SPECIALIZATION_FAIL(COMPARE_AND_BRANCH, compare_op_fail_kind(lhs, rhs)); goto failure;