diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 926c0041c34c28..a405e01f66107d 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1297,6 +1297,7 @@ struct opcode_macro_expansion { #define OPARG_TOP 5 #define OPARG_BOTTOM 6 #define OPARG_SET_IP 7 +#define OPARG_SAVE_CURRENT_IP 8 #define OPCODE_METADATA_FMT(OP) (_PyOpcode_opcode_metadata[(OP)].instr_format) #define SAME_OPCODE_METADATA(OP1, OP2) \ @@ -1589,7 +1590,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [_JUMP_TO_TOP] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG }, [_SET_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [_SAVE_CURRENT_IP] = { true, INSTR_FMT_IX, 0 }, + [_SAVE_CURRENT_IP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [_EXIT_TRACE] = { true, INSTR_FMT_IX, 0 }, [_INSERT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, }; @@ -1644,8 +1645,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { CALL_INTRINSIC_2, 0, 0 } } }, - [RETURN_VALUE] = { .nuops = 2, .uops = { { _SAVE_CURRENT_IP, 7, -1 }, { _POP_FRAME, 0, 0 } } }, - [RETURN_CONST] = { .nuops = 3, .uops = { { LOAD_CONST, 0, 0 }, { _SAVE_CURRENT_IP, 7, -1 }, { _POP_FRAME, 0, 0 } } }, + [RETURN_VALUE] = { .nuops = 2, .uops = { { _SAVE_CURRENT_IP, 8, 0 }, { _POP_FRAME, 0, 0 } } }, + [RETURN_CONST] = { .nuops = 3, .uops = { { LOAD_CONST, 0, 0 }, { _SAVE_CURRENT_IP, 8, 0 }, { _POP_FRAME, 0, 0 } } }, [GET_AITER] = { .nuops = 1, .uops = { { GET_AITER, 0, 0 } } }, [GET_ANEXT] = { .nuops = 1, .uops = { { GET_ANEXT, 0, 0 } } }, [GET_AWAITABLE] = { .nuops = 1, .uops = { { GET_AWAITABLE, 0, 0 } } }, @@ -1719,8 +1720,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_CURRENT_IP, 7, 2 }, { _PUSH_FRAME, 0, 0 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_CURRENT_IP, 7, 2 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_CURRENT_IP, 8, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_CURRENT_IP, 8, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { CALL_TYPE_1, 0, 0 } } }, [CALL_STR_1] = { .nuops = 1, .uops = { { CALL_STR_1, 0, 0 } } }, [CALL_TUPLE_1] = { .nuops = 1, .uops = { { CALL_TUPLE_1, 0, 0 } } }, diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 44115da8629e42..04fe07fad39937 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -920,6 +920,10 @@ break; } + case _SAVE_CURRENT_IP: { + break; + } + case _EXIT_TRACE: { break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 56519636f8d675..76745d977bd094 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3961,12 +3961,18 @@ dummy_func( } op(_SAVE_CURRENT_IP, (--)) { - TIER_ONE_ONLY + #if TIER_ONE assert(frame->next_instr_offset == 0); frame->next_instr_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->next_instr_offset = oparg; + #endif } op(_EXIT_TRACE, (--)) { + TIER_TWO_ONLY + frame->next_instr_offset = 0; // Dispatch to frame->instr_ptr _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); diff --git a/Python/executor.c b/Python/executor.c index e38ffce0c3411d..07fc0849cb793c 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -131,6 +131,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject // The caller recovers the frame from tstate->current_frame. DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + frame->next_instr_offset = 0; // Don't leave this random _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); return NULL; @@ -140,6 +141,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject // This presumes nothing was popped from the stack (nor pushed). DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + frame->next_instr_offset = 0; // Dispatch to frame->instr_ptr _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); return frame; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 12b87a1182213e..6d7f8cac7a595a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3273,7 +3273,20 @@ break; } + case _SAVE_CURRENT_IP: { + #if TIER_ONE + assert(frame->next_instr_offset == 0); + frame->next_instr_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->next_instr_offset = oparg; + #endif + break; + } + case _EXIT_TRACE: { + TIER_TWO_ONLY + frame->next_instr_offset = 0; // Dispatch to frame->instr_ptr _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7e08a61bb73e11..479ef7c28d8f2a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -990,9 +990,13 @@ PyObject *retval; // _SAVE_CURRENT_IP { - TIER_ONE_ONLY + #if TIER_ONE assert(frame->next_instr_offset == 0); frame->next_instr_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->next_instr_offset = oparg; + #endif } // _POP_FRAME retval = stack_pointer[-1]; @@ -1051,9 +1055,13 @@ } // _SAVE_CURRENT_IP { - TIER_ONE_ONLY + #if TIER_ONE assert(frame->next_instr_offset == 0); frame->next_instr_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->next_instr_offset = oparg; + #endif } // _POP_FRAME retval = value; @@ -3941,9 +3949,13 @@ // _SAVE_CURRENT_IP next_instr += 3; { - TIER_ONE_ONLY + #if TIER_ONE assert(frame->next_instr_offset == 0); frame->next_instr_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->next_instr_offset = oparg; + #endif } // _PUSH_FRAME STACK_SHRINK(oparg); @@ -4014,9 +4026,13 @@ // _SAVE_CURRENT_IP next_instr += 3; { - TIER_ONE_ONLY + #if TIER_ONE assert(frame->next_instr_offset == 0); frame->next_instr_offset = (uint16_t)(next_instr - frame->instr_ptr); + #endif + #if TIER_TWO + frame->next_instr_offset = oparg; + #endif } // _PUSH_FRAME STACK_SHRINK(oparg); diff --git a/Python/optimizer.c b/Python/optimizer.c index 21335390c2dc1a..37907db6377dd6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -679,9 +679,13 @@ translate_bytecode_to_trace( case OPARG_BOTTOM: // Second half of super-instr oparg = orig_oparg & 0xF; break; - case OPARG_SET_IP: // op==_SET_IP; oparg=next instr + case OPARG_SET_IP: // op=_SET_IP; oparg=next instr oparg = INSTR_IP(instr + offset, code); - uop = _SET_IP; + assert(uop == _SET_IP); + break; + case OPARG_SAVE_CURRENT_IP: // op=_SAVE_CURRENT_IP; oparg=next_instr_offset + oparg = offset; + assert(uop == _SAVE_CURRENT_IP); break; default: diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index b2fa0205bea342..aeb8a82cc7735e 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -372,8 +372,8 @@ def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: case Instruction() as instr: part, offset = self.analyze_instruction(instr, offset) parts.append(part) - if instr.name != "_SET_IP": - # _SET_IP in a macro is a no-op in Tier 1 + if instr.name != "_SAVE_CURRENT_IP": + # _SAVE_CURRENT_IP's oparg does not transfer flags.add(instr.instr_flags) case _: assert_never(component) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 04d617ba9f5173..ef8acdf0c029b0 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -68,6 +68,7 @@ "OPARG_TOP": 5, "OPARG_BOTTOM": 6, "OPARG_SET_IP": 7, + "OPARG_SAVE_CURRENT_IP": 8, } INSTR_FMT_PREFIX = "INSTR_FMT_" @@ -658,7 +659,7 @@ def write_macro_expansions( for part in parts: if isinstance(part, Component): # All component instructions must be viable uops - if not part.instr.is_viable_uop() and part.instr.name != "_SAVE_CURRENT_IP": + if not part.instr.is_viable_uop(): # This note just reminds us about macros that cannot # be expanded to Tier 2 uops. It is not an error. # It is sometimes emitted for macros that have a @@ -672,7 +673,7 @@ def write_macro_expansions( return if not part.active_caches: if part.instr.name == "_SAVE_CURRENT_IP": - size, offset = OPARG_SIZES["OPARG_SET_IP"], cache_offset - 1 + size, offset = OPARG_SIZES["OPARG_SAVE_CURRENT_IP"], cache_offset else: size, offset = OPARG_SIZES["OPARG_FULL"], 0 else: