From eaebe1529d132cfcde699e33217c1e3e4f96bbc9 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sun, 5 Sep 2021 00:31:13 +0200 Subject: [PATCH 1/5] Allow contained indirections in tailcalls on x64 This adds support for contained indirections in tailcalls on x64. The significant diff of this change is refactoring to be able to reuse the code for generating call instructions when generating tailcalls as well. Other than that, the main change is to allow contained indirs in lowering and to ensure LSRA uses volatile registers for the addressing mode so that the registers are not overridden by the epilog sequence. To be sure we insert rex. prefix correctly I also refactored the emitter to base the decision on a new instruction (INS_i_jmp) and emit it when necessary. The rex. prefix is necessary because the unwinder uses it to determine that a tail jmp is non-local and thus part of the epilog. Finally, unlike the OS unwinder our unwinder needs to compute the size of the epilog. This computation was wrong for jmp [foo] for addressing modes/rip-relative addressing, so fix this as well. Presumably that has not been a problem before since we did not generate these instructions in managed code (although native code may have had these instructions -- not sure if we use that unwinder for native code). This PR helps support #56669. --- src/coreclr/jit/codegen.h | 25 +- src/coreclr/jit/codegenarmarch.cpp | 347 ++++++++++--------- src/coreclr/jit/codegencommon.cpp | 122 ++----- src/coreclr/jit/codegenlinear.cpp | 22 +- src/coreclr/jit/codegenxarch.cpp | 513 +++++++++++++++------------- src/coreclr/jit/emitarm.cpp | 2 +- src/coreclr/jit/emitxarch.cpp | 68 ++-- src/coreclr/jit/gentree.h | 2 +- src/coreclr/jit/instrsxarch.h | 13 +- src/coreclr/jit/lowerxarch.cpp | 43 ++- src/coreclr/jit/lsraxarch.cpp | 9 +- src/coreclr/vm/amd64/excepamd64.cpp | 22 +- 12 files changed, 598 insertions(+), 590 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index e88fcb59ca440..92f3d975073f0 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -453,19 +453,20 @@ class CodeGen final : public CodeGenInterface emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize), IL_OFFSETX ilOffset, - regNumber base = REG_NA, - bool isJump = false); + regNumber base, + bool isJump); // clang-format on // clang-format off - void genEmitCall(int callType, - CORINFO_METHOD_HANDLE methHnd, - INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo) - GenTreeIndir* indir - X86_ARG(int argSize), - emitAttr retSize - MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize), - IL_OFFSETX ilOffset); + void genEmitCallIndir(int callType, + CORINFO_METHOD_HANDLE methHnd, + INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo) + GenTreeIndir* indir + X86_ARG(int argSize), + emitAttr retSize + MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize), + IL_OFFSETX ilOffset, + bool isJump); // clang-format on // @@ -1260,7 +1261,9 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genCodeForArrOffset(GenTreeArrOffs* treeNode); instruction genGetInsForOper(genTreeOps oper, var_types type); bool genEmitOptimizedGCWriteBarrier(GCInfo::WriteBarrierForm writeBarrierForm, GenTree* addr, GenTree* data); - void genCallInstruction(GenTreeCall* call); + GenTree* getCallTarget(const GenTreeCall* call, CORINFO_METHOD_HANDLE* methHnd); + void genCall(GenTreeCall* call); + void genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackArgBytes)); void genJmpMethod(GenTree* jmp); BasicBlock* genCallFinally(BasicBlock* block); void genCodeForJumpTrue(GenTreeOp* jtrue); diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 0c3643880e408..719de94354159 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -405,7 +405,7 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) #endif // FEATURE_ARG_SPLIT case GT_CALL: - genCallInstruction(treeNode->AsCall()); + genCall(treeNode->AsCall()); break; case GT_MEMORYBARRIER: @@ -2258,17 +2258,10 @@ void CodeGen::genCodeForInitBlkHelper(GenTreeBlk* initBlkNode) } //------------------------------------------------------------------------ -// genCallInstruction: Produce code for a GT_CALL node +// genCall: Produce code for a GT_CALL node // -void CodeGen::genCallInstruction(GenTreeCall* call) +void CodeGen::genCall(GenTreeCall* call) { - gtCallTypes callType = (gtCallTypes)call->gtCallType; - - IL_OFFSETX ilOffset = BAD_IL_OFFSET; - - // all virtuals should have been expanded into a control expression - assert(!call->IsVirtual() || call->gtControlExpr || call->gtCallAddr); - // Consume all the arg regs for (GenTreeCall::Use& use : call->LateArgs()) { @@ -2343,46 +2336,20 @@ void CodeGen::genCallInstruction(GenTreeCall* call) #endif // TARGET* } - // Either gtControlExpr != null or gtCallAddr != null or it is a direct non-virtual call to a user or helper - // method. - CORINFO_METHOD_HANDLE methHnd; - GenTree* target = call->gtControlExpr; - if (callType == CT_INDIRECT) - { - assert(target == nullptr); - target = call->gtCallAddr; - methHnd = nullptr; - } - else - { - methHnd = call->gtCallMethHnd; - } - - CORINFO_SIG_INFO* sigInfo = nullptr; -#ifdef DEBUG - // Pass the call signature information down into the emitter so the emitter can associate - // native call sites with the signatures they were generated from. - if (callType != CT_HELPER) - { - sigInfo = call->callSig; - } -#endif // DEBUG - - // If fast tail call, then we are done. In this case we setup the args (both reg args - // and stack args in incoming arg area) and call target. Epilog sequence would - // generate "br ". + // If fast tail call, then we are done here, we just have to load the call + // target into the right registers. We ensure in RA that target is loaded + // into a volatile register that won't be restored by epilog sequence. if (call->IsFastTailCall()) { // Don't support fast tail calling JIT helpers - assert(callType != CT_HELPER); + assert(call->gtCallType != CT_HELPER); + + GenTree* target = getCallTarget(call, nullptr); if (target != nullptr) { // Indirect fast tail calls materialize call target either in gtControlExpr or in gtCallAddr. genConsumeReg(target); - - // Use IP0 on ARM64 and R12 on ARM32 as the call target register. - inst_Mov(TYP_I_IMPL, REG_FASTTAILCALL_TARGET, target->GetRegNum(), /* canSkip */ true); } return; @@ -2397,6 +2364,112 @@ void CodeGen::genCallInstruction(GenTreeCall* call) genDefineTempLabel(genCreateTempLabel()); } + genCallInstruction(call); + + // if it was a pinvoke we may have needed to get the address of a label + if (genPendingCallLabel) + { + genDefineInlineTempLabel(genPendingCallLabel); + genPendingCallLabel = nullptr; + } + + // Update GC info: + // All Callee arg registers are trashed and no longer contain any GC pointers. + // TODO-Bug?: As a matter of fact shouldn't we be killing all of callee trashed regs here? + // For now we will assert that other than arg regs gc ref/byref set doesn't contain any other + // registers from RBM_CALLEE_TRASH + assert((gcInfo.gcRegGCrefSetCur & (RBM_CALLEE_TRASH & ~RBM_ARG_REGS)) == 0); + assert((gcInfo.gcRegByrefSetCur & (RBM_CALLEE_TRASH & ~RBM_ARG_REGS)) == 0); + gcInfo.gcRegGCrefSetCur &= ~RBM_ARG_REGS; + gcInfo.gcRegByrefSetCur &= ~RBM_ARG_REGS; + + var_types returnType = call->TypeGet(); + if (returnType != TYP_VOID) + { + regNumber returnReg; + + if (call->HasMultiRegRetVal()) + { + const ReturnTypeDesc* pRetTypeDesc = call->GetReturnTypeDesc(); + assert(pRetTypeDesc != nullptr); + unsigned regCount = pRetTypeDesc->GetReturnRegCount(); + + // If regs allocated to call node are different from ABI return + // regs in which the call has returned its result, move the result + // to regs allocated to call node. + for (unsigned i = 0; i < regCount; ++i) + { + var_types regType = pRetTypeDesc->GetReturnRegType(i); + returnReg = pRetTypeDesc->GetABIReturnReg(i); + regNumber allocatedReg = call->GetRegNumByIdx(i); + inst_Mov(regType, allocatedReg, returnReg, /* canSkip */ true); + } + } + else + { +#ifdef TARGET_ARM + if (call->IsHelperCall(compiler, CORINFO_HELP_INIT_PINVOKE_FRAME)) + { + // The CORINFO_HELP_INIT_PINVOKE_FRAME helper uses a custom calling convention that returns with + // TCB in REG_PINVOKE_TCB. fgMorphCall() sets the correct argument registers. + returnReg = REG_PINVOKE_TCB; + } + else if (compiler->opts.compUseSoftFP) + { + returnReg = REG_INTRET; + } + else +#endif // TARGET_ARM + if (varTypeUsesFloatArgReg(returnType)) + { + returnReg = REG_FLOATRET; + } + else + { + returnReg = REG_INTRET; + } + + if (call->GetRegNum() != returnReg) + { +#ifdef TARGET_ARM + if (compiler->opts.compUseSoftFP && returnType == TYP_DOUBLE) + { + inst_RV_RV_RV(INS_vmov_i2d, call->GetRegNum(), returnReg, genRegArgNext(returnReg), EA_8BYTE); + } + else if (compiler->opts.compUseSoftFP && returnType == TYP_FLOAT) + { + inst_Mov(returnType, call->GetRegNum(), returnReg, /* canSkip */ false); + } + else +#endif + { + inst_Mov(returnType, call->GetRegNum(), returnReg, /* canSkip */ false); + } + } + } + + genProduceReg(call); + } + + // If there is nothing next, that means the result is thrown away, so this value is not live. + // However, for minopts or debuggable code, we keep it live to support managed return value debugging. + if ((call->gtNext == nullptr) && !compiler->opts.MinOpts() && !compiler->opts.compDbgCode) + { + gcInfo.gcMarkRegSetNpt(RBM_INTRET); + } +} + +//------------------------------------------------------------------------ +// genCallInstruction - Generate instructions necessary to transfer control to the call. +// +// Arguments: +// call - the GT_CALL node +// +// Remaks: +// For tailcalls this function will generate a jump. +// +void CodeGen::genCallInstruction(GenTreeCall* call) +{ // Determine return value size(s). const ReturnTypeDesc* pRetTypeDesc = call->GetReturnTypeDesc(); emitAttr retSize = EA_PTRSIZE; @@ -2425,31 +2498,60 @@ void CodeGen::genCallInstruction(GenTreeCall* call) // an IL to native mapping record for the call, to support managed return value debugging. // We don't want tail call helper calls that were converted from normal calls to get a record, // so we skip this hash table lookup logic in that case. + + IL_OFFSETX ilOffset = BAD_IL_OFFSET; + if (compiler->opts.compDbgInfo && compiler->genCallSite2ILOffsetMap != nullptr && !call->IsTailCall()) { (void)compiler->genCallSite2ILOffsetMap->Lookup(call, &ilOffset); } + CORINFO_SIG_INFO* sigInfo = nullptr; +#ifdef DEBUG + // Pass the call signature information down into the emitter so the emitter can associate + // native call sites with the signatures they were generated from. + if (call->gtCallType != CT_HELPER) + { + sigInfo = call->callSig; + } +#endif // DEBUG + CORINFO_METHOD_HANDLE methHnd; + GenTree* target = getCallTarget(call, &methHnd); + if (target != nullptr) { // A call target can not be a contained indirection assert(!target->isContainedIndir()); - genConsumeReg(target); + // For fast tailcall we have already consumed the target. We ensure in + // RA that the target was allocated into a volatile register that will + // not be messed up by epilog sequence. + if (!call->IsFastTailCall()) + { + genConsumeReg(target); + } // We have already generated code for gtControlExpr evaluating it into a register. // We just need to emit "call reg" in this case. // assert(genIsValidIntReg(target->GetRegNum())); - genEmitCall(emitter::EC_INDIR_R, methHnd, - INDEBUG_LDISASM_COMMA(sigInfo) nullptr, // addr - retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), ilOffset, target->GetRegNum()); + // clang-format off + genEmitCall(emitter::EC_INDIR_R, + methHnd, + INDEBUG_LDISASM_COMMA(sigInfo) + nullptr, // addr + retSize + MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), + ilOffset, + target->GetRegNum(), + call->IsFastTailCall()); + // clang-format on } else if (call->IsR2ROrVirtualStubRelativeIndir()) { // Generate a indirect call to a virtual user defined function or helper method - assert(callType == CT_HELPER || callType == CT_USER_FUNC); + assert(call->gtCallType == CT_HELPER || call->gtCallType == CT_USER_FUNC); #ifdef FEATURE_READYTORUN assert(((call->IsR2RRelativeIndir()) && (call->gtEntryPoint.accessType == IAT_PVALUE)) || ((call->IsVirtualStubRelativeIndir()) && (call->gtEntryPoint.accessType == IAT_VALUE))); @@ -2467,14 +2569,22 @@ void CodeGen::genCallInstruction(GenTreeCall* call) // assert(genIsValidIntReg(tmpReg)); - genEmitCall(emitter::EC_INDIR_R, methHnd, - INDEBUG_LDISASM_COMMA(sigInfo) nullptr, // addr - retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), ilOffset, tmpReg); + // clang-format off + genEmitCall(emitter::EC_INDIR_R, + methHnd, + INDEBUG_LDISASM_COMMA(sigInfo) + nullptr, // addr + retSize + MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), + ilOffset, + tmpReg, + call->IsFastTailCall()); + // clang-format on } else { // Generate a direct call to a non-virtual user defined or helper method - assert(callType == CT_HELPER || callType == CT_USER_FUNC); + assert(call->gtCallType == CT_HELPER || call->gtCallType == CT_USER_FUNC); void* addr = nullptr; #ifdef FEATURE_READYTORUN @@ -2485,7 +2595,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) } else #endif // FEATURE_READYTORUN - if (callType == CT_HELPER) + if (call->gtCallType == CT_HELPER) { CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd); noway_assert(helperNum != CORINFO_HELP_UNDEF); @@ -2508,123 +2618,32 @@ void CodeGen::genCallInstruction(GenTreeCall* call) { regNumber tmpReg = call->GetSingleTempReg(); instGen_Set_Reg_To_Imm(EA_HANDLE_CNS_RELOC, tmpReg, (ssize_t)addr); - genEmitCall(emitter::EC_INDIR_R, methHnd, INDEBUG_LDISASM_COMMA(sigInfo) NULL, retSize, ilOffset, tmpReg); + // clang-format off + genEmitCall(emitter::EC_INDIR_R, + methHnd, + INDEBUG_LDISASM_COMMA(sigInfo) + NULL, + retSize, + ilOffset, + tmpReg, + call->IsFastTailCall()); + // clang-format on } else #endif // TARGET_ARM { - genEmitCall(emitter::EC_FUNC_TOKEN, methHnd, INDEBUG_LDISASM_COMMA(sigInfo) addr, - retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), ilOffset); - } -#if 0 && defined(TARGET_ARM64) - // Use this path if you want to load an absolute call target using - // a sequence of movs followed by an indirect call (blr instruction) - // If this path is enabled, we need to ensure that REG_IP0 is assigned during Lowering. - - // Load the call target address in x16 - instGen_Set_Reg_To_Imm(EA_8BYTE, REG_IP0, (ssize_t) addr); - - // indirect call to constant address in IP0 - genEmitCall(emitter::EC_INDIR_R, - methHnd, - INDEBUG_LDISASM_COMMA(sigInfo) - nullptr, //addr - retSize, - secondRetSize, - ilOffset, - REG_IP0); -#endif - } - - // if it was a pinvoke we may have needed to get the address of a label - if (genPendingCallLabel) - { - genDefineInlineTempLabel(genPendingCallLabel); - genPendingCallLabel = nullptr; - } - - // Update GC info: - // All Callee arg registers are trashed and no longer contain any GC pointers. - // TODO-Bug?: As a matter of fact shouldn't we be killing all of callee trashed regs here? - // For now we will assert that other than arg regs gc ref/byref set doesn't contain any other - // registers from RBM_CALLEE_TRASH - assert((gcInfo.gcRegGCrefSetCur & (RBM_CALLEE_TRASH & ~RBM_ARG_REGS)) == 0); - assert((gcInfo.gcRegByrefSetCur & (RBM_CALLEE_TRASH & ~RBM_ARG_REGS)) == 0); - gcInfo.gcRegGCrefSetCur &= ~RBM_ARG_REGS; - gcInfo.gcRegByrefSetCur &= ~RBM_ARG_REGS; - - var_types returnType = call->TypeGet(); - if (returnType != TYP_VOID) - { - regNumber returnReg; - - if (call->HasMultiRegRetVal()) - { - assert(pRetTypeDesc != nullptr); - unsigned regCount = pRetTypeDesc->GetReturnRegCount(); - - // If regs allocated to call node are different from ABI return - // regs in which the call has returned its result, move the result - // to regs allocated to call node. - for (unsigned i = 0; i < regCount; ++i) - { - var_types regType = pRetTypeDesc->GetReturnRegType(i); - returnReg = pRetTypeDesc->GetABIReturnReg(i); - regNumber allocatedReg = call->GetRegNumByIdx(i); - inst_Mov(regType, allocatedReg, returnReg, /* canSkip */ true); - } - } - else - { -#ifdef TARGET_ARM - if (call->IsHelperCall(compiler, CORINFO_HELP_INIT_PINVOKE_FRAME)) - { - // The CORINFO_HELP_INIT_PINVOKE_FRAME helper uses a custom calling convention that returns with - // TCB in REG_PINVOKE_TCB. fgMorphCall() sets the correct argument registers. - returnReg = REG_PINVOKE_TCB; - } - else if (compiler->opts.compUseSoftFP) - { - returnReg = REG_INTRET; - } - else -#endif // TARGET_ARM - if (varTypeUsesFloatArgReg(returnType)) - { - returnReg = REG_FLOATRET; - } - else - { - returnReg = REG_INTRET; - } - - if (call->GetRegNum() != returnReg) - { -#ifdef TARGET_ARM - if (compiler->opts.compUseSoftFP && returnType == TYP_DOUBLE) - { - inst_RV_RV_RV(INS_vmov_i2d, call->GetRegNum(), returnReg, genRegArgNext(returnReg), EA_8BYTE); - } - else if (compiler->opts.compUseSoftFP && returnType == TYP_FLOAT) - { - inst_Mov(returnType, call->GetRegNum(), returnReg, /* canSkip */ false); - } - else -#endif - { - inst_Mov(returnType, call->GetRegNum(), returnReg, /* canSkip */ false); - } - } + // clang-format off + genEmitCall(emitter::EC_FUNC_TOKEN, + methHnd, + INDEBUG_LDISASM_COMMA(sigInfo) + addr, + retSize + MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), + ilOffset, + REG_NA, + call->IsFastTailCall()); + // clang-format on } - - genProduceReg(call); - } - - // If there is nothing next, that means the result is thrown away, so this value is not live. - // However, for minopts or debuggable code, we keep it live to support managed return value debugging. - if ((call->gtNext == nullptr) && !compiler->opts.MinOpts() && !compiler->opts.compDbgCode) - { - gcInfo.gcMarkRegSetNpt(RBM_INTRET); } } diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 29a4696a56623..42297370a0fbe 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -7756,6 +7756,41 @@ void CodeGen::genFnProlog() #pragma warning(pop) #endif +//------------------------------------------------------------------------ +// getCallTarget - Get the node that evalutes to the call target +// +// Arguments: +// call - the GT_CALL node +// +// Returns: +// The node. Note that for direct calls this may still return non-null if the direct call +// requires a 'complex' tree to load the target (e.g. in R2R or because we go through a stub). +// +GenTree* CodeGen::getCallTarget(const GenTreeCall* call, CORINFO_METHOD_HANDLE* methHnd) +{ + // all virtuals should have been expanded into a control expression by this point. + assert(!call->IsVirtual() || call->gtControlExpr || call->gtCallAddr); + + if (call->gtCallType == CT_INDIRECT) + { + assert(call->gtControlExpr == nullptr); + + if (methHnd != nullptr) + { + *methHnd = nullptr; + } + + return call->gtCallAddr; + } + + if (methHnd != nullptr) + { + *methHnd = call->gtCallMethHnd; + } + + return call->gtControlExpr; +} + /***************************************************************************** * * Generates code for a function epilog. @@ -8017,46 +8052,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) #if FEATURE_FASTTAILCALL else { - // Fast tail call. - GenTreeCall* call = jmpNode->AsCall(); - gtCallTypes callType = (gtCallTypes)call->gtCallType; - - // Fast tail calls cannot happen to helpers. - assert((callType == CT_INDIRECT) || (callType == CT_USER_FUNC)); - - // Try to dispatch this as a direct branch; this is possible when the call is - // truly direct. In this case, the control expression will be null and the direct - // target address will be in gtDirectCallAddress. It is still possible that calls - // to user funcs require indirection, in which case the control expression will - // be non-null. - if ((callType == CT_USER_FUNC) && (call->gtControlExpr == nullptr)) - { - assert(call->gtCallMethHnd != nullptr); - // clang-format off - GetEmitter()->emitIns_Call(emitter::EC_FUNC_TOKEN, - call->gtCallMethHnd, - INDEBUG_LDISASM_COMMA(nullptr) - call->gtDirectCallAddress, - 0, // argSize - EA_UNKNOWN // retSize - ARM64_ARG(EA_UNKNOWN), // secondRetSize - gcInfo.gcVarPtrSetCur, - gcInfo.gcRegGCrefSetCur, - gcInfo.gcRegByrefSetCur, - BAD_IL_OFFSET, // IL offset - REG_NA, // ireg - REG_NA, // xreg - 0, // xmul - 0, // disp - true); // isJump - // clang-format on - } - else - { - // Target requires indirection to obtain. genCallInstruction will have materialized - // it into REG_FASTTAILCALL_TARGET already, so just branch to it. - GetEmitter()->emitIns_R(INS_br, emitTypeSize(TYP_I_IMPL), REG_FASTTAILCALL_TARGET); - } + genCallInstruction(jmpNode->AsCall()); } #endif // FEATURE_FASTTAILCALL } @@ -8452,51 +8448,7 @@ void CodeGen::genFnEpilog(BasicBlock* block) #if FEATURE_FASTTAILCALL else { -#ifdef TARGET_AMD64 - // Fast tail call. - GenTreeCall* call = jmpNode->AsCall(); - gtCallTypes callType = (gtCallTypes)call->gtCallType; - - // Fast tail calls cannot happen to helpers. - assert((callType == CT_INDIRECT) || (callType == CT_USER_FUNC)); - - // Calls to a user func can be dispatched as an RIP-relative jump when they are - // truly direct; in this case, the control expression will be null and the direct - // target address will be in gtDirectCallAddress. It is still possible that calls - // to user funcs require indirection, in which case the control expression will - // be non-null. - if ((callType == CT_USER_FUNC) && (call->gtControlExpr == nullptr)) - { - assert(call->gtCallMethHnd != nullptr); - // clang-format off - GetEmitter()->emitIns_Call( - emitter::EC_FUNC_TOKEN, - call->gtCallMethHnd, - INDEBUG_LDISASM_COMMA(nullptr) - call->gtDirectCallAddress, - 0, // argSize - EA_UNKNOWN // retSize - MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(EA_UNKNOWN),// secondRetSize - gcInfo.gcVarPtrSetCur, - gcInfo.gcRegGCrefSetCur, - gcInfo.gcRegByrefSetCur, - BAD_IL_OFFSET, REG_NA, REG_NA, 0, 0, /* iloffset, ireg, xreg, xmul, disp */ - true /* isJump */ - ); - // clang-format on - } - else - { - // Target requires indirection to obtain. genCallInstruction will have materialized - // it into RAX already, so just jump to it. The stack walker requires that a register - // indirect tail call be rex.w prefixed. - GetEmitter()->emitIns_R(INS_rex_jmp, emitTypeSize(TYP_I_IMPL), REG_RAX); - } - -#else - assert(!"Fast tail call as epilog+jmp"); - unreached(); -#endif // TARGET_AMD64 + genCallInstruction(jmpNode->AsCall()); } #endif // FEATURE_FASTTAILCALL } diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index f58a8db0997e3..9e6500cd08aa9 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -2325,20 +2325,19 @@ void CodeGen::genEmitCall(int callType, // retSize - emitter type of return for GC purposes, should be EA_BYREF, EA_GCREF, or EA_PTRSIZE(not GC) // // clang-format off -void CodeGen::genEmitCall(int callType, - CORINFO_METHOD_HANDLE methHnd, - INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo) - GenTreeIndir* indir - X86_ARG(int argSize), - emitAttr retSize - MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize), - IL_OFFSETX ilOffset) +void CodeGen::genEmitCallIndir(int callType, + CORINFO_METHOD_HANDLE methHnd, + INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo) + GenTreeIndir* indir + X86_ARG(int argSize), + emitAttr retSize + MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize), + IL_OFFSETX ilOffset, + bool isJump) { #if !defined(TARGET_X86) int argSize = 0; #endif // !defined(TARGET_X86) - genConsumeAddress(indir->Addr()); - GetEmitter()->emitIns_Call(emitter::EmitCallType(callType), methHnd, INDEBUG_LDISASM_COMMA(sigInfo) @@ -2353,7 +2352,8 @@ void CodeGen::genEmitCall(int callType, (indir->Base() != nullptr) ? indir->Base()->GetRegNum() : REG_NA, (indir->Index() != nullptr) ? indir->Index()->GetRegNum() : REG_NA, indir->Scale(), - indir->Offset()); + indir->Offset(), + isJump); } // clang-format on diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 7ed36e7f68aea..0a64a67ac7d39 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1715,7 +1715,7 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) break; case GT_CALL: - genCallInstruction(treeNode->AsCall()); + genCall(treeNode->AsCall()); break; case GT_JMP: @@ -5030,14 +5030,10 @@ bool CodeGen::genEmitOptimizedGCWriteBarrier(GCInfo::WriteBarrierForm writeBarri } // Produce code for a GT_CALL node -void CodeGen::genCallInstruction(GenTreeCall* call) +void CodeGen::genCall(GenTreeCall* call) { genAlignStackBeforeCall(call); - gtCallTypes callType = (gtCallTypes)call->gtCallType; - - IL_OFFSETX ilOffset = BAD_IL_OFFSET; - // all virtuals should have been expanded into a control expression assert(!call->IsVirtual() || call->gtControlExpr || call->gtCallAddr); @@ -5156,45 +5152,27 @@ void CodeGen::genCallInstruction(GenTreeCall* call) GetEmitter()->emitIns_AR_R(INS_cmp, EA_4BYTE, regThis, regThis, 0); } - // Either gtControlExpr != null or gtCallAddr != null or it is a direct non-virtual call to a user or helper method. - CORINFO_METHOD_HANDLE methHnd; - GenTree* target = call->gtControlExpr; - if (callType == CT_INDIRECT) - { - assert(target == nullptr); - target = call->gtCallAddr; - methHnd = nullptr; - } - else - { - methHnd = call->gtCallMethHnd; - } - - CORINFO_SIG_INFO* sigInfo = nullptr; -#ifdef DEBUG - // Pass the call signature information down into the emitter so the emitter can associate - // native call sites with the signatures they were generated from. - if (callType != CT_HELPER) - { - sigInfo = call->callSig; - } -#endif // DEBUG - - // If fast tail call, then we are done. In this case we setup the args (both reg args - // and stack args in incoming arg area) and call target in rax. Epilog sequence would - // generate "jmp rax". + // If fast tail call, then we are done here, we just have to load the call + // target into the right registers. We ensure in RA that the registers used + // for the target (e.g. contained indir) are loaded into volatile registers + // that won't be restored by epilog sequence. if (call->IsFastTailCall()) { // Don't support fast tail calling JIT helpers - assert(callType != CT_HELPER); + assert(call->gtCallType != CT_HELPER); - // If this is indirect then we go through RAX with epilog sequence - // generating "jmp rax". Otherwise epilog will try to generate a - // rip-relative jump. + GenTree* target = getCallTarget(call, nullptr); if (target != nullptr) { - genConsumeReg(target); - genCopyRegIfNeeded(target, REG_RAX); + if (target->isContainedIndir()) + { + genConsumeAddress(target->AsIndir()->Addr()); + } + else + { + assert(!target->isContained()); + genConsumeReg(target); + } } return; @@ -5209,30 +5187,6 @@ void CodeGen::genCallInstruction(GenTreeCall* call) genDefineTempLabel(genCreateTempLabel()); } - // Determine return value size(s). - const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc(); - emitAttr retSize = EA_PTRSIZE; - emitAttr secondRetSize = EA_UNKNOWN; - - if (call->HasMultiRegRetVal()) - { - retSize = emitTypeSize(retTypeDesc->GetReturnRegType(0)); - secondRetSize = emitTypeSize(retTypeDesc->GetReturnRegType(1)); - } - else - { - assert(!varTypeIsStruct(call)); - - if (call->gtType == TYP_REF) - { - retSize = EA_GCREF; - } - else if (call->gtType == TYP_BYREF) - { - retSize = EA_BYREF; - } - } - #if defined(DEBUG) && defined(TARGET_X86) // Store the stack pointer so we can check it after the call. if (compiler->opts.compStackCheckOnCall && call->gtCallType == CT_USER_FUNC) @@ -5244,32 +5198,6 @@ void CodeGen::genCallInstruction(GenTreeCall* call) } #endif // defined(DEBUG) && defined(TARGET_X86) - bool fPossibleSyncHelperCall = false; - CorInfoHelpFunc helperNum = CORINFO_HELP_UNDEF; - - // We need to propagate the IL offset information to the call instruction, so we can emit - // an IL to native mapping record for the call, to support managed return value debugging. - // We don't want tail call helper calls that were converted from normal calls to get a record, - // so we skip this hash table lookup logic in that case. - if (compiler->opts.compDbgInfo && compiler->genCallSite2ILOffsetMap != nullptr && !call->IsTailCall()) - { - (void)compiler->genCallSite2ILOffsetMap->Lookup(call, &ilOffset); - } - -#if defined(TARGET_X86) - bool fCallerPop = call->CallerPop(); - - // If the callee pops the arguments, we pass a positive value as the argSize, and the emitter will - // adjust its stack level accordingly. - // If the caller needs to explicitly pop its arguments, we must pass a negative value, and then do the - // pop when we're done. - target_ssize_t argSizeForEmitter = stackArgBytes; - if (fCallerPop) - { - argSizeForEmitter = -stackArgBytes; - } -#endif // defined(TARGET_X86) - // When it's a PInvoke call and the call type is USER function, we issue VZEROUPPER here // if the function contains 256bit AVX instructions, this is to avoid AVX-256 to Legacy SSE // transition penalty, assuming the user function contains legacy SSE instruction. @@ -5282,160 +5210,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) instGen(INS_vzeroupper); } - if (callType == CT_HELPER && compiler->info.compFlags & CORINFO_FLG_SYNCH) - { - fPossibleSyncHelperCall = true; - helperNum = compiler->eeGetHelperNum(methHnd); - noway_assert(helperNum != CORINFO_HELP_UNDEF); - } - - if (target != nullptr) - { -#ifdef TARGET_X86 - if (call->IsVirtualStub() && (call->gtCallType == CT_INDIRECT)) - { - // On x86, we need to generate a very specific pattern for indirect VSD calls: - // - // 3-byte nop - // call dword ptr [eax] - // - // Where EAX is also used as an argument to the stub dispatch helper. Make - // sure that the call target address is computed into EAX in this case. - - assert(compiler->virtualStubParamInfo->GetReg() == REG_VIRTUAL_STUB_TARGET); - - assert(target->isContainedIndir()); - assert(target->OperGet() == GT_IND); - - GenTree* addr = target->AsIndir()->Addr(); - assert(addr->isUsedFromReg()); - - genConsumeReg(addr); - genCopyRegIfNeeded(addr, REG_VIRTUAL_STUB_TARGET); - - GetEmitter()->emitIns_Nop(3); - - // clang-format off - GetEmitter()->emitIns_Call(emitter::EmitCallType(emitter::EC_INDIR_ARD), - methHnd, - INDEBUG_LDISASM_COMMA(sigInfo) - nullptr, - argSizeForEmitter, - retSize - MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), - gcInfo.gcVarPtrSetCur, - gcInfo.gcRegGCrefSetCur, - gcInfo.gcRegByrefSetCur, - ilOffset, REG_VIRTUAL_STUB_TARGET, REG_NA, 1, 0); - // clang-format on - } - else -#endif - if (target->isContainedIndir()) - { - if (target->AsIndir()->HasBase() && target->AsIndir()->Base()->isContainedIntOrIImmed()) - { - // Note that if gtControlExpr is an indir of an absolute address, we mark it as - // contained only if it can be encoded as PC-relative offset. - assert(target->AsIndir()->Base()->AsIntConCommon()->FitsInAddrBase(compiler)); - - // clang-format off - genEmitCall(emitter::EC_FUNC_TOKEN_INDIR, - methHnd, - INDEBUG_LDISASM_COMMA(sigInfo) - (void*) target->AsIndir()->Base()->AsIntConCommon()->IconValue() - X86_ARG(argSizeForEmitter), - retSize - MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), - ilOffset); - // clang-format on - } - else - { - // clang-format off - genEmitCall(emitter::EC_INDIR_ARD, - methHnd, - INDEBUG_LDISASM_COMMA(sigInfo) - target->AsIndir() - X86_ARG(argSizeForEmitter), - retSize - MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), - ilOffset); - // clang-format on - } - } - else - { - // We have already generated code for gtControlExpr evaluating it into a register. - // We just need to emit "call reg" in this case. - assert(genIsValidIntReg(target->GetRegNum())); - - // clang-format off - genEmitCall(emitter::EC_INDIR_R, - methHnd, - INDEBUG_LDISASM_COMMA(sigInfo) - nullptr // addr - X86_ARG(argSizeForEmitter), - retSize - MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), - ilOffset, - genConsumeReg(target)); - // clang-format on - } - } -#ifdef FEATURE_READYTORUN - else if (call->gtEntryPoint.addr != nullptr) - { - // clang-format off - genEmitCall((call->gtEntryPoint.accessType == IAT_VALUE) ? emitter::EC_FUNC_TOKEN - : emitter::EC_FUNC_TOKEN_INDIR, - methHnd, - INDEBUG_LDISASM_COMMA(sigInfo) - (void*) call->gtEntryPoint.addr - X86_ARG(argSizeForEmitter), - retSize - MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), - ilOffset); - // clang-format on - } -#endif - else - { - // Generate a direct call to a non-virtual user defined or helper method - assert(callType == CT_HELPER || callType == CT_USER_FUNC); - - void* addr = nullptr; - if (callType == CT_HELPER) - { - // Direct call to a helper method. - helperNum = compiler->eeGetHelperNum(methHnd); - noway_assert(helperNum != CORINFO_HELP_UNDEF); - - void* pAddr = nullptr; - addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr); - assert(pAddr == nullptr); - } - else - { - // Direct call to a non-virtual user function. - addr = call->gtDirectCallAddress; - } - - assert(addr != nullptr); - - // Non-virtual direct calls to known addresses - - // clang-format off - genEmitCall(emitter::EC_FUNC_TOKEN, - methHnd, - INDEBUG_LDISASM_COMMA(sigInfo) - addr - X86_ARG(argSizeForEmitter), - retSize - MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), - ilOffset); - // clang-format on - } + genCallInstruction(call X86_ARG(stackArgBytes)); // if it was a pinvoke or intrinsic we may have needed to get the address of a label if (genPendingCallLabel) @@ -5474,6 +5249,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) if (call->HasMultiRegRetVal()) { + const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc(); assert(retTypeDesc != nullptr); const unsigned regCount = retTypeDesc->GetReturnRegCount(); @@ -5544,7 +5320,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) noway_assert(compiler->lvaCallSpCheck != 0xCCCCCCCC && compiler->lvaTable[compiler->lvaCallSpCheck].lvDoNotEnregister && compiler->lvaTable[compiler->lvaCallSpCheck].lvOnFrame); - if (!fCallerPop && (stackArgBytes != 0)) + if (!call->CallerPop() && (stackArgBytes != 0)) { // ECX is trashed, so can be used to compute the expected SP. We saved the value of SP // after pushing all the stack arguments, but the caller popped the arguments, so we need @@ -5571,8 +5347,10 @@ void CodeGen::genCallInstruction(GenTreeCall* call) // This needs to be here, rather than above where fPossibleSyncHelperCall is set, // so the GC state vars have been updated before creating the label. - if (fPossibleSyncHelperCall) + if ((call->gtCallType == CT_HELPER) && (compiler->info.compFlags & CORINFO_FLG_SYNCH)) { + CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(call->gtCallMethHnd); + noway_assert(helperNum != CORINFO_HELP_UNDEF); switch (helperNum) { case CORINFO_HELP_MON_ENTER: @@ -5599,7 +5377,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) #if defined(TARGET_X86) // Is the caller supposed to pop the arguments? - if (fCallerPop && (stackArgBytes != 0)) + if (call->CallerPop() && (stackArgBytes != 0)) { stackAdjustBias = stackArgBytes; } @@ -5610,6 +5388,249 @@ void CodeGen::genCallInstruction(GenTreeCall* call) genRemoveAlignmentAfterCall(call, stackAdjustBias); } +//------------------------------------------------------------------------ +// genCallInstruction - Generate instructions necessary to transfer control to the call. +// +// Arguments: +// call - the GT_CALL node +// +// Remaks: +// For tailcalls this function will generate a jump. +// +void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackArgBytes)) +{ +#if defined(TARGET_X86) + // If the callee pops the arguments, we pass a positive value as the argSize, and the emitter will + // adjust its stack level accordingly. + // If the caller needs to explicitly pop its arguments, we must pass a negative value, and then do the + // pop when we're done. + target_ssize_t argSizeForEmitter = stackArgBytes; + if (call->CallerPop()) + { + argSizeForEmitter = -stackArgBytes; + } +#endif // defined(TARGET_X86) + + // Determine return value size(s). + const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc(); + emitAttr retSize = EA_PTRSIZE; + emitAttr secondRetSize = EA_UNKNOWN; + + if (call->HasMultiRegRetVal()) + { + retSize = emitTypeSize(retTypeDesc->GetReturnRegType(0)); + secondRetSize = emitTypeSize(retTypeDesc->GetReturnRegType(1)); + } + else + { + assert(!varTypeIsStruct(call)); + + if (call->gtType == TYP_REF) + { + retSize = EA_GCREF; + } + else if (call->gtType == TYP_BYREF) + { + retSize = EA_BYREF; + } + } + + // We need to propagate the IL offset information to the call instruction, so we can emit + // an IL to native mapping record for the call, to support managed return value debugging. + // We don't want tail call helper calls that were converted from normal calls to get a record, + // so we skip this hash table lookup logic in that case. + + IL_OFFSETX ilOffset = BAD_IL_OFFSET; + + if (compiler->opts.compDbgInfo && compiler->genCallSite2ILOffsetMap != nullptr && !call->IsTailCall()) + { + (void)compiler->genCallSite2ILOffsetMap->Lookup(call, &ilOffset); + } + + CORINFO_SIG_INFO* sigInfo = nullptr; +#ifdef DEBUG + // Pass the call signature information down into the emitter so the emitter can associate + // native call sites with the signatures they were generated from. + if (call->gtCallType != CT_HELPER) + { + sigInfo = call->callSig; + } +#endif // DEBUG + + CORINFO_METHOD_HANDLE methHnd; + GenTree* target = getCallTarget(call, &methHnd); + if (target != nullptr) + { +#ifdef TARGET_X86 + if (call->IsVirtualStub() && (call->gtCallType == CT_INDIRECT)) + { + // On x86, we need to generate a very specific pattern for indirect VSD calls: + // + // 3-byte nop + // call dword ptr [eax] + // + // Where EAX is also used as an argument to the stub dispatch helper. Make + // sure that the call target address is computed into EAX in this case. + + assert(compiler->virtualStubParamInfo->GetReg() == REG_VIRTUAL_STUB_TARGET); + + assert(target->isContainedIndir()); + assert(target->OperGet() == GT_IND); + + GenTree* addr = target->AsIndir()->Addr(); + assert(addr->isUsedFromReg()); + + genConsumeReg(addr); + genCopyRegIfNeeded(addr, REG_VIRTUAL_STUB_TARGET); + + GetEmitter()->emitIns_Nop(3); + + // clang-format off + GetEmitter()->emitIns_Call(emitter::EmitCallType(emitter::EC_INDIR_ARD), + methHnd, + INDEBUG_LDISASM_COMMA(sigInfo) + nullptr, + argSizeForEmitter, + retSize + MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), + gcInfo.gcVarPtrSetCur, + gcInfo.gcRegGCrefSetCur, + gcInfo.gcRegByrefSetCur, + ilOffset, REG_VIRTUAL_STUB_TARGET, REG_NA, 1, 0); + // clang-format on + } + else +#endif + if (target->isContainedIndir()) + { + if (target->AsIndir()->HasBase() && target->AsIndir()->Base()->isContainedIntOrIImmed()) + { + // Note that if gtControlExpr is an indir of an absolute address, we mark it as + // contained only if it can be encoded as PC-relative offset. + assert(target->AsIndir()->Base()->AsIntConCommon()->FitsInAddrBase(compiler)); + + // clang-format off + genEmitCall(emitter::EC_FUNC_TOKEN_INDIR, + methHnd, + INDEBUG_LDISASM_COMMA(sigInfo) + (void*) target->AsIndir()->Base()->AsIntConCommon()->IconValue() + X86_ARG(argSizeForEmitter), + retSize + MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), + ilOffset, + REG_NA, + call->IsFastTailCall()); + // clang-format on + } + else + { + // For fast tailcalls this is happening in epilog, so we should + // have already consumed target in genCall. + if (!call->IsFastTailCall()) + { + genConsumeAddress(target->AsIndir()->Addr()); + } + + // clang-format off + genEmitCallIndir(emitter::EC_INDIR_ARD, + methHnd, + INDEBUG_LDISASM_COMMA(sigInfo) + target->AsIndir() + X86_ARG(argSizeForEmitter), + retSize + MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), + ilOffset, + call->IsFastTailCall()); + // clang-format on + } + } + else + { + // We have already generated code for gtControlExpr evaluating it into a register. + // We just need to emit "call reg" in this case. + assert(genIsValidIntReg(target->GetRegNum())); + + // For fast tailcalls this is happening in epilog, so we should + // have already consumed target in genCall. + if (!call->IsFastTailCall()) + { + genConsumeReg(target); + } + + // clang-format off + genEmitCall(emitter::EC_INDIR_R, + methHnd, + INDEBUG_LDISASM_COMMA(sigInfo) + nullptr // addr + X86_ARG(argSizeForEmitter), + retSize + MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), + ilOffset, + target->GetRegNum(), + call->IsFastTailCall()); + // clang-format on + } + } +#ifdef FEATURE_READYTORUN + else if (call->gtEntryPoint.addr != nullptr) + { + emitter::EmitCallType type = (call->gtEntryPoint.accessType == IAT_VALUE) ? emitter::EC_FUNC_TOKEN + : emitter::EC_FUNC_TOKEN_INDIR; + // clang-format off + genEmitCall(type, + methHnd, + INDEBUG_LDISASM_COMMA(sigInfo) + (void*)call->gtEntryPoint.addr + X86_ARG(argSizeForEmitter), + retSize + MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), + ilOffset, + REG_NA, + call->IsFastTailCall()); + // clang-format on + } +#endif + else + { + // Generate a direct call to a non-virtual user defined or helper method + assert(call->gtCallType == CT_HELPER || call->gtCallType == CT_USER_FUNC); + + void* addr = nullptr; + if (call->gtCallType == CT_HELPER) + { + // Direct call to a helper method. + CorInfoHelpFunc helperNum = compiler->eeGetHelperNum(methHnd); + noway_assert(helperNum != CORINFO_HELP_UNDEF); + + void* pAddr = nullptr; + addr = compiler->compGetHelperFtn(helperNum, (void**)&pAddr); + assert(pAddr == nullptr); + } + else + { + // Direct call to a non-virtual user function. + addr = call->gtDirectCallAddress; + } + + assert(addr != nullptr); + + // Non-virtual direct calls to known addresses + + // clang-format off + genEmitCall(emitter::EC_FUNC_TOKEN, + methHnd, + INDEBUG_LDISASM_COMMA(sigInfo) + addr + X86_ARG(argSizeForEmitter), + retSize + MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), + ilOffset, + REG_NA, + call->IsFastTailCall()); + // clang-format on + } +} + // Produce code for a GT_JMP node. // The arguments of the caller needs to be transferred to the callee before exiting caller. // The actual jump to callee is generated as part of caller epilog sequence. diff --git a/src/coreclr/jit/emitarm.cpp b/src/coreclr/jit/emitarm.cpp index 8e3257ce40e63..ca4e94ce21973 100644 --- a/src/coreclr/jit/emitarm.cpp +++ b/src/coreclr/jit/emitarm.cpp @@ -2167,7 +2167,7 @@ void emitter::emitIns_Mov(instruction ins, EXTEND_COMMON: if (canSkip && (dstReg == srcReg)) { - // There are scenarios such as in genCallInstruction where the sign/zero extension should be elided + // There are scenarios such as in genCall where the sign/zero extension should be elided return; } diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index f2177ba9052c8..62973bcf2042d 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -581,9 +581,7 @@ bool emitter::TakesRexWPrefix(instruction ins, emitAttr attr) // of the source, not the dest). // A 4-byte movzx is equivalent to an 8 byte movzx, so it is not special // cased here. - // - // Rex_jmp = jmp with rex prefix always requires rex.w prefix. - if (ins == INS_movsx || ins == INS_rex_jmp) + if (ins == INS_movsx) { return true; } @@ -2346,8 +2344,8 @@ UNATIVE_OFFSET emitter::emitInsSizeAM(instrDesc* id, code_t code) assert(id->idIns() != INS_invalid); instruction ins = id->idIns(); emitAttr attrSize = id->idOpSize(); - /* The displacement field is in an unusual place for calls */ - ssize_t dsp = (ins == INS_call) ? emitGetInsCIdisp(id) : emitGetInsAmdAny(id); + /* The displacement field is in an unusual place for (tail-)calls */ + ssize_t dsp = (ins == INS_call) || (ins == INS_tail_i_jmp) ? emitGetInsCIdisp(id) : emitGetInsAmdAny(id); bool dspInByte = ((signed char)dsp == (ssize_t)dsp); bool dspIsZero = (dsp == 0); UNATIVE_OFFSET size; @@ -2450,7 +2448,7 @@ UNATIVE_OFFSET emitter::emitInsSizeAM(instrDesc* id, code_t code) // If this is just "call reg", we're done. if (id->idIsCallRegPtr()) { - assert(ins == INS_call); + assert(ins == INS_call || ins == INS_tail_i_jmp); assert(dsp == 0); return size; } @@ -4354,10 +4352,7 @@ bool emitter::IsJccInstruction(instruction ins) bool emitter::IsJmpInstruction(instruction ins) { return -#ifdef TARGET_AMD64 - (ins == INS_rex_jmp) || -#endif - (ins == INS_i_jmp) || (ins == INS_jmp) || (ins == INS_l_jmp); + (ins == INS_i_jmp) || (ins == INS_jmp) || (ins == INS_l_jmp) || (ins == INS_tail_i_jmp); } //---------------------------------------------------------------------------------------- @@ -7541,19 +7536,19 @@ void emitter::emitIns_Call(EmitCallType callType, emitThisGCrefRegs = gcrefRegs; emitThisByrefRegs = byrefRegs; - /* Set the instruction - special case jumping a function */ + /* Set the instruction - special case jumping a function (tail call) */ instruction ins = INS_call; if (isJump) { - assert(callType == EC_FUNC_TOKEN || callType == EC_FUNC_TOKEN_INDIR || callType == EC_INDIR_ARD); + assert(callType == EC_FUNC_TOKEN || callType == EC_FUNC_TOKEN_INDIR || callType == EC_INDIR_ARD || callType == EC_INDIR_R); if (callType == EC_FUNC_TOKEN) { ins = INS_l_jmp; } else { - ins = INS_i_jmp; + ins = INS_tail_i_jmp; } } id->idIns(ins); @@ -8331,7 +8326,7 @@ void emitter::emitDispAddrMode(instrDesc* id, bool noDetail) /* The displacement field is in an unusual place for calls */ - disp = (id->idIns() == INS_call) ? emitGetInsCIdisp(id) : emitGetInsAmdAny(id); + disp = (id->idIns() == INS_call) || (id->idIns() == INS_tail_i_jmp) ? emitGetInsCIdisp(id) : emitGetInsAmdAny(id); /* Display a jump table label if this is a switch table jump */ @@ -8861,17 +8856,18 @@ void emitter::emitDispIns( case IF_AWR: case IF_ARW: - if (ins == INS_call && id->idIsCallRegPtr()) + if (id->idIsCallRegPtr()) { printf("%s", emitRegName(id->idAddr()->iiaAddrMode.amBaseReg)); - break; + } + else + { + printf("%s", sstr); + emitDispAddrMode(id, isNew); + emitDispShift(ins); } - printf("%s", sstr); - emitDispAddrMode(id, isNew); - emitDispShift(ins); - - if ((ins == INS_call) || (ins == INS_i_jmp)) + if ((ins == INS_call) || (ins == INS_tail_i_jmp)) { assert(id->idInsFmt() == IF_ARD); @@ -8884,6 +8880,11 @@ void emitter::emitDispIns( assert(id->idDebugOnlyInfo()->idMemCookie); + if (id->idIsCallRegPtr()) + { + printf(" ; "); + } + /* This is a virtual call */ methodName = emitComp->eeGetMethodFullName((CORINFO_METHOD_HANDLE)id->idDebugOnlyInfo()->idMemCookie); @@ -9954,15 +9955,21 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) rgx = id->idAddr()->iiaAddrMode.amIndxReg; // For INS_call the instruction size is actually the return value size - if (ins == INS_call) + if ((ins == INS_call) || (ins == INS_tail_i_jmp)) { + if (ins == INS_tail_i_jmp) + { + // tail call with addressing mode (or through register) needs rex.w + // prefix to be recognized by unwinder as part of epilog. + code = AddRexWPrefix(ins, code); + } + // Special case: call via a register if (id->idIsCallRegPtr()) { - code_t opcode = insEncodeMRreg(INS_call, reg, EA_PTRSIZE, insCodeMR(INS_call)); - - dst += emitOutputRexOrVexPrefixIfNeeded(ins, dst, opcode); - dst += emitOutputWord(dst, opcode); + code = insEncodeMRreg(ins, reg, EA_PTRSIZE, code); + dst += emitOutputRexOrVexPrefixIfNeeded(ins, dst, code); + dst += emitOutputWord(dst, code); goto DONE; } @@ -13384,12 +13391,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) if (id->idInsFmt() == IF_METHPTR) { // This is call indirect via a method pointer + assert((ins == INS_call) || (ins == INS_tail_i_jmp)); code = insCodeMR(ins); - if (ins == INS_i_jmp) - { - code |= 1; - } if (id->idIsDspReloc()) { @@ -15111,9 +15115,7 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insLatency = PERFSCORE_LATENCY_BRANCH_DIRECT; break; -#ifdef TARGET_AMD64 - case INS_rex_jmp: -#endif // TARGET_AMD64 + case INS_tail_i_jmp: case INS_i_jmp: // branch to register result.insThroughput = PERFSCORE_THROUGHPUT_2C; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 1f032220be6d1..da00b49d80bca 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -4761,7 +4761,7 @@ struct GenTreeCall final : public GenTree GenTree* gtControlExpr; union { - CORINFO_METHOD_HANDLE gtCallMethHnd; // CT_USER_FUNC + CORINFO_METHOD_HANDLE gtCallMethHnd; // CT_USER_FUNC or CT_HELPER GenTree* gtCallAddr; // CT_INDIRECT }; diff --git a/src/coreclr/jit/instrsxarch.h b/src/coreclr/jit/instrsxarch.h index 262e0c052fc6a..7eac9f705eda7 100644 --- a/src/coreclr/jit/instrsxarch.h +++ b/src/coreclr/jit/instrsxarch.h @@ -729,14 +729,11 @@ INST1(setge, "setge", IUM_WR, 0x0F009D, INST1(setle, "setle", IUM_WR, 0x0F009E, Reads_OF | Reads_SF | Reads_ZF ) INST1(setg, "setg", IUM_WR, 0x0F009F, Reads_OF | Reads_SF | Reads_ZF ) -#ifdef TARGET_AMD64 -// A jump with rex prefix. This is used for register indirect -// tail calls. -INST1(rex_jmp, "rex.jmp", IUM_RD, 0x0020FE, INS_FLAGS_None) -#endif - -INST1(i_jmp, "jmp", IUM_RD, 0x0020FE, INS_FLAGS_None ) - +// Indirect jump used for tailcalls. We differentiate between func-internal +// indirect jump (e.g. used for switch) and tailcall indirect jumps because the +// x64 unwinder might require the latter to be rex.w prefixed. +INST1(tail_i_jmp, "tail.jmp", IUM_RD, 0x0020FF, INS_FLAGS_None ) +INST1(i_jmp, "jmp", IUM_RD, 0x0020FF, INS_FLAGS_None ) INST0(jmp, "jmp", IUM_RD, 0x0000EB, INS_FLAGS_None ) INST0(jo, "jo", IUM_RD, 0x000070, Reads_OF ) INST0(jno, "jno", IUM_RD, 0x000071, Reads_OF ) diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 38a7655da7988..796a2a00cf3be 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -4466,32 +4466,27 @@ void Lowering::ContainCheckCallOperands(GenTreeCall* call) // we should never see a gtControlExpr whose type is void. assert(ctrlExpr->TypeGet() != TYP_VOID); - // In case of fast tail implemented as jmp, make sure that gtControlExpr is - // computed into a register. - if (!call->IsFastTailCall()) - { #ifdef TARGET_X86 - // On x86, we need to generate a very specific pattern for indirect VSD calls: - // - // 3-byte nop - // call dword ptr [eax] - // - // Where EAX is also used as an argument to the stub dispatch helper. Make - // sure that the call target address is computed into EAX in this case. - if (call->IsVirtualStub() && (call->gtCallType == CT_INDIRECT)) - { - assert(ctrlExpr->isIndir()); - MakeSrcContained(call, ctrlExpr); - } - else + // On x86, we need to generate a very specific pattern for indirect VSD calls: + // + // 3-byte nop + // call dword ptr [eax] + // + // Where EAX is also used as an argument to the stub dispatch helper. Make + // sure that the call target address is computed into EAX in this case. + if (call->IsVirtualStub() && (call->gtCallType == CT_INDIRECT)) + { + assert(ctrlExpr->isIndir()); + MakeSrcContained(call, ctrlExpr); + } + else #endif // TARGET_X86 - if (ctrlExpr->isIndir()) - { - // We may have cases where we have set a register target on the ctrlExpr, but if it - // contained we must clear it. - ctrlExpr->SetRegNum(REG_NA); - MakeSrcContained(call, ctrlExpr); - } + if (ctrlExpr->isIndir()) + { + // We may have cases where we have set a register target on the ctrlExpr, but if it + // contained we must clear it. + ctrlExpr->SetRegNum(REG_NA); + MakeSrcContained(call, ctrlExpr); } } diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index f4401828f8ad2..f6c82699a0b7f 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -1213,13 +1213,12 @@ int LinearScan::BuildCall(GenTreeCall* call) regMaskTP ctrlExprCandidates = RBM_NONE; // In case of fast tail implemented as jmp, make sure that gtControlExpr is - // computed into a register. + // computed into appropriate registers. if (call->IsFastTailCall()) { - assert(!ctrlExpr->isContained()); - // Fast tail call - make sure that call target is always computed in RAX - // so that epilog sequence can generate "jmp rax" to achieve fast tail call. - ctrlExprCandidates = RBM_RAX; + // Fast tail call - make sure that call target is always computed in volatile registers + // that will not be restored in the epilog sequence. + ctrlExprCandidates = RBM_INT_CALLEE_TRASH; } #ifdef TARGET_X86 else if (call->IsVirtualStub() && (call->gtCallType == CT_INDIRECT)) diff --git a/src/coreclr/vm/amd64/excepamd64.cpp b/src/coreclr/vm/amd64/excepamd64.cpp index 17b9c4e7b3378..c30cc97e299fb 100644 --- a/src/coreclr/vm/amd64/excepamd64.cpp +++ b/src/coreclr/vm/amd64/excepamd64.cpp @@ -367,7 +367,7 @@ RtlVirtualUnwind_Worker ( // This is a jmp outside of the function, probably a tail call // to an import function. InEpilogue = TRUE; - NextByte += 2; + NextByte += 6; } else if (((TempOpcode & 0xf8) == AMD64_SIZE64_PREFIX) && (NextByte[1] == AMD64_JMP_IND_OP) @@ -382,7 +382,27 @@ RtlVirtualUnwind_Worker ( // Such an opcode is an unambiguous epilogue indication. // InEpilogue = TRUE; + // Account for displacement/SIB byte + int mod = NextByte[2] >> 6; + int rm = NextByte[2] & 0b111; NextByte += 3; + if (mod != 0b11) + { + // Has addressing mode + if (rm == 0b100) + { + NextByte++; // SIB byte + } + + if (mod == 0b01) + { + NextByte++; // disp8 + } + else if (mod == 0b10) + { + NextByte += 4; // disp32 + } + } } if (InEpilogue && HasUnmanagedBreakpoint) From e8c0b752edcb4b82c3e26ee297399bcd381165a1 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sun, 5 Sep 2021 00:47:29 +0200 Subject: [PATCH 2/5] Run jit-format --- src/coreclr/jit/codegenarmarch.cpp | 4 ++-- src/coreclr/jit/codegenxarch.cpp | 6 +++--- src/coreclr/jit/emitxarch.cpp | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 719de94354159..cc26945cd18eb 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -2390,7 +2390,7 @@ void CodeGen::genCall(GenTreeCall* call) if (call->HasMultiRegRetVal()) { - const ReturnTypeDesc* pRetTypeDesc = call->GetReturnTypeDesc(); + const ReturnTypeDesc* pRetTypeDesc = call->GetReturnTypeDesc(); assert(pRetTypeDesc != nullptr); unsigned regCount = pRetTypeDesc->GetReturnRegCount(); @@ -2516,7 +2516,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call) } #endif // DEBUG CORINFO_METHOD_HANDLE methHnd; - GenTree* target = getCallTarget(call, &methHnd); + GenTree* target = getCallTarget(call, &methHnd); if (target != nullptr) { diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 0a64a67ac7d39..d45695fb892ec 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -5458,7 +5458,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA #endif // DEBUG CORINFO_METHOD_HANDLE methHnd; - GenTree* target = getCallTarget(call, &methHnd); + GenTree* target = getCallTarget(call, &methHnd); if (target != nullptr) { #ifdef TARGET_X86 @@ -5574,8 +5574,8 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA #ifdef FEATURE_READYTORUN else if (call->gtEntryPoint.addr != nullptr) { - emitter::EmitCallType type = (call->gtEntryPoint.accessType == IAT_VALUE) ? emitter::EC_FUNC_TOKEN - : emitter::EC_FUNC_TOKEN_INDIR; + emitter::EmitCallType type = + (call->gtEntryPoint.accessType == IAT_VALUE) ? emitter::EC_FUNC_TOKEN : emitter::EC_FUNC_TOKEN_INDIR; // clang-format off genEmitCall(type, methHnd, diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 62973bcf2042d..fd16571b7552d 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -2345,7 +2345,7 @@ UNATIVE_OFFSET emitter::emitInsSizeAM(instrDesc* id, code_t code) instruction ins = id->idIns(); emitAttr attrSize = id->idOpSize(); /* The displacement field is in an unusual place for (tail-)calls */ - ssize_t dsp = (ins == INS_call) || (ins == INS_tail_i_jmp) ? emitGetInsCIdisp(id) : emitGetInsAmdAny(id); + ssize_t dsp = (ins == INS_call) || (ins == INS_tail_i_jmp) ? emitGetInsCIdisp(id) : emitGetInsAmdAny(id); bool dspInByte = ((signed char)dsp == (ssize_t)dsp); bool dspIsZero = (dsp == 0); UNATIVE_OFFSET size; @@ -4351,8 +4351,7 @@ bool emitter::IsJccInstruction(instruction ins) // bool emitter::IsJmpInstruction(instruction ins) { - return - (ins == INS_i_jmp) || (ins == INS_jmp) || (ins == INS_l_jmp) || (ins == INS_tail_i_jmp); + return (ins == INS_i_jmp) || (ins == INS_jmp) || (ins == INS_l_jmp) || (ins == INS_tail_i_jmp); } //---------------------------------------------------------------------------------------- @@ -7541,7 +7540,8 @@ void emitter::emitIns_Call(EmitCallType callType, if (isJump) { - assert(callType == EC_FUNC_TOKEN || callType == EC_FUNC_TOKEN_INDIR || callType == EC_INDIR_ARD || callType == EC_INDIR_R); + assert(callType == EC_FUNC_TOKEN || callType == EC_FUNC_TOKEN_INDIR || callType == EC_INDIR_ARD || + callType == EC_INDIR_R); if (callType == EC_FUNC_TOKEN) { ins = INS_l_jmp; From b41532eea858c65e6a8219d2e906e9daa3cd868e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sun, 5 Sep 2021 00:58:33 +0200 Subject: [PATCH 3/5] Add missing [rip+imm32] form --- src/coreclr/vm/amd64/excepamd64.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/amd64/excepamd64.cpp b/src/coreclr/vm/amd64/excepamd64.cpp index c30cc97e299fb..ca978482dc3c2 100644 --- a/src/coreclr/vm/amd64/excepamd64.cpp +++ b/src/coreclr/vm/amd64/excepamd64.cpp @@ -398,7 +398,7 @@ RtlVirtualUnwind_Worker ( { NextByte++; // disp8 } - else if (mod == 0b10) + else if (mod == 0b10 || (mod == 0b00 && rm == 0b101)) { NextByte += 4; // disp32 } From 5c89e13a6ff4187ded6728c83a4561d8317ca8a0 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sun, 5 Sep 2021 14:44:45 +0200 Subject: [PATCH 4/5] Expand ARM64 candidates for tailcall Also add some assertions that we are using non-volatile registers when generating the tailcall. Fixes the arm64 tests as a side effect. It seems there is a bug in the allocator when specifying only a single register as a candidate where it may use a local variable's home instead of the candidate. --- src/coreclr/jit/codegenlinear.cpp | 18 ++++++++++++++++-- src/coreclr/jit/lsraarmarch.cpp | 6 +++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 9e6500cd08aa9..b5d0e5cd39311 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -2306,6 +2306,11 @@ void CodeGen::genEmitCall(int callType, #if !defined(TARGET_X86) int argSize = 0; #endif // !defined(TARGET_X86) + + // This should have been put in volatile registers to ensure it does not + // get overridden by epilog sequence during tailcall. + noway_assert(!isJump || (base == REG_NA) || ((RBM_INT_CALLEE_TRASH & genRegMask(base)) != 0)); + GetEmitter()->emitIns_Call(emitter::EmitCallType(callType), methHnd, INDEBUG_LDISASM_COMMA(sigInfo) @@ -2338,6 +2343,15 @@ void CodeGen::genEmitCallIndir(int callType, #if !defined(TARGET_X86) int argSize = 0; #endif // !defined(TARGET_X86) + + regNumber iReg = (indir->Base() != nullptr) ? indir->Base()->GetRegNum() : REG_NA; + regNumber xReg = (indir->Index() != nullptr) ? indir->Index()->GetRegNum() : REG_NA; + + // These should have been put in volatile registers to ensure they do not + // get overridden by epilog sequence during tailcall. + noway_assert(!isJump || (iReg == REG_NA) || ((RBM_CALLEE_TRASH & genRegMask(iReg)) != 0)); + noway_assert(!isJump || (xReg == REG_NA) || ((RBM_CALLEE_TRASH & genRegMask(xReg)) != 0)); + GetEmitter()->emitIns_Call(emitter::EmitCallType(callType), methHnd, INDEBUG_LDISASM_COMMA(sigInfo) @@ -2349,8 +2363,8 @@ void CodeGen::genEmitCallIndir(int callType, gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur, ilOffset, - (indir->Base() != nullptr) ? indir->Base()->GetRegNum() : REG_NA, - (indir->Index() != nullptr) ? indir->Index()->GetRegNum() : REG_NA, + iReg, + xReg, indir->Scale(), indir->Offset(), isJump); diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index a470e19c94e64..7cde2796b5a1d 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -176,9 +176,9 @@ int LinearScan::BuildCall(GenTreeCall* call) // computed into a register. if (call->IsFastTailCall()) { - // Fast tail call - make sure that call target is always computed in R12(ARM32)/IP0(ARM64) - // so that epilog sequence can generate "br xip0/r12" to achieve fast tail call. - ctrlExprCandidates = RBM_FASTTAILCALL_TARGET; + // Fast tail call - make sure that call target is always computed in volatile registers + // that will not be overridden by epilog sequence. + ctrlExprCandidates = RBM_INT_CALLEE_TRASH; } } else if (call->IsR2ROrVirtualStubRelativeIndir()) From 10f1bac4a5e258939cce578380bf8e9584c8bcde Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sun, 5 Sep 2021 14:51:23 +0200 Subject: [PATCH 5/5] Remove REG/RBM_FASTTAILCALL_TARGET defines --- src/coreclr/jit/targetarm.h | 3 --- src/coreclr/jit/targetarm64.h | 3 --- 2 files changed, 6 deletions(-) diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h index 7a50f844ae869..f0545335231e8 100644 --- a/src/coreclr/jit/targetarm.h +++ b/src/coreclr/jit/targetarm.h @@ -69,9 +69,6 @@ #define REG_DEFAULT_HELPER_CALL_TARGET REG_R12 #define RBM_DEFAULT_HELPER_CALL_TARGET RBM_R12 - #define REG_FASTTAILCALL_TARGET REG_R12 // Target register for fast tail call - #define RBM_FASTTAILCALL_TARGET RBM_R12 - #define RBM_ALLINT (RBM_INT_CALLEE_SAVED | RBM_INT_CALLEE_TRASH) #define RBM_ALLFLOAT (RBM_FLT_CALLEE_SAVED | RBM_FLT_CALLEE_TRASH) #define RBM_ALLDOUBLE (RBM_F0|RBM_F2|RBM_F4|RBM_F6|RBM_F8|RBM_F10|RBM_F12|RBM_F14|RBM_F16|RBM_F18|RBM_F20|RBM_F22|RBM_F24|RBM_F26|RBM_F28|RBM_F30) diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index fc5b37e054458..cdab21582ffce 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -74,9 +74,6 @@ #define REG_DEFAULT_HELPER_CALL_TARGET REG_R12 #define RBM_DEFAULT_HELPER_CALL_TARGET RBM_R12 - #define REG_FASTTAILCALL_TARGET REG_IP0 // Target register for fast tail call - #define RBM_FASTTAILCALL_TARGET RBM_IP0 - #define RBM_ALLINT (RBM_INT_CALLEE_SAVED | RBM_INT_CALLEE_TRASH) #define RBM_ALLFLOAT (RBM_FLT_CALLEE_SAVED | RBM_FLT_CALLEE_TRASH) #define RBM_ALLDOUBLE RBM_ALLFLOAT