Respect explicit tailcalls when generating instantiating stubs

If the target function has explicit tailcalls out of it we must also
ensure that we explicitly tailcall into it when generating instantiating
stubs so that the instantiating stub does not leave a frame on the
stack.

Fix dotnet#40864

src/coreclr/src/vm/prestub.cpp

@@ -22,6 +22,7 @@
 #include "compile.h"
 #include "ecall.h"
 #include "virtualcallstub.h"
+#include "opinfo.h"
 
 #ifdef FEATURE_PREJIT
 #include "compile.h"
@@ -1453,6 +1454,27 @@ void CreateInstantiatingILStubTargetSig(MethodDesc *pBaseMD,
 #endif // TARGET_X86
 }
 
+bool ILHasTailPrefix(MethodDesc* pMD)
+{
+    if (!pMD->IsIL())
+        return false;
+
+    COR_ILMETHOD_DECODER decoder(pMD->GetILHeader());
+    const BYTE* code = decoder.Code;
+    const BYTE* end = code + decoder.GetCodeSize();
+
+    while (code < end)
+    {
+        OpArgsVal arg;
+        OpInfo op;
+        code = op.fetch(code, &arg);
+        if (op.getOpcode() == CEE_TAILCALL)
+            return true;
+    }
+
+    return false;
+}
+
 Stub * CreateUnboxingILStubForSharedGenericValueTypeMethods(MethodDesc* pTargetMD)
 {
 
@@ -1522,6 +1544,13 @@ Stub * CreateUnboxingILStubForSharedGenericValueTypeMethods(MethodDesc* pTargetM
     pCode->EmitLDC((TADDR)pTargetMD->GetMultiCallableAddrOfCode(CORINFO_ACCESS_ANY));
 
     // 2.6 Do the calli
+    if (ILHasTailPrefix(pTargetMD))
+    {
+        // If the target method requires explicit tailcalls out of it we must
+        // maintain this wish by also tailcalling into it.
+        pCode->EmitTAIL();
+    }
+
     pCode->EmitCALLI(TOKEN_ILSTUB_TARGET_SIG, msig.NumFixedArgs() + 1, msig.IsReturnTypeVoid() ? 0 : 1);
     pCode->EmitRET();
 
@@ -1624,6 +1653,13 @@ Stub * CreateInstantiatingILStub(MethodDesc* pTargetMD, void* pHiddenArg)
     pCode->EmitLDC((TADDR)pTargetMD->GetMultiCallableAddrOfCode(CORINFO_ACCESS_ANY));
 
     // 2.6 Do the calli
+    if (ILHasTailPrefix(pTargetMD))
+    {
+        // If the target method requires explicit tailcalls out of it we must
+        // maintain this wish by also tailcalling into it.
+        pCode->EmitTAIL();
+    }
+
     pCode->EmitCALLI(TOKEN_ILSTUB_TARGET_SIG, msig.NumFixedArgs() + 1, msig.IsReturnTypeVoid() ? 0 : 1);
     pCode->EmitRET();
 

src/coreclr/src/vm/stubgen.cpp

@@ -1806,6 +1806,11 @@ void ILCodeStream::EmitSUB()
     WRAPPER_NO_CONTRACT;
     Emit(CEE_SUB, -1, 0);
 }
+void ILCodeStream::EmitTAIL()
+{
+    WRAPPER_NO_CONTRACT;
+    Emit(CEE_TAILCALL, 0, 0);
+}
 void ILCodeStream::EmitTHROW()
 {
     WRAPPER_NO_CONTRACT;

src/coreclr/src/vm/stubgen.h

@@ -782,6 +782,7 @@ class ILCodeStream
     void EmitSTOBJ      (int token);
     void EmitSTSFLD     (int token);
     void EmitSUB        ();
+    void EmitTAIL       ();
     void EmitTHROW      ();
     void EmitUNALIGNED  (BYTE alignment);
 

src/tests/JIT/Directed/tailcall/more_tailcalls.cs

@@ -56,28 +56,42 @@ internal class Program
     private static readonly IntPtr s_calcStaticCalliRetbufOther;
     private static readonly IntPtr s_emptyCalliOther;
     private static readonly IntPtr s_instanceMethodOnValueType;
+    private static readonly IntPtr s_instantiatingStub2;
+    private static readonly IntPtr s_instantiatingStub2Other;
 
     static Program()
     {
         IL.Emit.Ldftn(new MethodRef(typeof(Program), nameof(CalcStaticCalli)));
         IL.Pop(out IntPtr calcStaticCalli);
+        s_calcStaticCalli = calcStaticCalli;
+
         IL.Emit.Ldftn(new MethodRef(typeof(Program), nameof(CalcStaticCalliOther)));
         IL.Pop(out IntPtr calcStaticCalliOther);
+        s_calcStaticCalliOther = calcStaticCalliOther;
+
         IL.Emit.Ldftn(new MethodRef(typeof(Program), nameof(CalcStaticCalliRetbuf)));
         IL.Pop(out IntPtr calcStaticCalliRetbuf);
+        s_calcStaticCalliRetbuf = calcStaticCalliRetbuf;
+
         IL.Emit.Ldftn(new MethodRef(typeof(Program), nameof(CalcStaticCalliRetbufOther)));
         IL.Pop(out IntPtr calcStaticCalliRetbufOther);
+        s_calcStaticCalliRetbufOther = calcStaticCalliRetbufOther;
+
         IL.Emit.Ldftn(new MethodRef(typeof(Program), nameof(EmptyCalliOther)));
         IL.Pop(out IntPtr emptyCalliOther);
+        s_emptyCalliOther = emptyCalliOther;
+
         IL.Emit.Ldftn(new MethodRef(typeof(S16), nameof(S16.InstanceMethod)));
         IL.Pop(out IntPtr instanceMethodOnValueType);
-
-        s_calcStaticCalli = calcStaticCalli;
-        s_calcStaticCalliOther = calcStaticCalliOther;
-        s_calcStaticCalliRetbuf = calcStaticCalliRetbuf;
-        s_calcStaticCalliRetbufOther = calcStaticCalliRetbufOther;
-        s_emptyCalliOther = emptyCalliOther;
         s_instanceMethodOnValueType = instanceMethodOnValueType;
+
+        IL.Emit.Ldftn(new MethodRef(typeof(Program), nameof(InstantiatingStub2)).MakeGenericMethod(typeof(string)));
+        IL.Pop(out IntPtr instantiatingStub2);
+        s_instantiatingStub2 = instantiatingStub2;
+
+        IL.Emit.Ldftn(new MethodRef(typeof(Program), nameof(InstantiatingStub2Other)).MakeGenericMethod(typeof(string)));
+        IL.Pop(out IntPtr instantiatingStub2Other);
+        s_instantiatingStub2Other = instantiatingStub2Other;
     }
 
     private static int Main()
@@ -183,6 +197,11 @@ void TestCalc<T>(Func<int, int, T> f, T expected, string name)
         Test(() => GenAbstractGString(ga1), "System.String", "Abstract generic without generic on method 1");
         Test(() => GenAbstractGInt(ga2), "System.Int32", "Abstract generic without generic on method 2");
 
+        int[] a = new int[1_000_000];
+        a[99] = 1;
+        Test(() => InstantiatingStub1(0, 0, "string", a), a.Length + 1, "Instantiating stub direct");
+        Test(() => InstantiatingStub2(0, 0, "string", a), a.Length + 1, "Instantiating stub + calli");
+
         if (result)
             Console.WriteLine("All tailcall-via-help succeeded");
         else
@@ -686,6 +705,66 @@ private static string GenAbstractGInt(GenAbstract<int> ga)
         IL.Emit.Callvirt(new MethodRef(typeof(GenAbstract<int>), nameof(GenAbstract<int>.G)));
         return IL.Return<string>();
     }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static int InstantiatingStub1<T>(int a, int r, T c, Span<int> d)
+    {
+        IL.Push(c);
+        IL.Push(a);
+        IL.Push(r);
+        IL.Emit.Ldarg(nameof(d));
+        IL.Push(r + d[99]);
+        IL.Emit.Tail();
+        IL.Emit.Call(new MethodRef(typeof(Program), nameof(InstantiatingStub1Other)).MakeGenericMethod(typeof(T)));
+        return IL.Return<int>();
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static int InstantiatingStub1Other<T>(T c, int a, int r, Span<int> d, int result)
+    {
+        if (a == d.Length) return result;
+        else
+        {
+            IL.Push(a + 1);
+            IL.Push(result);
+            IL.Push(c);
+            IL.Emit.Ldarg(nameof(d));
+            IL.Emit.Tail();
+            IL.Emit.Call(new MethodRef(typeof(Program), nameof(InstantiatingStub1)).MakeGenericMethod(typeof(T)));
+            return IL.Return<int>();
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static int InstantiatingStub2<T>(int a, int r, T c, Span<int> d)
+    {
+        IL.Push("string");
+        IL.Push(a);
+        IL.Push(r);
+        IL.Emit.Ldarg(nameof(d));
+        IL.Push(r + d[99]);
+        IL.Push(s_instantiatingStub2Other);
+        IL.Emit.Tail();
+        IL.Emit.Calli(new StandAloneMethodSig(CallingConventions.Standard, typeof(int), typeof(string), typeof(int), typeof(int), typeof(Span<int>), typeof(int)));
+        return IL.Return<int>();
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static int InstantiatingStub2Other<T>(T c, int a, int r, Span<int> d, int result)
+    {
+        if (a == d.Length) return result;
+        else
+        {
+            IL.Push(a + 1);
+            IL.Push(result);
+            IL.Push("string");
+            IL.Emit.Ldarg(nameof(d));
+            IL.Push(s_instantiatingStub2);
+            IL.Emit.Tail();
+            IL.Emit.Calli(new StandAloneMethodSig(CallingConventions.Standard, typeof(int), typeof(int), typeof(int), typeof(string), typeof(Span<int>)));
+            return IL.Return<int>();
+        }
+    }
 }
 
 class Instance