Make NativeCallableAttribute public. (#33005)

* Avoid creating a COM Delegate when calling NativeCallableAttribute.

* Create reverse P/Invoke frame.
Create Preempt and Coop PreStubWorker().

* Limit the exclusion for NativeCallableAttribute to Win-x86.

* Add Reverse P/Invoke JIT helpers to CrossGen2.

* Add test for generic class with NativeCallableAttribute method.

* Implement Unix EH change for NativeCallableAttribute method.

Co-authored-by: Jan Vorlicek <janvorli@microsoft.com>
Co-authored-by: Jan Kotas <jkotas@microsoft.com>

src/coreclr/src/inc/corcompile.h

@@ -702,7 +702,6 @@ enum CORCOMPILE_FIXUP_BLOB_KIND
     ENCODE_VARARGS_METHODREF,
     ENCODE_VARARGS_SIG,
     ENCODE_ACTIVE_DEPENDENCY,                       /* Conditional active dependency */
-    ENCODE_METHOD_NATIVE_ENTRY,                     /* NativeCallable method token */
 };
 
 enum EncodeMethodSigFlags

src/coreclr/src/inc/jithelpers.h

@@ -346,8 +346,8 @@
     JITHELPER(CORINFO_HELP_JIT_PINVOKE_BEGIN,         JIT_PInvokeBegin,     CORINFO_HELP_SIG_REG_ONLY)
     JITHELPER(CORINFO_HELP_JIT_PINVOKE_END,           JIT_PInvokeEnd,       CORINFO_HELP_SIG_REG_ONLY)
 
-    JITHELPER(CORINFO_HELP_JIT_REVERSE_PINVOKE_ENTER, NULL, CORINFO_HELP_SIG_UNDEF)
-    JITHELPER(CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT, NULL, CORINFO_HELP_SIG_UNDEF)
+    JITHELPER(CORINFO_HELP_JIT_REVERSE_PINVOKE_ENTER, JIT_ReversePInvokeEnter, CORINFO_HELP_SIG_REG_ONLY)
+    JITHELPER(CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT,  JIT_ReversePInvokeExit, CORINFO_HELP_SIG_REG_ONLY)
 
     JITHELPER(CORINFO_HELP_GVMLOOKUP_FOR_SLOT, NULL, CORINFO_HELP_SIG_NO_ALIGN_STUB)
 

src/coreclr/src/inc/readytorun.h

@@ -256,6 +256,8 @@ enum ReadyToRunHelper
     READYTORUN_HELPER_PInvokeBegin              = 0x42,
     READYTORUN_HELPER_PInvokeEnd                = 0x43,
     READYTORUN_HELPER_GCPoll                    = 0x44,
+    READYTORUN_HELPER_ReversePInvokeEnter       = 0x45,
+    READYTORUN_HELPER_ReversePInvokeExit        = 0x46,
 
     // Get string handle lazily
     READYTORUN_HELPER_GetString                 = 0x50,
@@ -382,7 +384,8 @@ struct READYTORUN_EXCEPTION_CLAUSE
 
 enum ReadyToRunRuntimeConstants : DWORD
 {
-    READYTORUN_PInvokeTransitionFrameSizeInPointerUnits = 11
+    READYTORUN_PInvokeTransitionFrameSizeInPointerUnits = 11,
+    READYTORUN_ReversePInvokeTransitionFrameSizeInPointerUnits = 2
 };
 
 #endif // __READYTORUN_H__

src/coreclr/src/inc/readytorunhelpers.h

@@ -114,6 +114,8 @@ HELPER(READYTORUN_HELPER_EndCatch,                  CORINFO_HELP_ENDCATCH,
 HELPER(READYTORUN_HELPER_PInvokeBegin,              CORINFO_HELP_JIT_PINVOKE_BEGIN,                 )
 HELPER(READYTORUN_HELPER_PInvokeEnd,                CORINFO_HELP_JIT_PINVOKE_END,                   )
 HELPER(READYTORUN_HELPER_GCPoll,                    CORINFO_HELP_POLL_GC,                           )
+HELPER(READYTORUN_HELPER_ReversePInvokeEnter,       CORINFO_HELP_JIT_REVERSE_PINVOKE_ENTER,         )
+HELPER(READYTORUN_HELPER_ReversePInvokeExit,        CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT,          )
 
 HELPER(READYTORUN_HELPER_MonitorEnter,              CORINFO_HELP_MON_ENTER,                         )
 HELPER(READYTORUN_HELPER_MonitorExit,               CORINFO_HELP_MON_EXIT,                          )

src/coreclr/src/tools/Common/Internal/Runtime/ReadyToRunConstants.cs

@@ -175,6 +175,8 @@ public enum ReadyToRunHelper
         PInvokeBegin                = 0x42,
         PInvokeEnd                  = 0x43,
         GCPoll                      = 0x44,
+        ReversePInvokeEnter         = 0x45,
+        ReversePInvokeExit          = 0x46,
 
         // Get string handle lazily
         GetString = 0x50,
@@ -295,10 +297,6 @@ public enum ReadyToRunHelper
         CheckCastInterface,
         CheckInstanceInterface,
 
-        // P/Invoke support
-        ReversePInvokeEnter,
-        ReversePInvokeExit,
-
         MonitorEnterStatic,
         MonitorExitStatic,
 
@@ -314,5 +312,6 @@ public enum ReadyToRunHelper
     public static class ReadyToRunRuntimeConstants
     {
         public const int READYTORUN_PInvokeTransitionFrameSizeInPointerUnits = 11;
+        public const int READYTORUN_ReversePInvokeTransitionFrameSizeInPointerUnits = 2;
     }
 }

src/coreclr/src/tools/Common/JitInterface/CorInfoImpl.cs

@@ -2220,7 +2220,7 @@ private void getEEInfo(ref CORINFO_EE_INFO pEEInfoOut)
             pEEInfoOut.offsetOfDelegateInstance = (uint)pointerSize;            // Delegate::m_firstParameter
             pEEInfoOut.offsetOfDelegateFirstTarget = OffsetOfDelegateFirstTarget;
 
-            pEEInfoOut.sizeOfReversePInvokeFrame = (uint)(2 * pointerSize);
+            pEEInfoOut.sizeOfReversePInvokeFrame = (uint)SizeOfReversePInvokeTransitionFrame;
 
             pEEInfoOut.osPageSize = new UIntPtr(0x1000);
 
@@ -2882,7 +2882,17 @@ private uint getJitFlags(ref CORJIT_FLAGS flags, uint sizeInBytes)
             }
 
             if (this.MethodBeingCompiled.IsNativeCallable)
+            {
+#if READYTORUN
+                if (targetArchitecture == TargetArchitecture.X86
+                    && _compilation.TypeSystemContext.Target.OperatingSystem == TargetOS.Windows)
+                {
+                    throw new RequiresRuntimeJitException("ReadyToRun: Methods with NativeCallableAttribute not implemented");
+                }
+#endif
+
                 flags.Set(CorJitFlag.CORJIT_FLAG_REVERSE_PINVOKE);
+            }
 
             if (this.MethodBeingCompiled.IsPInvoke)
             {

src/coreclr/src/tools/Common/TypeSystem/Common/ExceptionStringID.cs

@@ -34,6 +34,9 @@ public enum ExceptionStringID
         InvalidProgramNativeCallable,
         InvalidProgramCallAbstractMethod,
         InvalidProgramCallVirtStatic,
+        InvalidProgramNonStaticMethod,
+        InvalidProgramGenericMethod,
+        InvalidProgramNonBlittableTypes,
 
         // BadImageFormatException
         BadImageFormatGeneric,

src/coreclr/src/tools/crossgen2/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs

@@ -608,6 +608,14 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum)
                     id = ReadyToRunHelper.GCPoll;
                     break;
 
+                case CorInfoHelpFunc.CORINFO_HELP_JIT_REVERSE_PINVOKE_ENTER:
+                    id = ReadyToRunHelper.ReversePInvokeEnter;
+                    break;
+
+                case CorInfoHelpFunc.CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT:
+                    id = ReadyToRunHelper.ReversePInvokeExit;
+                    break;
+
                 case CorInfoHelpFunc.CORINFO_HELP_INITCLASS:
                 case CorInfoHelpFunc.CORINFO_HELP_INITINSTCLASS:
                 case CorInfoHelpFunc.CORINFO_HELP_THROW_ARGUMENTEXCEPTION:
@@ -616,8 +624,6 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum)
                 case CorInfoHelpFunc.CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPE_MAYBENULL:
                 case CorInfoHelpFunc.CORINFO_HELP_TYPEHANDLE_TO_RUNTIMETYPEHANDLE_MAYBENULL:
                 case CorInfoHelpFunc.CORINFO_HELP_GETREFANY:
-                case CorInfoHelpFunc.CORINFO_HELP_JIT_REVERSE_PINVOKE_ENTER:
-                case CorInfoHelpFunc.CORINFO_HELP_JIT_REVERSE_PINVOKE_EXIT:
                     throw new RequiresRuntimeJitException(ftnNum.ToString());
 
                 default:
@@ -1145,6 +1151,25 @@ private void ceeInfoGetCallInfo(
                 ThrowHelper.ThrowInvalidProgramException(ExceptionStringID.InvalidProgramCallVirtStatic, originalMethod);
             }
 
+            if ((flags & CORINFO_CALLINFO_FLAGS.CORINFO_CALLINFO_LDFTN) != 0
+                && originalMethod.IsNativeCallable)
+            {
+                if (!originalMethod.Signature.IsStatic) // Must be a static method
+                {
+                    ThrowHelper.ThrowInvalidProgramException(ExceptionStringID.InvalidProgramNonStaticMethod, originalMethod);
+                }
+
+                if (originalMethod.HasInstantiation || originalMethod.OwningType.HasInstantiation) // No generics involved
+                {
+                    ThrowHelper.ThrowInvalidProgramException(ExceptionStringID.InvalidProgramGenericMethod, originalMethod);
+                }
+
+                if (Marshaller.IsMarshallingRequired(originalMethod.Signature, Array.Empty<ParameterMetadata>())) // Only blittable arguments
+                {
+                    ThrowHelper.ThrowInvalidProgramException(ExceptionStringID.InvalidProgramNonBlittableTypes, originalMethod);
+                }
+            }
+
             exactType = type;
 
             constrainedType = null;
@@ -1635,6 +1660,14 @@ private void getCallInfo(ref CORINFO_RESOLVED_TOKEN pResolvedToken, CORINFO_RESO
 
             pResult->methodFlags = FilterNamedIntrinsicMethodAttribs(pResult->methodFlags, methodToCall);
 
+            var targetDetails = _compilation.TypeSystemContext.Target;
+            if (targetDetails.Architecture == TargetArchitecture.X86
+                && targetDetails.OperatingSystem == TargetOS.Windows
+                && targetMethod.IsNativeCallable)
+            {
+                throw new RequiresRuntimeJitException("ReadyToRun: References to methods with NativeCallableAttribute not implemented");
+            }
+
             if (pResult->thisTransform == CORINFO_THIS_TRANSFORM.CORINFO_BOX_THIS)
             {
                 // READYTORUN: FUTURE: Optionally create boxing stub at runtime
@@ -2297,6 +2330,7 @@ private bool canGetCookieForPInvokeCalliSig(CORINFO_SIG_INFO* szMetaSig)
         }
 
         private int SizeOfPInvokeTransitionFrame => ReadyToRunRuntimeConstants.READYTORUN_PInvokeTransitionFrameSizeInPointerUnits * _compilation.NodeFactory.Target.PointerSize;
+        private int SizeOfReversePInvokeTransitionFrame => ReadyToRunRuntimeConstants.READYTORUN_ReversePInvokeTransitionFrameSizeInPointerUnits * _compilation.NodeFactory.Target.PointerSize;
 
         private void setEHcount(uint cEH)
         {

src/coreclr/src/tools/crossgen2/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs

@@ -1366,6 +1366,14 @@ private void ParseHelper(StringBuilder builder)
                     builder.Append("GCPOLL");
                     break;
 
+                case ReadyToRunHelper.ReversePInvokeEnter:
+                    builder.Append("REVERSE_PINVOKE_ENTER");
+                    break;
+
+                case ReadyToRunHelper.ReversePInvokeExit:
+                    builder.Append("REVERSE_PINVOKE_EXIT");
+                    break;
+
                 // Get string handle lazily
                 case ReadyToRunHelper.GetString:
                     builder.Append("GET_STRING");

src/coreclr/src/vm/cgensys.h

@@ -53,6 +53,14 @@ extern "C" void GenericComPlusCallStub(void);
 extern "C" void GenericComCallStub(void);
 #endif // FEATURE_COMINTEROP
 
+// The GC mode for the thread that initially called ThePreStub().
+enum class CallerGCMode
+{
+    Unknown,
+    Coop,
+    Preemptive    // (e.g. NativeCallableAttribute)
+};
+
 // Non-CPU-specific helper functions called by the CPU-dependent code
 extern "C" PCODE STDCALL PreStubWorker(TransitionBlock * pTransitionBlock, MethodDesc * pMD);
 

src/coreclr/src/vm/codeversion.cpp

@@ -1611,6 +1611,7 @@ HRESULT CodeVersionManager::AddNativeCodeVersion(
 
 PCODE CodeVersionManager::PublishVersionableCodeIfNecessary(
     MethodDesc* pMethodDesc,
+    CallerGCMode callerGCMode,
     bool *doBackpatchRef,
     bool *doFullBackpatchRef)
 {
@@ -1655,7 +1656,7 @@ PCODE CodeVersionManager::PublishVersionableCodeIfNecessary(
         _ASSERTE(hr == E_OUTOFMEMORY);
         ReportCodePublishError(pMethodDesc, hr);
         *doBackpatchRef = false;
-        return pCode != NULL ? pCode : pMethodDesc->PrepareInitialCode();
+        return pCode != NULL ? pCode : pMethodDesc->PrepareInitialCode(callerGCMode);
     } while (false);
 
     while (true)
@@ -1670,6 +1671,9 @@ PCODE CodeVersionManager::PublishVersionableCodeIfNecessary(
         {
             PrepareCodeConfigBuffer configBuffer(activeVersion);
             PrepareCodeConfig *config = configBuffer.GetConfig();
+
+            // Record the caller's GC mode.
+            config->SetCallerGCMode(callerGCMode);
             pCode = pMethodDesc->PrepareCode(config);
 
         #ifdef FEATURE_CODE_VERSIONING

src/coreclr/src/vm/codeversion.h

@@ -10,7 +10,6 @@
 #ifndef CODE_VERSION_H
 #define CODE_VERSION_H
 
-class NativeCodeVersion;
 class ILCodeVersion;
 typedef DWORD NativeCodeVersionId;
 
@@ -572,7 +571,11 @@ class CodeVersionManager
 
     HRESULT AddILCodeVersion(Module* pModule, mdMethodDef methodDef, ReJITID rejitId, ILCodeVersion* pILCodeVersion);
     HRESULT AddNativeCodeVersion(ILCodeVersion ilCodeVersion, MethodDesc* pClosedMethodDesc, NativeCodeVersion::OptimizationTier optimizationTier, NativeCodeVersion* pNativeCodeVersion);
-    PCODE PublishVersionableCodeIfNecessary(MethodDesc* pMethodDesc, bool *doBackpatchRef, bool *doFullBackpatchRef);
+    PCODE PublishVersionableCodeIfNecessary(
+        MethodDesc* pMethodDesc,
+        CallerGCMode callerGCMode,
+        bool *doBackpatchRef,
+        bool *doFullBackpatchRef);
     HRESULT PublishNativeCodeVersion(MethodDesc* pMethodDesc, NativeCodeVersion nativeCodeVersion);
     HRESULT GetOrCreateMethodDescVersioningState(MethodDesc* pMethod, MethodDescVersioningState** ppMethodDescVersioningState);
     HRESULT GetOrCreateILCodeVersioningState(Module* pModule, mdMethodDef methodDef, ILCodeVersioningState** ppILCodeVersioningState);

src/coreclr/src/vm/comdelegate.cpp

@@ -1132,36 +1132,26 @@ void COMDelegate::BindToMethod(DELEGATEREF   *pRefThis,
     GCPROTECT_END();
 }
 
+#if defined(TARGET_X86) && defined(TARGET_WINDOWS)
 // Marshals a managed method to an unmanaged callback provided the
 // managed method is static and it's parameters require no marshalling.
 PCODE COMDelegate::ConvertToCallback(MethodDesc* pMD)
 {
     CONTRACTL
     {
         THROWS;
-    GC_TRIGGERS;
-    INJECT_FAULT(COMPlusThrowOM());
+        GC_TRIGGERS;
+        PRECONDITION(pMD != NULL);
+        INJECT_FAULT(COMPlusThrowOM());
     }
     CONTRACTL_END;
 
     PCODE pCode = NULL;
 
-    // only static methods are allowed
-    if (!pMD->IsStatic())
-        COMPlusThrow(kNotSupportedException, W("NotSupported_NonStaticMethod"));
-
-    // no generic methods
-    if (pMD->IsGenericMethodDefinition())
-        COMPlusThrow(kNotSupportedException, W("NotSupported_GenericMethod"));
-
-    // Arguments
-    if (NDirect::MarshalingRequired(pMD, pMD->GetSig(), pMD->GetModule()))
-        COMPlusThrow(kNotSupportedException, W("NotSupported_NonBlittableTypes"));
-
     // Get UMEntryThunk from the thunk cache.
     UMEntryThunk *pUMEntryThunk = pMD->GetLoaderAllocator()->GetUMEntryThunkCache()->GetUMEntryThunk(pMD);
 
-#if defined(TARGET_X86) && !defined(FEATURE_STUBS_AS_IL)
+#if !defined(FEATURE_STUBS_AS_IL)
 
     // System.Runtime.InteropServices.NativeCallableAttribute
     BYTE* pData = NULL;
@@ -1193,12 +1183,13 @@ PCODE COMDelegate::ConvertToCallback(MethodDesc* pMD)
             pUMThunkMarshalInfo->SetCallingConvention(callConv);
         }
 }
-#endif  //TARGET_X86 && !FEATURE_STUBS_AS_IL
+#endif  // !FEATURE_STUBS_AS_IL
 
     pCode = (PCODE)pUMEntryThunk->GetCode();
     _ASSERTE(pCode != NULL);
     return pCode;
 }
+#endif // defined(TARGET_X86) && defined(TARGET_WINDOWS)
 
 // Marshals a delegate to a unmanaged callback.
 LPVOID COMDelegate::ConvertToCallback(OBJECTREF pDelegateObj)

src/coreclr/src/vm/comdelegate.h

@@ -84,9 +84,11 @@ class COMDelegate
     // Marshals a delegate to a unmanaged callback.
     static LPVOID ConvertToCallback(OBJECTREF pDelegate);
 
-    // Marshals a managed method to an unmanaged callback , provided the method is static and uses only
-    // blittable parameter types.
+#if defined(TARGET_X86) && defined(TARGET_WINDOWS)
+    // Marshals a managed method to an unmanaged callback.
+    // This is only used on x86 Windows. See usage for further details.
     static PCODE ConvertToCallback(MethodDesc* pMD);
+#endif // defined(TARGET_X86) && defined(TARGET_WINDOWS)
 
     // Marshals an unmanaged callback to Delegate
     static OBJECTREF ConvertToDelegate(LPVOID pCallback, MethodTable* pMT);

src/coreclr/src/vm/exceptionhandling.cpp

@@ -4655,7 +4655,6 @@ VOID DECLSPEC_NORETURN UnwindManagedExceptionPass1(PAL_SEHException& ex, CONTEXT
             }
             else
             {
-                // TODO: This needs to implemented. Make it fail for now.
                 UNREACHABLE();
             }
         }
@@ -4664,6 +4663,20 @@ VOID DECLSPEC_NORETURN UnwindManagedExceptionPass1(PAL_SEHException& ex, CONTEXT
             controlPc = Thread::VirtualUnwindLeafCallFrame(frameContext);
         }
 
+        GcInfoDecoder gcInfoDecoder(codeInfo.GetGCInfoToken(), DECODE_REVERSE_PINVOKE_VAR);
+
+        if (gcInfoDecoder.GetReversePInvokeFrameStackSlot() != NO_REVERSE_PINVOKE_FRAME)
+        {
+            // Propagating exception from a method marked by NativeCallable attribute is prohibited on Unix
+            if (!GetThread()->HasThreadStateNC(Thread::TSNC_ProcessedUnhandledException))
+            {
+                LONG disposition = InternalUnhandledExceptionFilter_Worker(&ex.ExceptionPointers);
+                _ASSERTE(disposition == EXCEPTION_CONTINUE_SEARCH);
+            }
+            TerminateProcess(GetCurrentProcess(), 1);
+            UNREACHABLE();
+        }
+
         // Check whether we are crossing managed-to-native boundary
         while (!ExecutionManager::IsManagedCode(controlPc))
         {

src/coreclr/src/vm/frames.h

@@ -2785,6 +2785,12 @@ class UMThkCallFrame : public UnmanagedToManagedFrame
 };
 #endif // TARGET_X86 && !TARGET_UNIX
 
+// Frame for the Reverse PInvoke (i.e. NativeCallableAttribute).
+struct ReversePInvokeFrame
+{
+    Thread* currentThread;
+};
+
 #if defined(TARGET_X86) && defined(FEATURE_COMINTEROP)
 //-------------------------------------------------------------------------
 // Exception handler for COM to managed frame