Fold always false type checks

src/coreclr/inc/corinfo.h

@@ -1247,6 +1247,7 @@ struct CORINFO_LOOKUP_KIND
 #define CORINFO_USEHELPER ((uint16_t) 0xffff)
 #define CORINFO_USENULL ((uint16_t) 0xfffe)
 #define CORINFO_NO_SIZE_CHECK ((uint16_t) 0xffff)
+#define CORINFO_NEVER_INSTANTIATED (-1)
 
 struct CORINFO_RUNTIME_LOOKUP
 {
@@ -2983,6 +2984,8 @@ class ICorStaticInfo
     // Obtains a list of exact classes for a given base type. Returns 0 if the number of
     // the exact classes is greater than maxExactClasses or if more types might be loaded
     // in future.
+    // When called with maxExactClasses == 0, returns CORINFO_NEVER_INSTANTIATED if an
+    // exact class of this type cannot exist. Returns 0 otherwise.
     virtual int getExactClasses(
             CORINFO_CLASS_HANDLE        baseType,            /* IN */
             int                         maxExactClasses,     /* IN */

src/coreclr/jit/gentree.cpp

@@ -14050,13 +14050,8 @@ GenTree* Compiler::gtFoldTypeCompare(GenTree* tree)
         return compare;
     }
 
-    // If one operand creates a type from a handle and the other operand is fetching the type from an object,
-    // we can sometimes optimize the type compare into a simpler
-    // method table comparison.
-    //
-    // TODO: if other operand is null...
-    if (!(((op1Kind == TPK_GetType) && (op2Kind == TPK_Handle)) ||
-          ((op1Kind == TPK_Handle) && (op2Kind == TPK_GetType))))
+    // In the following code, we need one of these to be a handle.
+    if (op1Kind != TPK_Handle && op2Kind != TPK_Handle)
     {
         return tree;
     }
@@ -14074,6 +14069,28 @@ GenTree* Compiler::gtFoldTypeCompare(GenTree* tree)
         return tree;
     }
 
+    // Check if an object of this type can even exist
+    if (info.compCompHnd->getExactClasses(clsHnd, 0, nullptr) == CORINFO_NEVER_INSTANTIATED)
+    {
+        JITDUMP("Runtime reported %p (%s) is never allocated\n", dspPtr(clsHnd), eeGetClassName(clsHnd));
+
+        const bool operatorIsEQ  = (oper == GT_EQ);
+        const int  compareResult = operatorIsEQ ? 0 : 1;
+        JITDUMP("Runtime reports comparison is known at jit time: %u\n", compareResult);
+        GenTree* result = gtNewIconNode(compareResult);
+        return result;
+    }
+
+    // If one operand creates a type from a handle and the other operand is fetching the type from an object,
+    // we can sometimes optimize the type compare into a simpler
+    // method table comparison.
+    //
+    // TODO: if other operand is null...
+    if (op1Kind != TPK_GetType && op2Kind != TPK_GetType)
+    {
+        return tree;
+    }
+
     // We're good to go.
     JITDUMP("Optimizing compare of obj.GetType()"
             " and type-from-handle to compare method table pointer\n");

src/coreclr/jit/importer.cpp

@@ -5349,14 +5349,39 @@ GenTree* Compiler::impOptimizeCastClassOrIsInst(GenTree* op1, CORINFO_RESOLVED_T
         return nullptr;
     }
 
+    CORINFO_CLASS_HANDLE toClass = pResolvedToken->hClass;
+    if (info.compCompHnd->getExactClasses(toClass, 0, nullptr) == CORINFO_NEVER_INSTANTIATED)
+    {
+        JITDUMP("\nClass %s%p (%s) can never be allocated\n", dspPtr(toClass), eeGetClassName(toClass));
+
+        if (!isCastClass)
+        {
+            JITDUMP("Cast will fail, optimizing to return null\n");
+
+            // If the cast was fed by a box, we can remove that too.
+            if (op1->IsBoxedValue())
+            {
+                JITDUMP("Also removing upstream box\n");
+                gtTryRemoveBoxUpstreamEffects(op1);
+            }
+
+            if (gtTreeHasSideEffects(op1, GTF_SIDE_EFFECT))
+            {
+                impAppendTree(op1, CHECK_SPILL_ALL, impCurStmtDI);
+            }
+            return gtNewNull();
+        }
+
+        JITDUMP("Cast will always throw, but not optimizing yet\n");
+    }
+
     // See what we know about the type of the object being cast.
     bool                 isExact   = false;
     bool                 isNonNull = false;
     CORINFO_CLASS_HANDLE fromClass = gtGetClassHandle(op1, &isExact, &isNonNull);
 
     if (fromClass != nullptr)
     {
-        CORINFO_CLASS_HANDLE toClass = pResolvedToken->hClass;
         JITDUMP("\nConsidering optimization of %s from %s%p (%s) to %p (%s)\n", isCastClass ? "castclass" : "isinst",
                 isExact ? "exact " : "", dspPtr(fromClass), eeGetClassName(fromClass), dspPtr(toClass),
                 eeGetClassName(toClass));

src/coreclr/tools/Common/Compiler/DevirtualizationManager.cs

@@ -203,13 +203,17 @@ protected virtual MethodDesc ResolveVirtualMethod(MethodDesc declMethod, DefType
 
 #if !READYTORUN
         /// <summary>
-        /// Gets a value indicating whether it might be possible to obtain a constructed type data structure for the given type.
+        /// Gets a value indicating whether it might be possible to obtain a constructed type data structure for the given type
+        /// in this compilation (i.e. is it possible to reference a constructed MethodTable symbol for this).
         /// </summary>
-        /// <remarks>
-        /// This is a bit of a hack, but devirtualization manager has a global view of all allocated types
-        /// so it can answer this question.
-        /// </remarks>
-        public virtual bool CanConstructType(TypeDesc type) => true;
+        public virtual bool CanReferenceConstructedMethodTable(TypeDesc type) => true;
+
+        /// <summary>
+        /// Gets a value indicating whether a (potentially canonically-equlivalent) constructed MethodTable could
+        /// exist. This is similar to <see cref="CanReferenceConstructedMethodTable"/>, but will return true
+        /// for List&lt;__Canon&gt; if a constructed MethodTable for List&lt;object&gt; exists.
+        /// </summary>
+        public virtual bool CanTypeOrCanonicalFormOfTypeBeAllocated(TypeDesc type) => true;
 
         public virtual TypeDesc[] GetImplementingClasses(TypeDesc type) => null;
 #endif

src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs

@@ -2874,12 +2874,22 @@ private TypeCompareState compareTypesForEquality(CORINFO_CLASS_STRUCT_* cls1, CO
             TypeDesc type1 = HandleToObject(cls1);
             TypeDesc type2 = HandleToObject(cls2);
 
-            return TypeExtensions.CompareTypesForEquality(type1, type2) switch
+            TypeCompareState result = TypeExtensions.CompareTypesForEquality(type1, type2) switch
             {
                 true => TypeCompareState.Must,
                 false => TypeCompareState.MustNot,
                 _ => TypeCompareState.May,
             };
+
+#if !READYTORUN
+            if (result == TypeCompareState.May
+                && (canNeverHaveInstanceOfSubclassOf(type1) || canNeverHaveInstanceOfSubclassOf(type2)))
+            {
+                return TypeCompareState.MustNot;
+            }
+#endif
+
+            return result;
         }
 
         private bool isMoreSpecificType(CORINFO_CLASS_STRUCT_* cls1, CORINFO_CLASS_STRUCT_* cls2)
@@ -4284,7 +4294,7 @@ private HRESULT getPgoInstrumentationResults(CORINFO_METHOD_STRUCT_* ftnHnd, ref
 #pragma warning disable SA1001, SA1113, SA1115 // Commas should be spaced correctly
                     ComputeJitPgoInstrumentationSchema(ObjectToHandle, pgoResultsSchemas, out var nativeSchemas, _cachedMemoryStream
 #if !READYTORUN
-                        , _compilation.CanConstructType
+                        , _compilation.CanReferenceConstructedMethodTable
 #endif
                         );
 #pragma warning restore SA1001, SA1113, SA1115 // Commas should be spaced correctly

src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs

@@ -26,6 +26,7 @@ public static class CORINFO
         public const ushort USEHELPER = 0xffff;
         public const ushort USENULL = 0xfffe;
         public const ushort CORINFO_NO_SIZE_CHECK = 0xffff;
+        public const int CORINFO_NEVER_INSTANTIATED = -1;
     }
 
     public struct CORINFO_METHOD_STRUCT_

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/Compilation.cs

@@ -103,9 +103,14 @@ public bool CanInline(MethodDesc caller, MethodDesc callee)
             return _inliningPolicy.CanInline(caller, callee);
         }
 
-        public bool CanConstructType(TypeDesc type)
+        public bool CanReferenceConstructedMethodTable(TypeDesc type)
         {
-            return NodeFactory.DevirtualizationManager.CanConstructType(type);
+            return NodeFactory.DevirtualizationManager.CanReferenceConstructedMethodTable(type);
+        }
+
+        public bool CanTypeOrCanonicalFormOfTypeBeAllocated(TypeDesc type)
+        {
+            return NodeFactory.DevirtualizationManager.CanTypeOrCanonicalFormOfTypeBeAllocated(type);
         }
 
         public DelegateCreationInfo GetDelegateCtor(TypeDesc delegateType, MethodDesc target, TypeDesc constrainedType, bool followVirtualDispatch)
@@ -261,7 +266,7 @@ public bool NeedsRuntimeLookup(ReadyToRunHelperId lookupKind, object targetOfLoo
 
         public ReadyToRunHelperId GetLdTokenHelperForType(TypeDesc type)
         {
-            bool canConstructPerWholeProgramAnalysis = NodeFactory.DevirtualizationManager.CanConstructType(type);
+            bool canConstructPerWholeProgramAnalysis = NodeFactory.DevirtualizationManager.CanReferenceConstructedMethodTable(type);
             bool creationAllowed = ConstructedEETypeNode.CreationAllowed(type);
             return (canConstructPerWholeProgramAnalysis && creationAllowed)
                 ? ReadyToRunHelperId.TypeHandle

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ILScanner.cs

@@ -413,7 +413,8 @@ public override DictionaryLayoutNode GetLayout(TypeSystemEntity methodOrType)
 
         private sealed class ScannedDevirtualizationManager : DevirtualizationManager
         {
-            private HashSet<TypeDesc> _constructedTypes = new HashSet<TypeDesc>();
+            private HashSet<TypeDesc> _constructedMethodTables = new HashSet<TypeDesc>();
+            private HashSet<TypeDesc> _canonConstructedMethodTables = new HashSet<TypeDesc>();
             private HashSet<TypeDesc> _canonConstructedTypes = new HashSet<TypeDesc>();
             private HashSet<TypeDesc> _unsealedTypes = new HashSet<TypeDesc>();
             private Dictionary<TypeDesc, HashSet<TypeDesc>> _implementators = new();
@@ -442,7 +443,8 @@ public ScannedDevirtualizationManager(NodeFactory factory, ImmutableArray<Depend
 
                     if (type != null)
                     {
-                        _constructedTypes.Add(type);
+                        _constructedMethodTables.Add(type);
+                        _canonConstructedMethodTables.Add(type.ConvertToCanonForm(CanonicalFormKind.Specific));
 
                         if (type.IsInterface)
                         {
@@ -687,7 +689,11 @@ protected override MethodDesc ResolveVirtualMethod(MethodDesc declMethod, DefTyp
                 return result;
             }
 
-            public override bool CanConstructType(TypeDesc type) => _constructedTypes.Contains(type);
+            public override bool CanReferenceConstructedMethodTable(TypeDesc type)
+                => _constructedMethodTables.Contains(type);
+
+            public override bool CanTypeOrCanonicalFormOfTypeBeAllocated(TypeDesc type)
+                => _constructedMethodTables.Contains(type) || _canonConstructedMethodTables.Contains(type);
 
             public override TypeDesc[] GetImplementingClasses(TypeDesc type)
             {

src/coreclr/tools/aot/ILCompiler.RyuJit/Compiler/RyuJitCompilation.cs

@@ -72,7 +72,7 @@ public override IEETypeNode NecessaryTypeSymbolIfPossible(TypeDesc type)
             // information proving that it isn't, give RyuJIT the constructed symbol even
             // though we just need the unconstructed one.
             // https://github.com/dotnet/runtimelab/issues/1128
-            bool canPotentiallyConstruct = NodeFactory.DevirtualizationManager.CanConstructType(type);
+            bool canPotentiallyConstruct = NodeFactory.DevirtualizationManager.CanReferenceConstructedMethodTable(type);
             if (canPotentiallyConstruct)
                 return _nodeFactory.MaximallyConstructableType(type);
 
@@ -81,7 +81,7 @@ public override IEETypeNode NecessaryTypeSymbolIfPossible(TypeDesc type)
 
         public FrozenRuntimeTypeNode NecessaryRuntimeTypeIfPossible(TypeDesc type)
         {
-            bool canPotentiallyConstruct = NodeFactory.DevirtualizationManager.CanConstructType(type);
+            bool canPotentiallyConstruct = NodeFactory.DevirtualizationManager.CanReferenceConstructedMethodTable(type);
             if (canPotentiallyConstruct)
                 return _nodeFactory.SerializedMaximallyConstructableRuntimeTypeObject(type);
 

src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs

@@ -2261,6 +2261,24 @@ private void getFieldInfo(ref CORINFO_RESOLVED_TOKEN pResolvedToken, CORINFO_MET
             //       and STS::AccessCheck::CanAccess.
         }
 
+        private bool canNeverHaveInstanceOfSubclassOf(TypeDesc type)
+        {
+            // Don't try to optimize nullable
+            if (type.IsNullable)
+                return false;
+
+            // We don't track unconstructable types very well and they are rare anyway
+            if (!ConstructedEETypeNode.CreationAllowed(type))
+                return false;
+
+            TypeDesc canonType = type.ConvertToCanonForm(CanonicalFormKind.Specific);
+
+            // If we don't have a constructed MethodTable for the exact type or for its template,
+            // this type or any of its subclasses can never be instantiated.
+            return !_compilation.CanTypeOrCanonicalFormOfTypeBeAllocated(type)
+                && (type == canonType || !_compilation.CanReferenceConstructedMethodTable(canonType));
+        }
+
         private int getExactClasses(CORINFO_CLASS_STRUCT_* baseType, int maxExactClasses, CORINFO_CLASS_STRUCT_** exactClsRet)
         {
             MetadataType type = HandleToObject(baseType) as MetadataType;
@@ -2269,6 +2287,14 @@ private int getExactClasses(CORINFO_CLASS_STRUCT_* baseType, int maxExactClasses
                 return 0;
             }
 
+            if (maxExactClasses == 0)
+            {
+                if (canNeverHaveInstanceOfSubclassOf(type))
+                    return CORINFO.CORINFO_NEVER_INSTANTIATED;
+
+                return 0;
+            }
+
             // type is already sealed, return it
             if (_compilation.IsEffectivelySealed(type))
             {