Fix loading of R2R assemblies from byte arrays/streams (#74118)

* enable test

* flat layout is ok if it is not a file

* do not convert non-file layouts on OSX

* run some code when loading from byte array

* use flat layout on all platforms

* Suggestion from PR feedback

Co-authored-by: Jan Vorlicek <jan.vorlicek@volny.cz>

* typo

Co-authored-by: Jan Vorlicek <jan.vorlicek@volny.cz>

src/coreclr/vm/peimagelayout.cpp

@@ -85,16 +85,18 @@ PEImageLayout* PEImageLayout::LoadConverted(PEImage* pOwner)
     // ConvertedImageLayout may be able to handle them, but the fact that we were unable to
     // load directly implies that MAPMapPEFile could not consume what crossgen produced.
     // that is suspicious, one or another might have a bug.
-    _ASSERTE(!pFlat->HasReadyToRunHeader());
+    _ASSERTE(!pOwner->IsFile() || !pFlat->HasReadyToRunHeader());
 #endif
 
-    if (!pFlat->HasReadyToRunHeader() && !pFlat->HasWriteableSections())
+    // ignore R2R if the image is not a file.
+    if ((pFlat->HasReadyToRunHeader() && pOwner->IsFile()) ||
+        pFlat->HasWriteableSections())
     {
-        // we can use flat layout for this
-        return pFlat.Extract();
+        return new ConvertedImageLayout(pFlat);
     }
 
-    return new ConvertedImageLayout(pFlat);
+    // we can use flat layout for this
+    return pFlat.Extract();
 }
 
 PEImageLayout* PEImageLayout::Load(PEImage* pOwner, HRESULT* loadFailure)
@@ -448,7 +450,7 @@ ConvertedImageLayout::ConvertedImageLayout(FlatImageLayout* source)
 
     IfFailThrow(Init(loadedImage));
 
-    if (IsNativeMachineFormat() && g_fAllowNativeImages)
+    if (m_pOwner->IsFile() && IsNativeMachineFormat() && g_fAllowNativeImages)
     {
         // Do base relocation and exception hookup, if necessary.
         // otherwise R2R will be disabled for this image.

src/tests/readytorun/tests/main.cs

@@ -414,6 +414,13 @@ static void RVAFieldTest()
             Assert.AreEqual(value[i], (byte)(9 - i));
     }
 
+    // public constructor, so we run something when loading from byte array in the test below
+    public Program()
+    {
+        // do something in the constructor to see if it works
+        TestVirtualMethodCalls();
+    }
+
     static void TestLoadR2RImageFromByteArray()
     {
         Assembly assembly1 = typeof(Program).Assembly;
@@ -422,6 +429,8 @@ static void TestLoadR2RImageFromByteArray()
         Assembly assembly2 = Assembly.Load(array);
 
         Assert.AreEqual(assembly2.FullName, assembly1.FullName);
+
+        assembly2.CreateInstance("Program");
     }
 
     [MethodImplAttribute(MethodImplOptions.NoInlining)]
@@ -513,9 +522,8 @@ static void RunAllTests()
         Console.WriteLine("RVAFieldTest");
         RVAFieldTest();
 
-//        Disable for https://github.com/dotnet/runtime/issues/71507
-//        Console.WriteLine("TestLoadR2RImageFromByteArray");
-//        TestLoadR2RImageFromByteArray();
+        Console.WriteLine("TestLoadR2RImageFromByteArray");
+        TestLoadR2RImageFromByteArray();
 
         Console.WriteLine("TestILBodyChange");
         TestILBodyChange();