Move to lower

dotnet · May 1, 2024 · 7f24e1b · 7f24e1b
1 parent aab8803
commit 7f24e1b
Show file tree

Hide file tree

Showing 19 changed files with 127 additions and 26 deletions.
diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h
@@ -43,11 +43,11 @@ typedef const GUID *LPCGUID;
 #define GUID_DEFINED
 #endif // !GUID_DEFINED
 
-constexpr GUID JITEEVersionIdentifier = { /* 32d71f8e-c1f5-41cb-88cc-4e8504cabf40 */
-    0x32d71f8e,
-    0xc1f5,
-    0x41cb,
-    {0x88, 0xcc, 0x4e, 0x85, 0x04, 0xca, 0xbf, 0x40}
+constexpr GUID JITEEVersionIdentifier = { /* bd8c41d4-8531-49c1-a600-0ae9bfe05de1 */
+    0xbd8c41d4,
+    0x8531,
+    0x49c1,
+    {0xa6, 0x00, 0x0a, 0xe9, 0xbf, 0xe0, 0x5d, 0xe1}
   };
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////

diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h
@@ -153,8 +153,7 @@
     JITHELPER(CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, JIT_WriteBarrierEnsureNonHeapTarget,CORINFO_HELP_SIG_REG_ONLY)
 
     DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, JIT_ByRefWriteBarrier,CORINFO_HELP_SIG_NO_ALIGN_STUB)
-
-    JITHELPER(CORINFO_HELP_ASSIGN_STRUCT,       JIT_StructWriteBarrier,CORINFO_HELP_SIG_4_STACK)
+    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_STRUCT, NULL, CORINFO_HELP_SIG_REG_ONLY)
 
     // Accessing fields
     JITHELPER(CORINFO_HELP_GETFIELD8,                   JIT_GetField8,CORINFO_HELP_SIG_REG_ONLY)

diff --git a/src/coreclr/inc/readytorun.h b/src/coreclr/inc/readytorun.h
@@ -20,7 +20,7 @@
 // If you update this, ensure you run `git grep MINIMUM_READYTORUN_MAJOR_VERSION`
 // and handle pending work.
 #define READYTORUN_MAJOR_VERSION 0x0009
-#define READYTORUN_MINOR_VERSION 0x0002
+#define READYTORUN_MINOR_VERSION 0x0003
 
 #define MINIMUM_READYTORUN_MAJOR_VERSION 0x009
 
@@ -34,6 +34,7 @@
 // R2R Version 9.0 adds support for the Vector512 type
 // R2R Version 9.1 adds new helpers to allocate objects on frozen segments
 // R2R Version 9.2 adds MemZero and NativeMemSet helpers
+// R2R Version 9.3 adds AssignStruct helper
 
 
 struct READYTORUN_CORE_HEADER
@@ -321,6 +322,7 @@ enum ReadyToRunHelper
     READYTORUN_HELPER_WriteBarrier              = 0x30,
     READYTORUN_HELPER_CheckedWriteBarrier       = 0x31,
     READYTORUN_HELPER_ByRefWriteBarrier         = 0x32,
+    READYTORUN_HELPER_AssignStruct              = 0x33,
 
     // Array helpers
     READYTORUN_HELPER_Stelem_Ref                = 0x38,

diff --git a/src/coreclr/inc/readytorunhelpers.h b/src/coreclr/inc/readytorunhelpers.h
@@ -24,6 +24,7 @@ HELPER(READYTORUN_HELPER_ThrowDivZero,              CORINFO_HELP_THROWDIVZERO,
 HELPER(READYTORUN_HELPER_WriteBarrier,              CORINFO_HELP_ASSIGN_REF,                        )
 HELPER(READYTORUN_HELPER_CheckedWriteBarrier,       CORINFO_HELP_CHECKED_ASSIGN_REF,                )
 HELPER(READYTORUN_HELPER_ByRefWriteBarrier,         CORINFO_HELP_ASSIGN_BYREF,                      )
+HELPER(READYTORUN_HELPER_AssignStruct,              CORINFO_HELP_ASSIGN_STRUCT,                     )
 
 HELPER(READYTORUN_HELPER_Stelem_Ref,                CORINFO_HELP_ARRADDR_ST,                        )
 HELPER(READYTORUN_HELPER_Ldelema_Ref,               CORINFO_HELP_LDELEMA_REF,                       )

diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h
@@ -7803,6 +7803,7 @@ class Compiler
     GenTree*     optVNBasedFoldConstExpr(BasicBlock* block, GenTree* parent, GenTree* tree);
     GenTree*     optVNBasedFoldExpr(BasicBlock* block, GenTree* parent, GenTree* tree);
     GenTree*     optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, GenTreeCall* call);
+    GenTree*     optVNBasedFoldExpr_Blk(BasicBlock* block, GenTree* parent, GenTreeBlk* blk);
     GenTree*     optExtractSideEffListFromConst(GenTree* tree);
 
     AssertionIndex GetAssertionCount()

diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp
@@ -8212,6 +8212,91 @@ void Lowering::ContainCheckBitCast(GenTree* node)
     }
 }
 
+bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk)
+{
+    if (comp->opts.OptimizationDisabled() || !ISMETHOD("Test"))
+    {
+        return false;
+    }
+
+    // Replace STORE_BLK (struct copy) with CORINFO_HELP_ASSIGN_STRUCT which performs
+    // bulk copy for byrefs.
+    const unsigned bulkCopyThreshold = 4;
+    if (blk->OperIs(GT_STORE_BLK) && !blk->OperIsInitBlkOp() &&
+        (blk->GetLayout()->GetGCPtrCount() >= bulkCopyThreshold))
+    {
+        GenTree* addr = blk->Addr();
+        GenTree* data = blk->Data();
+
+        const unsigned gcPtrs = blk->GetLayout()->GetGCPtrCount();
+        if (!CheckedOps::MulOverflows((int)gcPtrs, TARGET_POINTER_SIZE, true))
+        {
+            if (data->OperIs(GT_IND))
+            {
+                // Drop GT_IND nodes
+                BlockRange().Remove(data);
+                data = data->AsIndir()->Addr();
+            }
+            else
+            {
+                assert(data->OperIs(GT_LCL_VAR, GT_LCL_FLD));
+
+                // Convert local to LCL_ADDR
+                unsigned lclOffset = data->AsLclVarCommon()->GetLclOffs();
+                data->ChangeOper(GT_LCL_ADDR);
+                data->ChangeType(TYP_I_IMPL);
+                data->AsLclFld()->SetLclOffs(lclOffset);
+                data->ClearContained();
+            }
+
+            // Size is a constant
+            GenTreeIntCon* size = comp->gtNewIconNode((ssize_t)gcPtrs * TARGET_POINTER_SIZE, TYP_I_IMPL);
+            BlockRange().InsertBefore(data, size);
+
+            // A hacky way to safely call fgMorphTree in Lower
+            GenTree* destPlaceholder = comp->gtNewZeroConNode(addr->TypeGet());
+            GenTree* dataPlaceholder = comp->gtNewZeroConNode(genActualType(data));
+            GenTree* sizePlaceholder = comp->gtNewZeroConNode(genActualType(size));
+
+            GenTreeCall* call = comp->gtNewHelperCallNode(CORINFO_HELP_ASSIGN_STRUCT, TYP_VOID, destPlaceholder, dataPlaceholder, sizePlaceholder);
+            comp->fgMorphArgs(call);
+
+            LIR::Range range = LIR::SeqTree(comp, call);
+            GenTree* rangeStart = range.FirstNode();
+            GenTree* rangeEnd = range.LastNode();
+
+            BlockRange().InsertBefore(blk, std::move(range));
+            blk->gtBashToNOP();
+
+            LIR::Use destUse;
+            LIR::Use sizeUse;
+            BlockRange().TryGetUse(destPlaceholder, &destUse);
+            BlockRange().TryGetUse(sizePlaceholder, &sizeUse);
+            destUse.ReplaceWith(addr);
+            sizeUse.ReplaceWith(size);
+            destPlaceholder->SetUnusedValue();
+            sizePlaceholder->SetUnusedValue();
+
+            LIR::Use dataUse;
+            BlockRange().TryGetUse(dataPlaceholder, &dataUse);
+            dataUse.ReplaceWith(data);
+            dataPlaceholder->SetUnusedValue();
+
+            LowerRange(rangeStart, rangeEnd);
+
+            // Finally move all GT_PUTARG_* nodes
+            // Re-use the existing logic for CFG call args here
+            MoveCFGCallArgs(call);
+
+            BlockRange().Remove(destPlaceholder);
+            BlockRange().Remove(sizePlaceholder);
+            BlockRange().Remove(dataPlaceholder);
+            return true;
+        }
+    }
+    return false;
+}
+
 //------------------------------------------------------------------------
 // LowerBlockStoreAsHelperCall: Lower a block store node as a memset/memcpy call
 //

diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h
@@ -334,6 +334,7 @@ class Lowering final : public Phase
     void     LowerBlockStore(GenTreeBlk* blkNode);
     void     LowerBlockStoreCommon(GenTreeBlk* blkNode);
     void     LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode);
+    bool     TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blkNode);
     void     LowerLclHeap(GenTree* node);
     void     ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenTree* addr, GenTree* addrParent);
     void     LowerPutArgStkOrSplit(GenTreePutArgStk* putArgNode);

diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp
@@ -457,6 +457,11 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode)
 
         if (doCpObj)
         {
+            if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode))
+            {
+                return;
+            }
+
             assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL));
 
             // If we have a long enough sequence of slots that do not require write barriers then

diff --git a/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h b/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h
@@ -12,7 +12,7 @@ struct ReadyToRunHeaderConstants
     static const uint32_t Signature = 0x00525452; // 'RTR'
 
     static const uint32_t CurrentMajorVersion = 9;
-    static const uint32_t CurrentMinorVersion = 2;
+    static const uint32_t CurrentMinorVersion = 3;
 };
 
 struct ReadyToRunHeader

diff --git a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs
@@ -16,7 +16,7 @@ internal struct ReadyToRunHeaderConstants
         public const uint Signature = 0x00525452; // 'RTR'
 
         public const ushort CurrentMajorVersion = 9;
-        public const ushort CurrentMinorVersion = 2;
+        public const ushort CurrentMinorVersion = 3;
     }
 #if READYTORUN
 #pragma warning disable 0169

diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs
@@ -233,6 +233,7 @@ public enum ReadyToRunHelper
         WriteBarrier                = 0x30,
         CheckedWriteBarrier         = 0x31,
         ByRefWriteBarrier           = 0x32,
+        AssignStruct                = 0x33,
 
         // Array helpers
         Stelem_Ref                  = 0x38,

diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs
@@ -67,6 +67,9 @@ public static void GetEntryPoint(TypeSystemContext context, ReadyToRunHelper id,
                 case ReadyToRunHelper.CheckedWriteBarrier:
                     mangledName = context.Target.Architecture == TargetArchitecture.ARM64 ? "RhpCheckedAssignRefArm64" : "RhpCheckedAssignRef";
                     break;
+                case ReadyToRunHelper.AssignStruct:
+                   mangledName = "RhBuffer_BulkMoveWithWriteBarrier";
+                   break;
                 case ReadyToRunHelper.ByRefWriteBarrier:
                     mangledName = context.Target.Architecture == TargetArchitecture.ARM64 ? "RhpByRefAssignRefArm64" : "RhpByRefAssignRef";
                     break;

diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs
@@ -1000,6 +1000,9 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum)
                 case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_BYREF:
                     id = ReadyToRunHelper.ByRefWriteBarrier;
                     break;
+                case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_STRUCT:
+                    id = ReadyToRunHelper.AssignStruct;
+                    break;
 
                 case CorInfoHelpFunc.CORINFO_HELP_ARRADDR_ST:
                     id = ReadyToRunHelper.Stelem_Ref;

diff --git a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs
@@ -1672,6 +1672,10 @@ private void ParseHelper(StringBuilder builder)
                     builder.Append("BYREF_WRITE_BARRIER");
                     break;
 
+                case ReadyToRunHelper.AssignStruct:
+                    builder.Append("ASSIGN_STRUCT");
+                    break;
+
                 // Array helpers
                 case ReadyToRunHelper.Stelem_Ref:
                     builder.Append("STELEM_REF");

diff --git a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs
@@ -497,6 +497,9 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum)
                 case CorInfoHelpFunc.CORINFO_HELP_CHECKED_ASSIGN_REF:
                     id = ReadyToRunHelper.CheckedWriteBarrier;
                     break;
+                case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_STRUCT:
+                    id = ReadyToRunHelper.AssignStruct;
+                    break;
                 case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_BYREF:
                     id = ReadyToRunHelper.ByRefWriteBarrier;
                     break;

diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h
@@ -891,6 +891,7 @@ DEFINE_METHOD(DEBUGGER,             BREAK,                  Break,
 DEFINE_CLASS(BUFFER,                System,                 Buffer)
 DEFINE_METHOD(BUFFER,               MEMCPY_PTRBYTE_ARRBYTE, Memcpy,                 SM_PtrByte_Int_ArrByte_Int_Int_RetVoid)
 DEFINE_METHOD(BUFFER,               MEMCPY,                 Memcpy,                 SM_PtrByte_PtrByte_Int_RetVoid)
+DEFINE_METHOD(BUFFER,               MEMCOPYGC,              BulkMoveWithWriteBarrier, SM_RefByte_RefByte_UIntPtr_RetVoid)
 
 DEFINE_CLASS(STUBHELPERS,           StubHelpers,            StubHelpers)
 DEFINE_METHOD(STUBHELPERS,          GET_DELEGATE_TARGET,    GetDelegateTarget,          SM_Delegate_RetIntPtr)

diff --git a/src/coreclr/vm/ecall.cpp b/src/coreclr/vm/ecall.cpp
@@ -157,6 +157,10 @@ void ECall::PopulateManagedHelpers()
     pDest = pMD->GetMultiCallableAddrOfCode();
     SetJitHelperFunction(CORINFO_HELP_MEMCPY, pDest);
 
+    pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__BUFFER__MEMCOPYGC));
+    pDest = pMD->GetMultiCallableAddrOfCode();
+    SetJitHelperFunction(CORINFO_HELP_ASSIGN_STRUCT, pDest);
+
     pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__MATH__ROUND));
     pDest = pMD->GetMultiCallableAddrOfCode();
     SetJitHelperFunction(CORINFO_HELP_DBLROUND, pDest);

diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp
@@ -55,8 +55,8 @@
 #ifndef FEATURE_EH_FUNCLETS
 #include "excep.h"
 #endif
-
 #include "exinfo.h"
+#include "arraynative.inl"
 
 using std::isfinite;
 using std::isnan;
@@ -4779,21 +4779,6 @@ HCIMPLEND
 //
 //========================================================================
 
-/*************************************************************/
-HCIMPL3(VOID, JIT_StructWriteBarrier, void *dest, void* src, CORINFO_CLASS_HANDLE typeHnd_)
-{
-    FCALL_CONTRACT;
-
-    TypeHandle typeHnd(typeHnd_);
-    MethodTable *pMT = typeHnd.AsMethodTable();
-
-    HELPER_METHOD_FRAME_BEGIN_NOPOLL();    // Set up a frame
-    CopyValueClass(dest, src, pMT);
-    HELPER_METHOD_FRAME_END_POLL();
-
-}
-HCIMPLEND
-
 /*************************************************************/
 // Slow helper to tailcall from the fast one
 NOINLINE HCIMPL0(void, JIT_PollGC_Framed)

diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffer.cs b/src/libraries/System.Private.CoreLib/src/System/Buffer.cs
@@ -177,6 +177,9 @@ ref Unsafe.As<T, byte>(ref source),
         private const uint BulkMoveWithWriteBarrierChunk = 0x4000;
 #endif
 
+#if NATIVEAOT
+        [System.Runtime.RuntimeExport("RhBuffer_BulkMoveWithWriteBarrier")]
+#endif
         internal static void BulkMoveWithWriteBarrier(ref byte destination, ref byte source, nuint byteCount)
         {
             if (byteCount <= BulkMoveWithWriteBarrierChunk)