Move to lower

Author: EgorBo

src/coreclr/inc/jiteeversionguid.h

@@ -43,11 +43,11 @@ typedef const GUID *LPCGUID;
 #define GUID_DEFINED
 #endif // !GUID_DEFINED
 
-constexpr GUID JITEEVersionIdentifier = { /* 32d71f8e-c1f5-41cb-88cc-4e8504cabf40 */
-    0x32d71f8e,
-    0xc1f5,
-    0x41cb,
-    {0x88, 0xcc, 0x4e, 0x85, 0x04, 0xca, 0xbf, 0x40}
+constexpr GUID JITEEVersionIdentifier = { /* bd8c41d4-8531-49c1-a600-0ae9bfe05de1 */
+    0xbd8c41d4,
+    0x8531,
+    0x49c1,
+    {0xa6, 0x00, 0x0a, 0xe9, 0xbf, 0xe0, 0x5d, 0xe1}
   };
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////

src/coreclr/inc/jithelpers.h

@@ -153,8 +153,7 @@
     JITHELPER(CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, JIT_WriteBarrierEnsureNonHeapTarget,CORINFO_HELP_SIG_REG_ONLY)
 
     DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, JIT_ByRefWriteBarrier,CORINFO_HELP_SIG_NO_ALIGN_STUB)
-
-    JITHELPER(CORINFO_HELP_ASSIGN_STRUCT,       JIT_StructWriteBarrier,CORINFO_HELP_SIG_4_STACK)
+    DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_STRUCT, NULL, CORINFO_HELP_SIG_REG_ONLY)
 
     // Accessing fields
     JITHELPER(CORINFO_HELP_GETFIELD8,                   JIT_GetField8,CORINFO_HELP_SIG_REG_ONLY)

src/coreclr/inc/readytorun.h

@@ -20,7 +20,7 @@
 // If you update this, ensure you run `git grep MINIMUM_READYTORUN_MAJOR_VERSION`
 // and handle pending work.
 #define READYTORUN_MAJOR_VERSION 0x0009
-#define READYTORUN_MINOR_VERSION 0x0002
+#define READYTORUN_MINOR_VERSION 0x0003
 
 #define MINIMUM_READYTORUN_MAJOR_VERSION 0x009
 
@@ -34,6 +34,7 @@
 // R2R Version 9.0 adds support for the Vector512 type
 // R2R Version 9.1 adds new helpers to allocate objects on frozen segments
 // R2R Version 9.2 adds MemZero and NativeMemSet helpers
+// R2R Version 9.3 adds AssignStruct helper
 
 
 struct READYTORUN_CORE_HEADER
@@ -321,6 +322,7 @@ enum ReadyToRunHelper
     READYTORUN_HELPER_WriteBarrier              = 0x30,
     READYTORUN_HELPER_CheckedWriteBarrier       = 0x31,
     READYTORUN_HELPER_ByRefWriteBarrier         = 0x32,
+    READYTORUN_HELPER_AssignStruct              = 0x33,
 
     // Array helpers
     READYTORUN_HELPER_Stelem_Ref                = 0x38,

src/coreclr/inc/readytorunhelpers.h

@@ -24,6 +24,7 @@ HELPER(READYTORUN_HELPER_ThrowDivZero,              CORINFO_HELP_THROWDIVZERO,
 HELPER(READYTORUN_HELPER_WriteBarrier,              CORINFO_HELP_ASSIGN_REF,                        )
 HELPER(READYTORUN_HELPER_CheckedWriteBarrier,       CORINFO_HELP_CHECKED_ASSIGN_REF,                )
 HELPER(READYTORUN_HELPER_ByRefWriteBarrier,         CORINFO_HELP_ASSIGN_BYREF,                      )
+HELPER(READYTORUN_HELPER_AssignStruct,              CORINFO_HELP_ASSIGN_STRUCT,                     )
 
 HELPER(READYTORUN_HELPER_Stelem_Ref,                CORINFO_HELP_ARRADDR_ST,                        )
 HELPER(READYTORUN_HELPER_Ldelema_Ref,               CORINFO_HELP_LDELEMA_REF,                       )

src/coreclr/jit/compiler.h

@@ -7803,6 +7803,7 @@ class Compiler
     GenTree*     optVNBasedFoldConstExpr(BasicBlock* block, GenTree* parent, GenTree* tree);
     GenTree*     optVNBasedFoldExpr(BasicBlock* block, GenTree* parent, GenTree* tree);
     GenTree*     optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, GenTreeCall* call);
+    GenTree*     optVNBasedFoldExpr_Blk(BasicBlock* block, GenTree* parent, GenTreeBlk* blk);
     GenTree*     optExtractSideEffListFromConst(GenTree* tree);
 
     AssertionIndex GetAssertionCount()

src/coreclr/jit/lower.cpp

@@ -8212,6 +8212,91 @@ void Lowering::ContainCheckBitCast(GenTree* node)
     }
 }
 
+bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk)
+{
+    if (comp->opts.OptimizationDisabled() || !ISMETHOD("Test"))
+    {
+        return false;
+    }
+
+    // Replace STORE_BLK (struct copy) with CORINFO_HELP_ASSIGN_STRUCT which performs
+    // bulk copy for byrefs.
+    const unsigned bulkCopyThreshold = 4;
+    if (blk->OperIs(GT_STORE_BLK) && !blk->OperIsInitBlkOp() &&
+        (blk->GetLayout()->GetGCPtrCount() >= bulkCopyThreshold))
+    {
+        GenTree* addr = blk->Addr();
+        GenTree* data = blk->Data();
+
+        const unsigned gcPtrs = blk->GetLayout()->GetGCPtrCount();
+        if (!CheckedOps::MulOverflows((int)gcPtrs, TARGET_POINTER_SIZE, true))
+        {
+            if (data->OperIs(GT_IND))
+            {
+                // Drop GT_IND nodes
+                BlockRange().Remove(data);
+                data = data->AsIndir()->Addr();
+            }
+            else
+            {
+                assert(data->OperIs(GT_LCL_VAR, GT_LCL_FLD));
+
+                // Convert local to LCL_ADDR
+                unsigned lclOffset = data->AsLclVarCommon()->GetLclOffs();
+                data->ChangeOper(GT_LCL_ADDR);
+                data->ChangeType(TYP_I_IMPL);
+                data->AsLclFld()->SetLclOffs(lclOffset);
+                data->ClearContained();
+            }
+
+            // Size is a constant
+            GenTreeIntCon* size = comp->gtNewIconNode((ssize_t)gcPtrs * TARGET_POINTER_SIZE, TYP_I_IMPL);
+            BlockRange().InsertBefore(data, size);
+
+            // A hacky way to safely call fgMorphTree in Lower
+            GenTree* destPlaceholder = comp->gtNewZeroConNode(addr->TypeGet());
+            GenTree* dataPlaceholder = comp->gtNewZeroConNode(genActualType(data));
+            GenTree* sizePlaceholder = comp->gtNewZeroConNode(genActualType(size));
+
+            GenTreeCall* call = comp->gtNewHelperCallNode(CORINFO_HELP_ASSIGN_STRUCT, TYP_VOID, destPlaceholder, dataPlaceholder, sizePlaceholder);
+            comp->fgMorphArgs(call);
+
+            LIR::Range range = LIR::SeqTree(comp, call);
+            GenTree* rangeStart = range.FirstNode();
+            GenTree* rangeEnd = range.LastNode();
+
+            BlockRange().InsertBefore(blk, std::move(range));
+            blk->gtBashToNOP();
+
+            LIR::Use destUse;
+            LIR::Use sizeUse;
+            BlockRange().TryGetUse(destPlaceholder, &destUse);
+            BlockRange().TryGetUse(sizePlaceholder, &sizeUse);
+            destUse.ReplaceWith(addr);
+            sizeUse.ReplaceWith(size);
+            destPlaceholder->SetUnusedValue();
+            sizePlaceholder->SetUnusedValue();
+
+            LIR::Use dataUse;
+            BlockRange().TryGetUse(dataPlaceholder, &dataUse);
+            dataUse.ReplaceWith(data);
+            dataPlaceholder->SetUnusedValue();
+
+            LowerRange(rangeStart, rangeEnd);
+
+            // Finally move all GT_PUTARG_* nodes
+            // Re-use the existing logic for CFG call args here
+            MoveCFGCallArgs(call);
+
+            BlockRange().Remove(destPlaceholder);
+            BlockRange().Remove(sizePlaceholder);
+            BlockRange().Remove(dataPlaceholder);
+            return true;
+        }
+    }
+    return false;
+}
+
 //------------------------------------------------------------------------
 // LowerBlockStoreAsHelperCall: Lower a block store node as a memset/memcpy call
 //

src/coreclr/jit/lower.h

@@ -334,6 +334,7 @@ class Lowering final : public Phase
     void     LowerBlockStore(GenTreeBlk* blkNode);
     void     LowerBlockStoreCommon(GenTreeBlk* blkNode);
     void     LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode);
+    bool     TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blkNode);
     void     LowerLclHeap(GenTree* node);
     void     ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenTree* addr, GenTree* addrParent);
     void     LowerPutArgStkOrSplit(GenTreePutArgStk* putArgNode);

src/coreclr/jit/lowerxarch.cpp

@@ -457,6 +457,11 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode)
 
         if (doCpObj)
         {
+            if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode))
+            {
+                return;
+            }
+
             assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL));
 
             // If we have a long enough sequence of slots that do not require write barriers then

src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h

@@ -12,7 +12,7 @@ struct ReadyToRunHeaderConstants
     static const uint32_t Signature = 0x00525452; // 'RTR'
 
     static const uint32_t CurrentMajorVersion = 9;
-    static const uint32_t CurrentMinorVersion = 2;
+    static const uint32_t CurrentMinorVersion = 3;
 };
 
 struct ReadyToRunHeader

src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs

@@ -16,7 +16,7 @@ internal struct ReadyToRunHeaderConstants
         public const uint Signature = 0x00525452; // 'RTR'
 
         public const ushort CurrentMajorVersion = 9;
-        public const ushort CurrentMinorVersion = 2;
+        public const ushort CurrentMinorVersion = 3;
     }
 #if READYTORUN
 #pragma warning disable 0169