Fix #1977: always create RBP chains on Unix

The JIT will now always create RBP chains on Unix platforms.

To do this, the VM is extended with a new Unix-only AMD64 unwind code: UWOP_SET_FPREG_LARGE.
The existing unwind code which is used to establish a frame pointer, UWOP_SET_FPREG, requires
that the frame pointer, when established, be no more than 240 bytes offset from the stack pointer.
This doesn't work well for frames that use localloc. (Large frames are ok, because we don't
report the frame pointer in the unwind info except for in functions with localloc or EnC.)

The new code has a 32-bit range, scaled by 16. If used, UNWIND_INFO.FrameRegister
must be set to the frame pointer register, and UNWIND_INFO.FrameOffset must be set to 15
(its maximum value). This code is followed by two UNWIND_CODEs that are combined to form
its 32-bit offset. This offset is then scaled by 16. This result is used as the FP register
offset from SP at the time the frame pointer is established.

src/inc/clrnt.h

@@ -817,6 +817,15 @@ typedef enum _UNWIND_OP_CODES {
     UWOP_SAVE_XMM128,
     UWOP_SAVE_XMM128_FAR,
     UWOP_PUSH_MACHFRAME
+
+#ifdef PLATFORM_UNIX
+    , UWOP_SET_FPREG_LARGE      // UWOP_SET_FPREG has a 240 byte SP->FP offset range; this has a 32-bit range scaled by 16,
+                                // but is not part of the standard Windows unwind codes. If used, UNWIND_INFO.FrameRegister
+                                // must be set to the frame pointer register, and UNWIND_INFO.FrameOffset must be set to 15
+                                // (its maximum value). This code is followed by two UNWIND_CODEs that are combined to form
+                                // a 32-bit offset. This offset is then scaled by 16. This result is used as the FP register
+                                // offset from SP at the time the frame pointer is established.
+#endif // PLATFORM_UNIX
 } UNWIND_OP_CODES, *PUNWIND_OP_CODES;
 
 static const UCHAR UnwindOpExtraSlotTable[] = {
@@ -831,6 +840,10 @@ static const UCHAR UnwindOpExtraSlotTable[] = {
     1,          // UWOP_SAVE_XMM128
     2,          // UWOP_SAVE_XMM128_FAR
     0           // UWOP_PUSH_MACHFRAME
+
+#ifdef PLATFORM_UNIX
+    , 2         // UWOP_SET_FPREG_LARGE
+#endif // PLATFORM_UNIX
 };
 
 //

src/jit/codegenxarch.cpp

@@ -7671,6 +7671,16 @@ int CodeGenInterface::genSPtoFPdelta()
 {
     int delta;
 
+#ifdef PLATFORM_UNIX
+
+    // We require frame chaining on Unix to support native tool unwinding (such as
+    // unwinding by the native debugger). We have a CLR-only extension to the
+    // unwind codes (UWOP_SET_FPREG_LARGE) to support SP->FP offsets larger than 240.
+    // If Unix ever supports EnC, the RSP == RBP assumption will have to be reevaluated.
+    delta = genTotalFrameSize();
+
+#else // !PLATFORM_UNIX
+
     // As per Amd64 ABI, RBP offset from initial RSP can be between 0 and 240 if
     // RBP needs to be reported in unwind codes.  This case would arise for methods
     // with localloc.
@@ -7695,6 +7705,8 @@ int CodeGenInterface::genSPtoFPdelta()
         delta = genTotalFrameSize();
     }
 
+#endif // !PLATFORM_UNIX
+
     return delta;
 }
 

src/jit/emitxarch.cpp

@@ -1676,12 +1676,14 @@ UNATIVE_OFFSET      emitter::emitInsSizeSV(size_t code, int var, int dsp)
 
             if  (EBPbased)
             {
-#ifdef _TARGET_AMD64_
+#if defined(_TARGET_AMD64_) && !defined(PLATFORM_UNIX)
                 // If localloc is not used, then ebp chaining is done and hence
                 // offset of locals will be at negative offsets, Otherwise offsets
                 // will be positive.  In future, when RBP gets positioned in the
                 // middle of the frame so as to optimize instruction encoding size,
                 // the below asserts needs to be modified appropriately.
+                // However, for Unix platforms, we always do frame pointer chaining,
+                // so offsets from the frame pointer will always be negative.
                 if (emitComp->compLocallocUsed || emitComp->opts.compDbgEnC)
                 {
                     noway_assert((int)offs >= 0);

src/jit/unwindamd64.cpp

@@ -380,17 +380,41 @@ void Compiler::unwindSetFrameRegWindows(regNumber reg, unsigned offset)
 
     assert(func->unwindHeader.Version == 1); // Can't call this before unwindBegProlog
     assert(func->unwindHeader.CountOfUnwindCodes == 0); // Can't call this after unwindReserve
-    assert(func->unwindCodeSlot > sizeof(UNWIND_CODE));
-    UNWIND_CODE * code = (UNWIND_CODE*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(UNWIND_CODE)];
     unsigned int cbProlog = unwindGetCurrentOffset(func);
     noway_assert((BYTE)cbProlog == cbProlog);
-    code->CodeOffset = (BYTE)cbProlog;
-    code->UnwindOp = UWOP_SET_FPREG;
-    code->OpInfo = 0;
+
     func->unwindHeader.FrameRegister = (BYTE)reg;
-    assert(offset <= 240);
-    assert(offset % 16 == 0);
-    func->unwindHeader.FrameOffset = offset / 16;
+
+#ifdef PLATFORM_UNIX
+    if (offset > 240)
+    {
+        // On Unix only, we have a CLR-only extension to the AMD64 unwind codes: UWOP_SET_FPREG_LARGE.
+        // It has a 32-bit offset (scaled). You must set UNWIND_INFO.FrameOffset to 15. The 32-bit
+        // offset follows in 2 UNWIND_CODE fields.
+
+        assert(func->unwindCodeSlot > (sizeof(UNWIND_CODE) + sizeof(ULONG)));
+        ULONG* codedSize = (ULONG*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(ULONG)];
+        assert(offset % 16 == 0);
+        *codedSize = offset / 16;
+
+        UNWIND_CODE* code = (UNWIND_CODE*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(UNWIND_CODE)];
+        code->CodeOffset = (BYTE)cbProlog;
+        code->OpInfo = 0;
+        code->UnwindOp = UWOP_SET_FPREG_LARGE;
+        func->unwindHeader.FrameOffset = 15;
+    }
+    else
+#endif // PLATFORM_UNIX
+    {
+        assert(func->unwindCodeSlot > sizeof(UNWIND_CODE));
+        UNWIND_CODE* code = (UNWIND_CODE*)&func->unwindCodes[func->unwindCodeSlot -= sizeof(UNWIND_CODE)];
+        code->CodeOffset = (BYTE)cbProlog;
+        code->OpInfo = 0;
+        code->UnwindOp = UWOP_SET_FPREG;
+        assert(offset <= 240);
+        assert(offset % 16 == 0);
+        func->unwindHeader.FrameOffset = offset / 16;
+    }
 }
 
 #ifdef UNIX_AMD64_ABI

src/unwinder/amd64/unwinder_amd64.cpp

@@ -799,6 +799,25 @@ Return Value:
                     ContextRecord->Rsp = IntegerRegister[UnwindInfo->FrameRegister];
                     ContextRecord->Rsp -= UnwindInfo->FrameOffset * 16;
                     break;
+
+#ifdef PLATFORM_UNIX
+                    //
+                    // Establish the the frame pointer register using a large size displacement.
+                    // UNWIND_INFO.FrameOffset must be 15 (the maximum value, corresponding to a scaled
+                    // offset of 15 * 16 == 240). The next two codes contain a 32-bit offset, which
+                    // is also scaled by 16, since the stack must remain 16-bit aligned.
+                    //
+
+                case UWOP_SET_FPREG_LARGE:
+                    UNWINDER_ASSERT(UnwindInfo->FrameOffset == 15);
+                    Index += 2;
+                    FrameOffset = UnwindInfo->UnwindCode[Index - 1].FrameOffset;
+                    FrameOffset += UnwindInfo->UnwindCode[Index].FrameOffset << 16;
+                    FrameOffset *= 16;
+                    ContextRecord->Rsp = IntegerRegister[UnwindInfo->FrameRegister];
+                    ContextRecord->Rsp -= FrameOffset * 16;
+                    break;
+#endif // PLATFORM_UNIX
 
                     //
                     // Save nonvolatile integer register on the stack using a
@@ -1055,6 +1074,7 @@ Routine Description:
     ULONG EpilogueSize;
     PEXCEPTION_ROUTINE FoundHandler;
     ULONG FrameRegister;
+    ULONG FrameOffset;
     ULONG Index;
     BOOL InEpilogue;
     PULONG64 IntegerAddress;
@@ -1115,23 +1135,53 @@ Routine Description:
     } else if ((PrologOffset >= UnwindInfo->SizeOfProlog) ||
                ((UnwindInfo->Flags & UNW_FLAG_CHAININFO) != 0)) {
 
+        FrameOffset = UnwindInfo->FrameOffset;
+
+#ifdef PLATFORM_UNIX
+        // If UnwindInfo->FrameOffset == 15 (the maximum value), then we assume there might be a UWOP_SET_FPREG_LARGE,
+        // so we need to look for it. If we don't find UWOP_SET_FPREG_LARGE, then just use (scaled) FrameOffset of 240.
+        if (FrameOffset == 15) {
+            Index = 0;
+            while (Index < UnwindInfo->CountOfUnwindCodes) {
+                UnwindOp = UnwindInfo->UnwindCode[Index];
+                if (UnwindOp.UnwindOp == UWOP_SET_FPREG_LARGE) {
+                    FrameOffset = UnwindInfo->UnwindCode[Index + 1].FrameOffset;
+                    FrameOffset += UnwindInfo->UnwindCode[Index + 2].FrameOffset << 16;
+                    break;
+                }
+
+                Index += UnwindOpSlots(UnwindOp);
+            }
+        }
+#endif // PLATFORM_UNIX
+
         *EstablisherFrame = (&ContextRecord->Rax)[UnwindInfo->FrameRegister];
-        *EstablisherFrame -= UnwindInfo->FrameOffset * 16;
+        *EstablisherFrame -= FrameOffset * 16;
 
     } else {
+        FrameOffset = 0;
         Index = 0;
         while (Index < UnwindInfo->CountOfUnwindCodes) {
             UnwindOp = UnwindInfo->UnwindCode[Index];
             if (UnwindOp.UnwindOp == UWOP_SET_FPREG) {
+                FrameOffset = UnwindInfo->FrameOffset;
+                break;
+            }
+#ifdef PLATFORM_UNIX
+            else if (UnwindOp.UnwindOp == UWOP_SET_FPREG_LARGE) {
+                UNWINDER_ASSERT(UnwindInfo->FrameOffset == 15);
+                FrameOffset = UnwindInfo->UnwindCode[Index + 1].FrameOffset;
+                FrameOffset += UnwindInfo->UnwindCode[Index + 2].FrameOffset << 16;
                 break;
             }
+#endif // PLATFORM_UNIX
 
             Index += UnwindOpSlots(UnwindOp);
         }
 
         if (PrologOffset >= UnwindInfo->UnwindCode[Index].CodeOffset) {
             *EstablisherFrame = (&ContextRecord->Rax)[UnwindInfo->FrameRegister];
-            *EstablisherFrame -= UnwindInfo->FrameOffset * 16;
+            *EstablisherFrame -= FrameOffset * 16;
 
         } else {
             *EstablisherFrame = ContextRecord->Rsp;