[Loader] Guard frame-pointer spill slots that live in the guest red zone

Host signal frames are pushed onto the guest stack, so GuestRedZonePatcher
protects functions that keep live data in the 128-byte red zone. It only
recognized RSP-relative accesses. Clang spills just as often through the frame
pointer: after 'push rbp; mov rbp, rsp' and N callee-saved pushes, RSP sits at
RBP-N and every [rbp-d] with d > N is below RSP. A fault taken there (lazy
commit, write tracking) overwrote those slots, which showed up as garbage
pointers on random threads in Poppy Playtime's level load.

Derive RBP-to-RSP from the standard prologue (leading pushes and the first
'sub rsp, imm') and treat [rbp-d] in (delta, delta + 128] as red-zone use. An
under-estimated delta only widens the guarded window. In Poppy's eboot this
protects 787 functions instead of 485.
This commit is contained in:
kuba 2026-09-27 06:57:47 +02:00
parent 4d65ff6d51
commit abe224732f
2 changed files with 97 additions and 1 deletions

View File

@ -207,7 +207,10 @@ internal static class GuestRedZonePatcher
functionCount++;
instructionCount += decoded.Count;
var usesRedZone = protectRedZone && decoded.Any(static entry => UsesRedZone(entry.Instruction));
var frameDelta = protectRedZone ? ComputeFramePointerDelta(decoded) : -1;
var usesRedZone = protectRedZone &&
(decoded.Any(static entry => UsesRedZone(entry.Instruction)) ||
decoded.Any(entry => UsesFramePointerRedZone(entry.Instruction, frameDelta)));
if (!usesRedZone && !splitVectorStores && !rewriteSha)
{
@ -575,6 +578,84 @@ internal static class GuestRedZonePatcher
return displacement < 0 && displacement >= -GuestRedZoneBytes;
}
// Clang spills into the red zone through the frame pointer as often as through
// RSP: with a standard 'push rbp; mov rbp, rsp' frame plus N callee-saved
// pushes, RSP sits at RBP-N, so every [rbp-d] with d > N is below RSP and a
// host exception frame would overwrite it. Matching only RSP-relative use left
// those functions unprotected.
internal static bool UsesFramePointerRedZone(in Instruction instruction, int frameDelta)
{
if (frameDelta < 0 || !HasMemoryOperand(instruction) || instruction.MemoryBase != Register.RBP)
{
return false;
}
var displacement = unchecked((long)instruction.MemoryDisplacement64);
return displacement < -frameDelta && displacement >= -(frameDelta + (long)GuestRedZoneBytes);
}
// Distance from the frame pointer down to RSP for a standard prologue, or -1
// when the function does not establish one. Only the leading pushes and the
// first 'sub rsp, imm' count, which under-estimates frames that grow later:
// under-estimating only widens the guarded window, so it stays conservative.
private static int ComputeFramePointerDelta(List<DecodedInstruction> decoded)
{
if (decoded.Count < 2)
{
return -1;
}
var push = decoded[0].Instruction;
if (push.Mnemonic != Mnemonic.Push ||
push.OpCount != 1 ||
push.GetOpKind(0) != OpKind.Register ||
push.GetOpRegister(0) != Register.RBP)
{
return -1;
}
var move = decoded[1].Instruction;
if (move.Mnemonic != Mnemonic.Mov ||
move.OpCount != 2 ||
move.GetOpKind(0) != OpKind.Register ||
move.GetOpRegister(0) != Register.RBP ||
move.GetOpKind(1) != OpKind.Register ||
move.GetOpRegister(1) != Register.RSP)
{
return -1;
}
var delta = 0;
for (var index = 2; index < decoded.Count; index++)
{
var instruction = decoded[index].Instruction;
if (instruction.Mnemonic == Mnemonic.Push &&
instruction.OpCount == 1 &&
instruction.GetOpKind(0) == OpKind.Register)
{
delta += 8;
continue;
}
if (instruction.Mnemonic == Mnemonic.Sub &&
instruction.OpCount == 2 &&
instruction.GetOpKind(0) == OpKind.Register &&
instruction.GetOpRegister(0) == Register.RSP &&
instruction.GetOpKind(1) is OpKind.Immediate8 or OpKind.Immediate8to64 or OpKind.Immediate32to64 or OpKind.Immediate32)
{
var immediate = (long)instruction.GetImmediate(1);
if (immediate > 0 && immediate < int.MaxValue - delta)
{
delta += (int)immediate;
}
}
break;
}
return delta;
}
private static bool UsesStackMemory(in Instruction instruction)
{
if (!HasMemoryOperand(instruction))

View File

@ -31,6 +31,21 @@ public sealed class GuestRedZonePatcherTests
Assert.Equal(expected, GuestRedZonePatcher.UsesRedZone(instruction));
}
// After 'push rbp; mov rbp, rsp' and five more pushes, RSP = RBP - 0x28: [rbp-0x30] is below
// RSP inside the red zone, [rbp-0x28] is the last pushed register, [rbp-0xB0] is below the zone.
[Theory]
[InlineData(new byte[] { 0x48, 0x89, 0x45, 0xD0 }, 0x28, true)]
[InlineData(new byte[] { 0x48, 0x89, 0x45, 0xD8 }, 0x28, false)]
[InlineData(new byte[] { 0x48, 0x89, 0x85, 0x50, 0xFF, 0xFF, 0xFF }, 0x28, false)]
[InlineData(new byte[] { 0x48, 0x89, 0x45, 0xD0 }, -1, false)]
[InlineData(new byte[] { 0x48, 0x89, 0x44, 0x24, 0xD0 }, 0x28, false)]
public void RecognizesFramePointerSpillsBelowTheStackPointer(byte[] instructionBytes, int frameDelta, bool expected)
{
var instruction = Decode(instructionBytes);
Assert.Equal(expected, GuestRedZonePatcher.UsesFramePointerRedZone(instruction, frameDelta));
}
[Fact]
public void RelocatesOnlyOrdinaryNonStackMemoryInstructions()
{