From abe224732f23ec543f05ed9e00491da0cd6c555d Mon Sep 17 00:00:00 2001 From: kuba Date: Sun, 27 Sep 2026 06:57:47 +0200 Subject: [PATCH] [Loader] Guard frame-pointer spill slots that live in the guest red zone Host signal frames are pushed onto the guest stack, so GuestRedZonePatcher protects functions that keep live data in the 128-byte red zone. It only recognized RSP-relative accesses. Clang spills just as often through the frame pointer: after 'push rbp; mov rbp, rsp' and N callee-saved pushes, RSP sits at RBP-N and every [rbp-d] with d > N is below RSP. A fault taken there (lazy commit, write tracking) overwrote those slots, which showed up as garbage pointers on random threads in Poppy Playtime's level load. Derive RBP-to-RSP from the standard prologue (leading pushes and the first 'sub rsp, imm') and treat [rbp-d] in (delta, delta + 128] as red-zone use. An under-estimated delta only widens the guarded window. In Poppy's eboot this protects 787 functions instead of 485. --- .../Loader/GuestRedZonePatcher.cs | 83 ++++++++++++++++++- .../Loader/GuestRedZonePatcherTests.cs | 15 ++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs b/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs index d541b3c..f73c90b 100644 --- a/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs +++ b/src/SharpEmu.Core/Loader/GuestRedZonePatcher.cs @@ -207,7 +207,10 @@ internal static class GuestRedZonePatcher functionCount++; instructionCount += decoded.Count; - var usesRedZone = protectRedZone && decoded.Any(static entry => UsesRedZone(entry.Instruction)); + var frameDelta = protectRedZone ? ComputeFramePointerDelta(decoded) : -1; + var usesRedZone = protectRedZone && + (decoded.Any(static entry => UsesRedZone(entry.Instruction)) || + decoded.Any(entry => UsesFramePointerRedZone(entry.Instruction, frameDelta))); if (!usesRedZone && !splitVectorStores && !rewriteSha) { @@ -575,6 +578,84 @@ internal static class GuestRedZonePatcher return displacement < 0 && displacement >= -GuestRedZoneBytes; } + // Clang spills into the red zone through the frame pointer as often as through + // RSP: with a standard 'push rbp; mov rbp, rsp' frame plus N callee-saved + // pushes, RSP sits at RBP-N, so every [rbp-d] with d > N is below RSP and a + // host exception frame would overwrite it. Matching only RSP-relative use left + // those functions unprotected. + internal static bool UsesFramePointerRedZone(in Instruction instruction, int frameDelta) + { + if (frameDelta < 0 || !HasMemoryOperand(instruction) || instruction.MemoryBase != Register.RBP) + { + return false; + } + + var displacement = unchecked((long)instruction.MemoryDisplacement64); + return displacement < -frameDelta && displacement >= -(frameDelta + (long)GuestRedZoneBytes); + } + + // Distance from the frame pointer down to RSP for a standard prologue, or -1 + // when the function does not establish one. Only the leading pushes and the + // first 'sub rsp, imm' count, which under-estimates frames that grow later: + // under-estimating only widens the guarded window, so it stays conservative. + private static int ComputeFramePointerDelta(List decoded) + { + if (decoded.Count < 2) + { + return -1; + } + + var push = decoded[0].Instruction; + if (push.Mnemonic != Mnemonic.Push || + push.OpCount != 1 || + push.GetOpKind(0) != OpKind.Register || + push.GetOpRegister(0) != Register.RBP) + { + return -1; + } + + var move = decoded[1].Instruction; + if (move.Mnemonic != Mnemonic.Mov || + move.OpCount != 2 || + move.GetOpKind(0) != OpKind.Register || + move.GetOpRegister(0) != Register.RBP || + move.GetOpKind(1) != OpKind.Register || + move.GetOpRegister(1) != Register.RSP) + { + return -1; + } + + var delta = 0; + for (var index = 2; index < decoded.Count; index++) + { + var instruction = decoded[index].Instruction; + if (instruction.Mnemonic == Mnemonic.Push && + instruction.OpCount == 1 && + instruction.GetOpKind(0) == OpKind.Register) + { + delta += 8; + continue; + } + + if (instruction.Mnemonic == Mnemonic.Sub && + instruction.OpCount == 2 && + instruction.GetOpKind(0) == OpKind.Register && + instruction.GetOpRegister(0) == Register.RSP && + instruction.GetOpKind(1) is OpKind.Immediate8 or OpKind.Immediate8to64 or OpKind.Immediate32to64 or OpKind.Immediate32) + { + var immediate = (long)instruction.GetImmediate(1); + if (immediate > 0 && immediate < int.MaxValue - delta) + { + delta += (int)immediate; + } + } + + break; + } + + return delta; + } + private static bool UsesStackMemory(in Instruction instruction) { if (!HasMemoryOperand(instruction)) diff --git a/tests/SharpEmu.Libs.Tests/Loader/GuestRedZonePatcherTests.cs b/tests/SharpEmu.Libs.Tests/Loader/GuestRedZonePatcherTests.cs index 1f6e40e..21b8bd1 100644 --- a/tests/SharpEmu.Libs.Tests/Loader/GuestRedZonePatcherTests.cs +++ b/tests/SharpEmu.Libs.Tests/Loader/GuestRedZonePatcherTests.cs @@ -31,6 +31,21 @@ public sealed class GuestRedZonePatcherTests Assert.Equal(expected, GuestRedZonePatcher.UsesRedZone(instruction)); } + // After 'push rbp; mov rbp, rsp' and five more pushes, RSP = RBP - 0x28: [rbp-0x30] is below + // RSP inside the red zone, [rbp-0x28] is the last pushed register, [rbp-0xB0] is below the zone. + [Theory] + [InlineData(new byte[] { 0x48, 0x89, 0x45, 0xD0 }, 0x28, true)] + [InlineData(new byte[] { 0x48, 0x89, 0x45, 0xD8 }, 0x28, false)] + [InlineData(new byte[] { 0x48, 0x89, 0x85, 0x50, 0xFF, 0xFF, 0xFF }, 0x28, false)] + [InlineData(new byte[] { 0x48, 0x89, 0x45, 0xD0 }, -1, false)] + [InlineData(new byte[] { 0x48, 0x89, 0x44, 0x24, 0xD0 }, 0x28, false)] + public void RecognizesFramePointerSpillsBelowTheStackPointer(byte[] instructionBytes, int frameDelta, bool expected) + { + var instruction = Decode(instructionBytes); + + Assert.Equal(expected, GuestRedZonePatcher.UsesFramePointerRedZone(instruction, frameDelta)); + } + [Fact] public void RelocatesOnlyOrdinaryNonStackMemoryInstructions() {