[CPU] Run libc memcpy/memmove as a native guest-side stub
Guest memcpy went through the managed gateway on every call: two write tracking notifications, the memory reader lock and two region lookups, about 0.8 us per call under Rosetta even for a copy of a few bytes. Dreaming Sarah makes roughly 45k of those calls a second. Guest memory is identity-mapped, and every fault a copy can take -- write tracking, lazy commit -- is resolved from the fault address alone, so the copy can run as a forward rep movsb in a fast-path stub. A destination that sits inside the source, where a forward copy would read bytes it has already written, and the write-watch diagnostic both keep the managed export.
This commit is contained in:
kuba
parent
6db42fe981
commit
5fb1cbacdc
|
|
@ -52,6 +52,7 @@ These variables change how the emulator operates.
|
|||
| --- | --- | --- | --- |
|
||||
| `SHARPEMU_AGC_VERSION13_DEFAULTS` | text (`legacy`, `generic` or `0`) | Selects the register defaults for AGC version 13 requests. The default uses the version 11 compact tables. The values `legacy`, `generic` and `0` use the older generic layout. | `AgcExports.Init.cs` |
|
||||
| `SHARPEMU_DISABLE_LLE_LIBC` | `1` | Stops the use of the guest libc module for libc exports. The emulator then uses its HLE handlers for these exports. | `DirectExecutionBackend.cs` |
|
||||
| `SHARPEMU_HLE_FAST_MEMCPY` | `0` | Set to `0` to stop the native fast-path stub for the guest libc `memcpy` and `memmove`, so those imports use the managed HLE gateway again. | `DirectExecutionBackend.FastPath.cs` |
|
||||
| `SHARPEMU_HLE_FAST_PATH` | `0` | Set to `0` to stop the native fast-path stubs for `pthread_self` and `pthread_getspecific`, so those imports use the managed HLE gateway again. The stubs are on by default on macOS x64 and are off on any run that traces imports or pthreads. | `DirectExecutionBackend.FastPath.cs` |
|
||||
| `SHARPEMU_HOLD_FIRST_FLIP_MS` | number | Stops the thread that submits the flip for this number of milliseconds on one guest flip. The range is 0 to 60000 and the default is 0 (off). `SHARPEMU_HOLD_FLIP_NUMBER` selects the flip. | `VideoOutExports.cs` |
|
||||
| `SHARPEMU_HOLD_FLIP_NUMBER` | number | Selects the guest flip that `SHARPEMU_HOLD_FIRST_FLIP_MS` holds. The minimum is 1 and the default is 1 (the first flip). | `VideoOutExports.cs` |
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ public sealed unsafe partial class DirectExecutionBackend
|
|||
None,
|
||||
PthreadSelf,
|
||||
PthreadGetspecific,
|
||||
MemoryCopy,
|
||||
}
|
||||
|
||||
// scePthreadSelf / pthread_self.
|
||||
|
|
@ -32,6 +33,16 @@ public sealed unsafe partial class DirectExecutionBackend
|
|||
private const string PthreadGetspecificNid = "eoht7mQOCmo";
|
||||
private const string PosixPthreadGetspecificNid = "0-KXaS70xy4";
|
||||
|
||||
// memcpy / memmove.
|
||||
private const string MemcpyNid = "Q3VBxCXhUHs";
|
||||
private const string MemmoveNid = "+P6FRGH4LfA";
|
||||
|
||||
// Guest memory is identity-mapped and every fault the copy can take (write tracking, lazy
|
||||
// commit) is resolved from the fault address alone, so the copy may run as guest-side code.
|
||||
// SHARPEMU_HLE_FAST_MEMCPY=0 keeps it on the managed export.
|
||||
private static readonly bool GuestMemoryCopyStubEnabled =
|
||||
!string.Equals(Environment.GetEnvironmentVariable("SHARPEMU_HLE_FAST_MEMCPY"), "0", StringComparison.Ordinal);
|
||||
|
||||
private uint _fastPathBlockTlsIndex = uint.MaxValue;
|
||||
private bool _guestFastPathEnabled;
|
||||
private int _guestFastPathStubCount;
|
||||
|
|
@ -88,6 +99,7 @@ public sealed unsafe partial class DirectExecutionBackend
|
|||
{
|
||||
PthreadSelfNid or PosixPthreadSelfNid => GuestFastPathStub.PthreadSelf,
|
||||
PthreadGetspecificNid or PosixPthreadGetspecificNid => GuestFastPathStub.PthreadGetspecific,
|
||||
MemcpyNid or MemmoveNid => GuestFastPathStub.MemoryCopy,
|
||||
_ => GuestFastPathStub.None,
|
||||
};
|
||||
|
||||
|
|
@ -105,7 +117,9 @@ public sealed unsafe partial class DirectExecutionBackend
|
|||
}
|
||||
|
||||
var kind = ClassifyGuestFastPathStub(nid);
|
||||
if (kind == GuestFastPathStub.None)
|
||||
if (kind == GuestFastPathStub.None ||
|
||||
// The write watch inspects every managed guest write, which a native copy never reaches.
|
||||
(kind == GuestFastPathStub.MemoryCopy && (!GuestMemoryCopyStubEnabled || GuestWriteWatch.Armed)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
|
@ -154,6 +168,11 @@ public sealed unsafe partial class DirectExecutionBackend
|
|||
uint blockTlsIndex,
|
||||
nint fallbackTrampoline)
|
||||
{
|
||||
if (kind == GuestFastPathStub.MemoryCopy)
|
||||
{
|
||||
return EmitGuestMemoryCopyStub(fallbackTrampoline);
|
||||
}
|
||||
|
||||
var code = new List<byte>(48);
|
||||
var slowPathFixups = new List<int>();
|
||||
|
||||
|
|
@ -241,4 +260,27 @@ public sealed unsafe partial class DirectExecutionBackend
|
|||
|
||||
return code;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// memcpy/memmove(dst, src, n) as a forward <c>rep movsb</c>, returning dst. The libc
|
||||
/// export copies with memmove semantics, so a destination that starts inside the source
|
||||
/// (where a forward copy would read bytes it already overwrote) takes the managed path.
|
||||
/// Only RAX, RCX, RSI, RDI and the flags change - all caller-saved - and the direction
|
||||
/// flag is clear on entry per the SysV ABI.
|
||||
/// </summary>
|
||||
internal static List<byte> EmitGuestMemoryCopyStub(nint fallbackTrampoline)
|
||||
{
|
||||
var code = new List<byte>(40);
|
||||
// mov rcx, rdi / sub rcx, rsi / cmp rcx, rdx / jb slow: dst - src < n (unsigned) is
|
||||
// exactly a destination in (src, src + n) or equal to src with a non-empty copy.
|
||||
code.AddRange([0x48, 0x89, 0xF9, 0x48, 0x29, 0xF1, 0x48, 0x39, 0xD1, 0x72, 0x00]);
|
||||
var slowPathFixup = code.Count - 1;
|
||||
// mov rax, rdi / mov rcx, rdx / rep movsb / ret
|
||||
code.AddRange([0x48, 0x89, 0xF8, 0x48, 0x89, 0xD1, 0xF3, 0xA4, 0xC3]);
|
||||
code[slowPathFixup] = checked((byte)(code.Count - (slowPathFixup + 1)));
|
||||
// jmp qword [rip+0] with the absolute trampoline address after it.
|
||||
code.AddRange([0xFF, 0x25, 0x00, 0x00, 0x00, 0x00]);
|
||||
code.AddRange(BitConverter.GetBytes((long)fallbackTrampoline));
|
||||
return code;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -235,6 +235,103 @@ public sealed unsafe class GuestFastPathStubTests
|
|||
return elapsed * 1_000_000_000.0 / Stopwatch.Frequency / iterations;
|
||||
}
|
||||
|
||||
private const ulong MemoryCopySlowPathMarker = 0x5105_1A7B_AC4E_0001UL;
|
||||
|
||||
[Fact]
|
||||
public void MemoryCopyStub_CopiesForwardAndReturnsTheDestination()
|
||||
{
|
||||
if (!CanRunStubs())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var stub = new MemoryCopyStub();
|
||||
var source = Enumerable.Range(1, 300).Select(value => (byte)value).ToArray();
|
||||
var destination = new byte[300];
|
||||
fixed (byte* sourcePointer = source)
|
||||
fixed (byte* destinationPointer = destination)
|
||||
{
|
||||
Assert.Equal((ulong)destinationPointer, stub.Call(destinationPointer, sourcePointer, 300));
|
||||
Assert.Equal((ulong)destinationPointer, stub.Call(destinationPointer + 7, sourcePointer, 0) - 7);
|
||||
}
|
||||
|
||||
Assert.Equal(source, destination);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MemoryCopyStub_HandlesADestinationBelowAnOverlappingSource()
|
||||
{
|
||||
if (!CanRunStubs())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var stub = new MemoryCopyStub();
|
||||
var buffer = Enumerable.Range(0, 64).Select(value => (byte)value).ToArray();
|
||||
fixed (byte* pointer = buffer)
|
||||
{
|
||||
Assert.Equal((ulong)pointer, stub.Call(pointer, pointer + 8, 40));
|
||||
}
|
||||
|
||||
Assert.Equal(Enumerable.Range(8, 40).Select(value => (byte)value), buffer.Take(40));
|
||||
Assert.Equal(Enumerable.Range(40, 24).Select(value => (byte)value), buffer.Skip(40));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MemoryCopyStub_LeavesADestinationInsideTheSourceToTheManagedExport()
|
||||
{
|
||||
if (!CanRunStubs())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var stub = new MemoryCopyStub();
|
||||
var buffer = Enumerable.Range(0, 64).Select(value => (byte)value).ToArray();
|
||||
var original = buffer.ToArray();
|
||||
fixed (byte* pointer = buffer)
|
||||
{
|
||||
Assert.Equal(MemoryCopySlowPathMarker, stub.Call(pointer + 8, pointer, 40));
|
||||
Assert.Equal(MemoryCopySlowPathMarker, stub.Call(pointer, pointer, 1));
|
||||
}
|
||||
|
||||
Assert.Equal(original, buffer);
|
||||
}
|
||||
|
||||
/// <summary>The memcpy stub called with the platform (SysV) convention the guest uses.</summary>
|
||||
private sealed class MemoryCopyStub : IDisposable
|
||||
{
|
||||
private readonly byte* _page;
|
||||
|
||||
public MemoryCopyStub()
|
||||
{
|
||||
_page = (byte*)HostMemory.Alloc(
|
||||
null,
|
||||
4096,
|
||||
HostMemory.MEM_COMMIT | HostMemory.MEM_RESERVE,
|
||||
HostMemory.PAGE_EXECUTE_READWRITE);
|
||||
Assert.True(_page != null);
|
||||
|
||||
// Fallback at +0x100: mov rax, marker / ret.
|
||||
_page[0x100] = 0x48;
|
||||
_page[0x101] = 0xB8;
|
||||
*(ulong*)(_page + 0x102) = MemoryCopySlowPathMarker;
|
||||
_page[0x10A] = 0xC3;
|
||||
|
||||
var stub = DirectExecutionBackend.EmitGuestMemoryCopyStub((nint)(_page + 0x100));
|
||||
for (var i = 0; i < stub.Count; i++)
|
||||
{
|
||||
_page[i] = stub[i];
|
||||
}
|
||||
|
||||
Assert.True(HostMemory.Protect(_page, 4096, HostMemory.PAGE_EXECUTE_READ, out _));
|
||||
}
|
||||
|
||||
public ulong Call(byte* destination, byte* source, nuint count) =>
|
||||
((delegate* unmanaged<byte*, byte*, nuint, ulong>)_page)(destination, source, count);
|
||||
|
||||
public void Dispose() => _ = HostMemory.Free(_page, 0, HostMemory.MEM_RELEASE);
|
||||
}
|
||||
|
||||
private static bool CanRunStubs() =>
|
||||
OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.X64;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue