diff --git a/core/libs/prx/libSceAgc/CMakeLists.txt b/core/libs/prx/libSceAgc/CMakeLists.txt index 544eb0e9f..d3bb48f0b 100644 --- a/core/libs/prx/libSceAgc/CMakeLists.txt +++ b/core/libs/prx/libSceAgc/CMakeLists.txt @@ -69,6 +69,14 @@ add_test_executable(agc_command_tests EXCLUDE_FROM_ALL tests/Commands.cpp) target_include_directories(agc_command_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_command_tests PRIVATE libSceAgc libc) configure_windows_unwind(agc_command_tests) +if(BUILD_TESTING) + set_target_properties(agc_command_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_initialization COMMAND agc_command_tests defaults) + if(WIN32) + set_tests_properties(agc_initialization PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() configure_windows_unwind(libSceAgc) add_test_executable(agc_platform_tests EXCLUDE_FROM_ALL tests/Platform.cpp) diff --git a/core/libs/prx/libSceAgc/DcbState/src/Init.cpp b/core/libs/prx/libSceAgc/DcbState/src/Init.cpp index 737e0b62a..1b63094d1 100644 --- a/core/libs/prx/libSceAgc/DcbState/src/Init.cpp +++ b/core/libs/prx/libSceAgc/DcbState/src/Init.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgc/DcbState/include/Init.hpp" #include "prx/libSceAgc/Command/include/Packet.hpp" +#include "prx/libSceAgc/Command/include/RegisterDefaults.hpp" #include #include #include "SceTypes.hpp" @@ -8,11 +9,17 @@ extern "C" { -APS5_EXPORT("23LRUSvYu1M", sceAgcInit); -int APS5_VABI sceAgcInit(std::uint32_t* state, std::uint32_t version) { +APS5_EXPORT("23LRUSvYu1M", sceAgcUnknownInitState); +int APS5_VABI sceAgcUnknownInitState(std::uint32_t* state, std::uint32_t version) { Agc::Command::CheckAddress(reinterpret_cast(state), alignof(std::uint32_t), __func__); Agc::Command::Require(version < 14, __func__, "unsupported AGC initialization version"); return 0; } +int APS5_VABI sceAgcInit(std::uint32_t version) { + Agc::Command::GetRegisterDefaults(version, false, __func__); + Agc::Command::GetRegisterDefaults(version, true, __func__); + return 0; +} + } diff --git a/core/libs/prx/libSceAgc/tests/Commands.cpp b/core/libs/prx/libSceAgc/tests/Commands.cpp index 5b394c586..4e11a5bed 100644 --- a/core/libs/prx/libSceAgc/tests/Commands.cpp +++ b/core/libs/prx/libSceAgc/tests/Commands.cpp @@ -13,7 +13,8 @@ extern "C" std::uint32_t* APS5_VABI sceAgcDcbResetQueue(CommandBuffer* buf, std::uint32_t op, std::uint32_t state); extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetFlip(CommandBuffer* buf, std::uint32_t handle, std::int32_t index, std::uint32_t mode, std::int64_t argument); extern "C" int APS5_VABI sceAgcSuspendPoint(); -extern "C" int APS5_VABI sceAgcInit(std::uint32_t* state, std::uint32_t version); +extern "C" int APS5_VABI sceAgcInit(std::uint32_t version); +extern "C" int APS5_VABI sceAgcUnknownInitState(std::uint32_t* state, std::uint32_t version); extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexAuto(CommandBuffer* buf, std::uint32_t indexCount, std::uint64_t modifier); extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexIndirect(CommandBuffer* buf, std::uint32_t dataOffsetInBytes, std::uint64_t modifier); extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexIndirectMulti(CommandBuffer* buf, std::uint32_t dataOffsetInBytes, std::uint32_t countIndirect, std::uint32_t maxCountOrCount, const volatile void* countAddress, std::uint32_t strideInBytes, std::uint64_t modifier); @@ -336,9 +337,13 @@ void testMemory() { void testDefaults() { std::uint32_t state = 0x12345678; - check(sceAgcInit(&state, 8) == 0 && state == 0x12345678, "AGC initialization failed or modified caller state"); - expectFailure([] { sceAgcInit(nullptr, 8); }); - expectFailure([&] { sceAgcInit(&state, 14); }); + check(sceAgcUnknownInitState(&state, 8) == 0 && state == 0x12345678, "AGC initialization failed or modified caller state"); + check(sceAgcUnknownInitState(&state, 13) == 0 && state == 0x12345678, "AGC version 13 initialization changed caller state"); + expectFailure([] { sceAgcUnknownInitState(nullptr, 8); }); + expectFailure([&] { sceAgcUnknownInitState(&state, 14); }); + check(sceAgcInit(8) == 0, "AGC version initialization failed"); + expectFailure([] { sceAgcInit(14); }); + expectFailure([] { sceAgcInit(0xffffffffu); }); for (std::uint32_t version = 0; version < 14; ++version) { for (const bool internal : {false, true}) { auto* first = Agc::Command::GetRegisterDefaults(version, internal, __func__); @@ -351,8 +356,14 @@ void testDefaults() { } -int main() { +int main(int argc, char** argv) { try { + if (argc == 2 && std::strcmp(argv[1], "defaults") == 0) { + testDefaults(); + LibcRunShutdown_nid_postfix(); + std::puts("AGC initialization tests passed"); + return 0; + } testPackets(); testIndexedIndirectDraws(); testMarkers(); diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 90d971f7a..c3bb3b625 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -214,6 +214,7 @@ add_test_executable(agc_driver_tests EXCLUDE_FROM_ALL target_include_directories(agc_driver_tests PRIVATE ${LIBS_INCLUDE_DIR} + ${CMAKE_SOURCE_DIR}/core/shader/recompiler ) target_link_libraries(agc_driver_tests PRIVATE diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp index 6189cbfec..1f6012ccb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp @@ -38,6 +38,7 @@ struct ShaderSnapshot { using ShaderRegistry = std::map>; bool FailureMemo(); +std::shared_ptr ReadRawComputeShader(std::uint64_t address); std::uint64_t NullPixelProgramAddress(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index 25c05fb21..25ff62716 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -12,10 +12,13 @@ namespace AgcDriver::DriverDetail { void Driver::dispatch(QueueState& queue, std::span packet, const Submission& submission, std::uint64_t indirectArguments) { const auto address = (static_cast(readRegister(queue.shader, 0x20c)) << 8u) | (static_cast(readRegister(queue.shader, 0x20d) & 0xffu) << 40u); auto it = submission.shaders->upper_bound(address); - require(it != submission.shaders->begin(), "compute program does not belong to a registered shader"); - --it; - const auto& snapshot = *it->second; - require(address - snapshot.codeAddress < snapshot.code.size() * sizeof(std::uint32_t), "compute program is outside registered shader code"); + std::shared_ptr registeredShader; + if (it != submission.shaders->begin()) { + --it; + if (address - it->second->codeAddress < it->second->code.size() * sizeof(std::uint32_t)) registeredShader = it->second; + } + if (!registeredShader) registeredShader = ReadRawComputeShader(address); + const auto& snapshot = *registeredShader; require(snapshot.type == 0, "compute program refers to a non-compute shader"); const auto userCount = (readRegister(queue.shader, 0x213) >> 1u) & 0x1fu; std::vector userData; @@ -23,7 +26,8 @@ void Driver::dispatch(QueueState& queue, std::span packet, userData.push_back(readRegister(queue.shader, 0x240 + i)); } auto compute = Graphics::DecodeComputeStageInfo(queue.shader); - const std::array memory{{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}}; + std::vector memory{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}}; + if (!snapshot.header.empty()) memory.push_back({snapshot.headerAddress, snapshot.header}); static const bool unlockedDevice = std::getenv("APS5_NO_UNLOCKED_DEVICE") == nullptr; std::shared_ptr localDevice = unlockedDevice ? device.Load() : nullptr; @@ -174,7 +178,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, previous.key = key; } - if (!stampValidate()) mix(reinterpret_cast(it->second.get())); + if (!stampValidate()) mix(reinterpret_cast(registeredShader.get())); phaseTiming.Phase(PhaseKey); lookupDispatch(address, submission, key, noDispatchCache, traceCache, profile, memory, phaseTiming, phaseMs, compiledResult, keepVariant, captured, liveWords, dataHit, cached, validated, missedEntry, missedDiffering); if (cached) { @@ -226,7 +230,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, } recompileMs += phaseTiming.Elapsed(); phaseTiming.Phase(PhaseRecompile); - insertDispatch(address, key, noDispatchCache, profile, it->second, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming); + insertDispatch(address, key, noDispatchCache, profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming); } if (verifyDataHits() && dataHit) verifyDataHit(snapshot, codeOffset, localDevice->Serial(), request, memory, address, *keepVariant, liveWords, *compiledResult); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index abd485168..8ba86476a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -6,10 +6,78 @@ #include "Optimization/ResourceProgram.hpp" #include #include +#include #include +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" namespace AgcDriver::DriverDetail { +std::shared_ptr ReadRawComputeShader(std::uint64_t address) { + GuestMemory::CheckRange(reinterpret_cast(address), sizeof(std::uint32_t), 256); + static std::mutex cacheMutex; + static std::list> cache; + static std::size_t cacheBytes = 0; + std::shared_ptr cached; + { + std::lock_guard lock(cacheMutex); + const auto found = std::find_if(cache.begin(), cache.end(), [address](const auto& entry) { return entry->codeAddress == address; }); + if (found != cache.end()) cached = *found; + } + if (cached) { + const auto code = std::as_bytes(std::span(cached->code)); + GuestMemory::FlushGpuWrites(address, code.size()); + if (GuestMemory::CompareMapped(address, code) == GuestMemory::Compare::Equal) { + std::lock_guard lock(cacheMutex); + const auto found = std::find(cache.begin(), cache.end(), cached); + if (found != cache.end()) cache.splice(cache.begin(), cache, found); + return cached; + } + } + constexpr std::size_t limit = 1024 * 1024; + const auto ranges = GuestMemory::CommittedRanges(address, limit); + std::uint64_t end = address; + for (const auto& range : ranges) { + if (range.first != end) break; + end = range.second; + } + const auto available = static_cast(end - address) / sizeof(std::uint32_t); + ShaderSnapshot snapshot{address, 0, 0, {}, {}}; + while (snapshot.code.size() < available) { + const auto previous = snapshot.code.size(); + snapshot.code.resize(std::min(available, std::max(64, previous * 2))); + GuestMemory::Read(address + previous * sizeof(std::uint32_t), + std::as_writable_bytes(std::span(snapshot.code).subspan(previous)), alignof(std::uint32_t)); + try { + const auto decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(snapshot.code); + const auto& last = decoded.instructions.back(); + snapshot.code.resize(last.programCounter / sizeof(std::uint32_t) + last.wordCount); + auto result = std::make_shared(std::move(snapshot)); + std::lock_guard lock(cacheMutex); + const auto found = std::find_if(cache.begin(), cache.end(), [address](const auto& entry) { return entry->codeAddress == address; }); + if (found != cache.end()) { + if ((*found)->code == result->code) { + result = *found; + cache.splice(cache.begin(), cache, found); + return result; + } + cacheBytes -= (*found)->code.size() * sizeof(std::uint32_t); + cache.erase(found); + } + const auto bytes = result->code.size() * sizeof(std::uint32_t); + while (!cache.empty() && (cache.size() >= 64 || cacheBytes + bytes > 8 * 1024 * 1024)) { + cacheBytes -= cache.back()->code.size() * sizeof(std::uint32_t); + cache.pop_back(); + } + cache.push_front(result); + cacheBytes += bytes; + return result; + } catch (const std::out_of_range&) { + if (snapshot.code.size() == available) break; + } + } + throw std::runtime_error("AGC driver: raw compute program has no reachable end within mapped code or the size limit"); +} + bool FailureMemo() { static const bool memo = std::getenv("APS5_NO_FAILURE_MEMO") == nullptr; return memo; diff --git a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp index 28a8f081a..6888c50b6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/QueueState.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" #include "prx/libc/include/Shutdown.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "prx/libSceAgcDriver/Submit/include/Acb.hpp" @@ -7,6 +8,7 @@ #include "prx/libSceAgcDriver/Eq/include/Event.hpp" #include "prx/libkernel/Equeue/Equeue.hpp" #include +#include #include #include #include @@ -15,6 +17,9 @@ #include #include #include +#ifdef _WIN32 +#include +#endif extern "C" int APS5_VABI sceKernelCreateEqueue(KernelEqueue* eq, const char* name); extern "C" int APS5_VABI sceKernelDeleteEqueue(KernelEqueue eq); @@ -362,6 +367,68 @@ void testWorkerFailure() { int main() { try { + alignas(256) std::array rawCode{}; + rawCode.fill(0xbf800000); + rawCode[0] = 0xbe8003ff; + rawCode[1] = 0xbf810000; + rawCode[2] = 0xbf810000; + const auto rawAddress = reinterpret_cast(rawCode.data()); + const auto literal = AgcDriver::DriverDetail::ReadRawComputeShader(rawAddress); + check(literal->code.size() == 3 && literal->header.empty(), "raw compute stopped at an instruction literal"); + check(AgcDriver::DriverDetail::ReadRawComputeShader(rawAddress) == literal, "unchanged raw compute code lost its snapshot identity"); + rawCode[0] = 0xbf820002; + rawCode[1] = 0xbf810000; + rawCode[2] = 0xbf800000; + rawCode[3] = 0xbf810000; + const auto branched = AgcDriver::DriverDetail::ReadRawComputeShader(rawAddress); + check(branched->code.size() == 4 && literal->code[0] == 0xbe8003ff, "raw compute lost branch targets or modified an earlier snapshot"); + check(branched != literal, "changed raw compute code reused a stale snapshot"); + std::array, 8> concurrent; + std::vector readers; + for (auto& snapshot : concurrent) readers.emplace_back([&snapshot, rawAddress] { + snapshot = AgcDriver::DriverDetail::ReadRawComputeShader(rawAddress); + }); + for (auto& reader : readers) reader.join(); + for (const auto& snapshot : concurrent) check(snapshot == branched, "concurrent raw compute readers lost snapshot reuse"); + alignas(256) std::array, 65> programs{}; + for (auto& program : programs) program[0] = 0xbf810000; + const auto firstAddress = reinterpret_cast(programs.front().data()); + readers.clear(); + for (auto& snapshot : concurrent) readers.emplace_back([&snapshot, firstAddress] { + snapshot = AgcDriver::DriverDetail::ReadRawComputeShader(firstAddress); + }); + for (auto& reader : readers) reader.join(); + for (const auto& snapshot : concurrent) check(snapshot == concurrent.front(), "concurrent raw compute misses duplicated snapshots"); + const auto evicted = AgcDriver::DriverDetail::ReadRawComputeShader(firstAddress); + for (std::size_t i = 1; i < programs.size(); ++i) + AgcDriver::DriverDetail::ReadRawComputeShader(reinterpret_cast(programs[i].data())); + check(AgcDriver::DriverDetail::ReadRawComputeShader(firstAddress) != evicted && evicted->code[0] == 0xbf810000, + "raw compute cache eviction lost snapshot lifetime or exceeded its entry limit"); + check(!expectFailure([&] { AgcDriver::DriverDetail::ReadRawComputeShader(rawAddress + 4); }).empty(), "raw compute accepted a misaligned entry"); + check(!expectFailure([] { AgcDriver::DriverDetail::ReadRawComputeShader(0); }).empty(), "raw compute accepted an unmapped entry"); +#ifdef _WIN32 + auto* mapping = static_cast(VirtualAlloc(nullptr, 8192, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + check(mapping != nullptr, "cannot allocate raw compute boundary test"); + DWORD protection = 0; + check(VirtualProtect(mapping + 1024, 4096, PAGE_NOACCESS, &protection) != 0, "cannot protect raw compute boundary"); + auto* boundedCode = mapping + 1024 - 64; + std::fill_n(boundedCode, 64, 0xbf800000u); + const auto boundedAddress = reinterpret_cast(boundedCode); + const auto unterminated = expectFailure([&] { AgcDriver::DriverDetail::ReadRawComputeShader(boundedAddress); }); + boundedCode[63] = 0xbf810000; + const auto bounded = AgcDriver::DriverDetail::ReadRawComputeShader(boundedAddress); + check(VirtualProtect(mapping + 1024, 4096, PAGE_READWRITE, &protection) != 0, "cannot extend raw compute mapping"); + boundedCode[63] = 0xbf800000; + boundedCode[64] = 0xbf810000; + const auto extended = AgcDriver::DriverDetail::ReadRawComputeShader(boundedAddress); + check(extended != bounded && extended->code.size() == 65, "raw compute reused code before its end changed"); + check(VirtualProtect(mapping + 1024, 4096, PAGE_NOACCESS, &protection) != 0, "cannot revoke cached raw compute tail"); + check(!expectFailure([&] { AgcDriver::DriverDetail::ReadRawComputeShader(boundedAddress); }).empty(), "raw compute reused an inaccessible cached tail"); + check(VirtualProtect(mapping, 4096, PAGE_NOACCESS, &protection) != 0, "cannot revoke cached raw compute code"); + check(!expectFailure([&] { AgcDriver::DriverDetail::ReadRawComputeShader(boundedAddress); }).empty(), "raw compute reused inaccessible cached code"); + check(VirtualFree(mapping, 0, MEM_RELEASE) != 0, "cannot release raw compute boundary test"); + check(!unterminated.empty() && bounded->code.size() == 64, "raw compute crossed inaccessible memory or missed its last instruction"); +#endif testEvents(); testValidation(); testClearState();