shifathosn
/
AnyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into perf/shader-coherent-alias

This commit is contained in:
Karan Vora 2026-10-11 04:57:31 -04:00
parent c3742a3995 1c75f4d228
commit 2069fd1262
535 changed files with 29311 additions and 1904 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/labels.yml vendored
View File

@ -24,7 +24,7 @@ jobs:
force_label() { gh label create "$1" --color "$2" --description "$3" --force; }
force_label 'needs rebase' b60205 'Conflicts with main, rebase needed'
force_label 'intake paused' d93f0b 'Closed while PR intake is limited to authors with at least 20 PRs merged into main'
force_label 'intake paused' d93f0b 'Closed while PR intake is limited to authors with at least 10 PRs merged into main'
force_label ci 5319e7 '.github'
force_label agc 1d76db 'libSceAgc*'
force_label shader 1d76db 'core/shader'

4
.github/workflows/pr-intake.yml vendored
View File

@ -27,7 +27,7 @@ jobs:
if (pr.state !== 'open' || pr.base.ref !== 'main') {
return;
}
const minimumMergedPRs = 20;
const minimumMergedPRs = 10;
const { data: mergedPRs } = await github.rest.search.issuesAndPullRequests({
q: `repo:${owner}/${repo} is:pr is:merged base:main author:${pr.user.login}`,
per_page: 1
@ -48,7 +48,7 @@ jobs:
try {
await github.rest.issues.createLabel({
owner, repo, name, color: 'd93f0b',
description: 'Closed while PR intake is limited to authors with at least 20 PRs merged into main'
description: 'Closed while PR intake is limited to authors with at least 10 PRs merged into main'
});
} catch (createError) {
if (createError.status !== 422) {

28
CMakeLists.txt
View File

@ -25,7 +25,9 @@ endif()
option(ANYPS5_ENABLE_SPIRV_TOOLS "Enable SPIRV-Tools validation and optimization" OFF)
option(APS5_ENABLE_TIMING_LOG "Write frame timings asynchronously to frame-timing.log" OFF)
option(ANYPS5_USE_SYSTEM_DEPS "Resolve SDL2, freetype, Vulkan-Headers, SPIRV-Headers, glslang and SPIRV-Tools from the system instead of the bundled submodules" OFF)
add_compile_definitions(APS5_ENABLE_TIMING_LOG=$<BOOL:${APS5_ENABLE_TIMING_LOG}>)
include(cmake/SystemDeps.cmake)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
@ -39,6 +41,8 @@ if(BUILD_TESTING)
if(Python3_Interpreter_FOUND)
add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py)
add_test(NAME pr_overlap_dependencies COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_dependencies.py)
add_test(NAME pr_overlap_tools COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap.py)
add_test(NAME progress_report COMMAND ${Python3_EXECUTABLE}
@ -80,10 +84,12 @@ set(SDL_HAPTIC OFF CACHE BOOL "" FORCE)
set(SDL_HIDAPI ON CACHE BOOL "" FORCE)
set(SDL_SENSOR OFF CACHE BOOL "" FORCE)
set(SDL_STATIC_PIC ON CACHE BOOL "" FORCE)
set(ANYPS5_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
set(CMAKE_EXE_LINKER_FLAGS "")
add_subdirectory(3rdparty/SDL2)
set(CMAKE_EXE_LINKER_FLAGS "${ANYPS5_EXE_LINKER_FLAGS}")
if(NOT ANYPS5_USE_SYSTEM_DEPS)
set(ANYPS5_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
set(CMAKE_EXE_LINKER_FLAGS "")
add_subdirectory(3rdparty/SDL2)
set(CMAKE_EXE_LINKER_FLAGS "${ANYPS5_EXE_LINKER_FLAGS}")
endif()
set(ARCHITECTURE x86_64)
add_subdirectory(3rdparty/ffmpeg-core)
if(MINGW)
@ -98,12 +104,14 @@ set(FT_DISABLE_PNG ON CACHE BOOL "" FORCE)
set(FT_DISABLE_HARFBUZZ ON CACHE BOOL "" FORCE)
set(FT_DISABLE_BROTLI ON CACHE BOOL "" FORCE)
set(SKIP_INSTALL_ALL ON CACHE BOOL "" FORCE)
add_subdirectory(3rdparty/freetype EXCLUDE_FROM_ALL)
set_target_properties(freetype PROPERTIES POSITION_INDEPENDENT_CODE ON)
if(WIN32)
get_target_property(freetypeSources freetype SOURCES)
list(FILTER freetypeSources EXCLUDE REGEX "ftver\\.rc$")
set_target_properties(freetype PROPERTIES SOURCES "${freetypeSources}")
if(NOT ANYPS5_USE_SYSTEM_DEPS)
add_subdirectory(3rdparty/freetype EXCLUDE_FROM_ALL)
set_target_properties(freetype PROPERTIES POSITION_INDEPENDENT_CODE ON)
if(WIN32)
get_target_property(freetypeSources freetype SOURCES)
list(FILTER freetypeSources EXCLUDE REGEX "ftver\\.rc$")
set_target_properties(freetype PROPERTIES SOURCES "${freetypeSources}")
endif()
endif()
add_subdirectory(core/shader/recompiler)

2
CONTRIBUTING.md
View File

@ -11,7 +11,7 @@
- Don't add replacements for modules that titles ship themselves in `sce_module/`, `sce_modules/` or `prx/`: engine or middleware modules (Cohtml, FMOD, GOG Galaxy) and SDK libraries that only wrap other system libraries (NpCppWebApi over NpWebApi2). The relinker converts and loads the title's own module, and a host library with the same name is left out of `DT_NEEDED`. Only system libraries, which reach the kernel or the hardware, are reimplemented in [core/libs/prx](core/libs/prx).
- Avoid non-standard extensions (`__attribute__`, etc.) where standard C++ is enough. Helper symbols that must not become NIDs use the `_nid_no_patch` or `_nid_no_patch_cut` suffix.
- The relinker uses only the C++20 standard library.
- Third-party code is added as a submodule under `3rdparty/` and built from source, not found on the system.
- Third-party code is added as a submodule under `3rdparty/` and built from source, not found on the system. With `ANYPS5_USE_SYSTEM_DEPS=ON` the packages packaged by the distributions (SDL2, freetype, Vulkan-Headers, SPIRV-Headers, glslang and SPIRV-Tools) come from the system instead; that resolution lives in [cmake/SystemDeps.cmake](cmake/SystemDeps.cmake) and the default build never reads the system.
- Don't add tests that only check that a symbol is exported: a missing export already fails at startup.
- Each test finishes in 30 s on the CI runners, Vulkan tests included (Linux runs them on lavapipe), counting the first run with no shader cache. ctest runs in parallel there, so a slow test competes for the cores and times out on some runs and not others. Split or shrink a test that gets close instead of raising its timeout. CI reports every test over 30 s as a warning on the pull request.

118
cmake/SystemDeps.cmake Normal file
View File

@ -0,0 +1,118 @@
set(ANYPS5_VULKAN_HEADERS_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include")
set(ANYPS5_SPIRV_HEADERS_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include")
set(ANYPS5_SDL2_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/3rdparty/SDL2/include")
set(ANYPS5_GLSLANG_INCLUDE_DIRS "${CMAKE_SOURCE_DIR}/3rdparty/glslang")
set(ANYPS5_SPIRV_HEADERS_SOURCE_DIR "${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers")
set(ANYPS5_GLSLANG_SOURCE_DIR "${CMAKE_SOURCE_DIR}/3rdparty/glslang")
if(NOT ANYPS5_USE_SYSTEM_DEPS)
return()
endif()
include(CheckCXXSourceCompiles)
find_package(SDL2 CONFIG REQUIRED)
if(NOT TARGET SDL2::SDL2)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: SDL2 was found but does not provide SDL2::SDL2")
endif()
add_library(SDL2-static INTERFACE)
target_link_libraries(SDL2-static INTERFACE SDL2::SDL2)
get_target_property(ANYPS5_SDL2_INCLUDE_DIR SDL2::SDL2 INTERFACE_INCLUDE_DIRECTORIES)
if(NOT ANYPS5_SDL2_INCLUDE_DIR)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: SDL2::SDL2 exposes no INTERFACE_INCLUDE_DIRECTORIES")
endif()
set(ANYPS5_SDL2_PROBE_INCLUDES ${ANYPS5_SDL2_INCLUDE_DIR})
foreach(_dir IN LISTS ANYPS5_SDL2_INCLUDE_DIR)
list(APPEND ANYPS5_SDL2_PROBE_INCLUDES "${_dir}/SDL2")
endforeach()
list(REMOVE_DUPLICATES ANYPS5_SDL2_PROBE_INCLUDES)
set(CMAKE_REQUIRED_INCLUDES ${ANYPS5_SDL2_PROBE_INCLUDES})
check_cxx_source_compiles("#include \"SDL_config.h\"\n#ifndef SDL_JOYSTICK_HIDAPI\n#error SDL_JOYSTICK_HIDAPI is not defined\n#endif\nint main() { return 0; }" ANYPS5_SDL2_HAS_HIDAPI)
unset(CMAKE_REQUIRED_INCLUDES)
if(NOT ANYPS5_SDL2_HAS_HIDAPI)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: the system SDL2 does not define SDL_JOYSTICK_HIDAPI, so a DualSense gets no light bar, touchpad or trigger effects and the disk audio driver does not preserve the requested format; sdl2-compat (SDL3 underneath) and SDL2 builds with HIDAPI disabled both fail the pad and audio tests; install a real SDL2 2.x built with HIDAPI")
endif()
find_package(Freetype REQUIRED)
if(NOT TARGET Freetype::Freetype)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: freetype was found but does not provide Freetype::Freetype")
endif()
add_library(freetype INTERFACE)
target_link_libraries(freetype INTERFACE Freetype::Freetype)
find_package(VulkanHeaders CONFIG REQUIRED)
if(NOT TARGET Vulkan::Headers)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: Vulkan-Headers was found but does not provide Vulkan::Headers")
endif()
get_target_property(ANYPS5_VULKAN_HEADERS_INCLUDE_DIR Vulkan::Headers INTERFACE_INCLUDE_DIRECTORIES)
if(NOT ANYPS5_VULKAN_HEADERS_INCLUDE_DIR)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: Vulkan::Headers exposes no INTERFACE_INCLUDE_DIRECTORIES")
endif()
set(CMAKE_REQUIRED_INCLUDES ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
check_cxx_source_compiles("#include <vulkan/vulkan.h>\nint main() { VkPhysicalDeviceMaintenance8FeaturesKHR features{}; (void)features; return 0; }" ANYPS5_VULKAN_HAS_MAINTENANCE8)
unset(CMAKE_REQUIRED_INCLUDES)
if(NOT ANYPS5_VULKAN_HAS_MAINTENANCE8)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: the system Vulkan-Headers do not provide VkPhysicalDeviceMaintenance8FeaturesKHR (VK_KHR_maintenance8), which the agc driver uses; install Vulkan-Headers 1.4.310 or newer")
endif()
find_package(SPIRV-Headers CONFIG REQUIRED)
if(NOT TARGET SPIRV-Headers::SPIRV-Headers)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: SPIRV-Headers was found but does not provide SPIRV-Headers::SPIRV-Headers")
endif()
get_target_property(ANYPS5_SPIRV_HEADERS_INCLUDE_DIR SPIRV-Headers::SPIRV-Headers INTERFACE_INCLUDE_DIRECTORIES)
if(NOT ANYPS5_SPIRV_HEADERS_INCLUDE_DIR)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: SPIRV-Headers::SPIRV-Headers exposes no INTERFACE_INCLUDE_DIRECTORIES")
endif()
find_package(glslang CONFIG REQUIRED)
if(NOT TARGET glslang::glslang OR NOT TARGET glslang::SPIRV)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: glslang was found but does not provide glslang::glslang and glslang::SPIRV")
endif()
add_library(glslang INTERFACE)
target_link_libraries(glslang INTERFACE glslang::glslang)
add_library(SPIRV INTERFACE)
target_link_libraries(SPIRV INTERFACE glslang::SPIRV)
find_path(ANYPS5_GLSLANG_PUBLIC_INCLUDE_DIR NAMES glslang/Public/ShaderLang.h)
if(NOT ANYPS5_GLSLANG_PUBLIC_INCLUDE_DIR)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: glslang/Public/ShaderLang.h was not found; the glslang headers are incomplete")
endif()
set(ANYPS5_GLSLANG_SPIRV_INCLUDE_DIR "${ANYPS5_GLSLANG_PUBLIC_INCLUDE_DIR}/glslang")
if(NOT EXISTS "${ANYPS5_GLSLANG_SPIRV_INCLUDE_DIR}/SPIRV/GlslangToSpv.h")
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: ${ANYPS5_GLSLANG_SPIRV_INCLUDE_DIR}/SPIRV/GlslangToSpv.h was not found")
endif()
set(ANYPS5_GLSLANG_INCLUDE_DIRS "${ANYPS5_GLSLANG_PUBLIC_INCLUDE_DIR};${ANYPS5_GLSLANG_SPIRV_INCLUDE_DIR}")
find_program(ANYPS5_GLSLANG_STANDALONE_EXECUTABLE NAMES glslang glslangValidator)
if(NOT ANYPS5_GLSLANG_STANDALONE_EXECUTABLE)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: the glslang compiler (glslang or glslangValidator) was not found; install the glslang tools package")
endif()
add_executable(glslang-standalone IMPORTED GLOBAL)
set_target_properties(glslang-standalone PROPERTIES IMPORTED_LOCATION "${ANYPS5_GLSLANG_STANDALONE_EXECUTABLE}")
if(ANYPS5_ENABLE_SPIRV_TOOLS)
find_package(SPIRV-Tools-opt CONFIG REQUIRED)
if(NOT TARGET SPIRV-Tools-opt)
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: SPIRV-Tools was found but does not provide SPIRV-Tools-opt")
endif()
if(NOT TARGET SPIRV-Tools-static)
if(TARGET SPIRV-Tools)
add_library(SPIRV-Tools-static INTERFACE)
target_link_libraries(SPIRV-Tools-static INTERFACE SPIRV-Tools)
else()
message(FATAL_ERROR "ANYPS5_USE_SYSTEM_DEPS: SPIRV-Tools was found but provides neither SPIRV-Tools-static nor SPIRV-Tools")
endif()
endif()
endif()
set(ANYPS5_SYSTEM_DEPS_STAMP "")
list(APPEND ANYPS5_SYSTEM_DEPS_PACKAGES VulkanHeaders SPIRV-Headers glslang)
if(ANYPS5_ENABLE_SPIRV_TOOLS)
list(APPEND ANYPS5_SYSTEM_DEPS_PACKAGES SPIRV-Tools-opt)
endif()
foreach(_package IN LISTS ANYPS5_SYSTEM_DEPS_PACKAGES)
set(_version "unknown")
if(DEFINED ${_package}_VERSION)
set(_version "${${_package}_VERSION}")
endif()
string(APPEND ANYPS5_SYSTEM_DEPS_STAMP " ${_package}=${_version}")
endforeach()

52
core/libs/CMakeLists.txt
View File

@ -3,6 +3,12 @@ if(NOT WIN32)
string(STRIP "${CMAKE_EXE_LINKER_FLAGS}" CMAKE_EXE_LINKER_FLAGS)
endif()
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
include(CheckPIESupported)
check_pie_supported(LANGUAGES C CXX)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
endif()
add_executable(nid_patcher
${CMAKE_CURRENT_SOURCE_DIR}/nid/src/Sha1.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nid/src/NidCompute.cpp
@ -47,6 +53,9 @@ set(LIBS_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
add_subdirectory(${LIBC_DIR} ${CMAKE_CURRENT_BINARY_DIR}/prx/libc)
target_include_directories(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
configure_windows_unwind(libc)
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
target_link_options(libc PRIVATE LINKER:-z,now)
endif()
set(UNPATCHED_DIR ${CMAKE_CURRENT_BINARY_DIR}/libs/unpatched)
set(PATCHED_DIR ${CMAKE_CURRENT_BINARY_DIR}/libs)
@ -96,6 +105,9 @@ foreach(LIB_DIR ${LIBS_DIRS})
else()
target_link_libraries(${LIB_NAME} PRIVATE libkernel libc)
endif()
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
target_link_options(${LIB_NAME} PRIVATE LINKER:-z,now)
endif()
if(WIN32 AND LIB_NAME STREQUAL "libSceNet")
target_link_libraries(${LIB_NAME} PRIVATE ws2_32)
target_compile_definitions(${LIB_NAME} PRIVATE _WIN32_WINNT=0x0602)
@ -167,6 +179,36 @@ if(BUILD_TESTING)
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceHttp>")
endif()
add_test_executable(guest_rudp_status_tests tests/GuestRudpStatus.cpp)
target_include_directories(guest_rudp_status_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_rudp_status_tests PRIVATE libSceRudp libc)
configure_windows_unwind(guest_rudp_status_tests)
add_test(NAME guest_rudp_status COMMAND guest_rudp_status_tests)
if(WIN32)
set_tests_properties(guest_rudp_status PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceRudp>")
endif()
add_test_executable(guest_memory_pool_stats_tests tests/GuestMemoryPoolStats.cpp)
target_include_directories(guest_memory_pool_stats_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_memory_pool_stats_tests PRIVATE libSceSsl libSceHttp2 libSceNet libkernel libc)
configure_windows_unwind(guest_memory_pool_stats_tests)
add_test(NAME guest_memory_pool_stats COMMAND guest_memory_pool_stats_tests)
if(WIN32)
set_tests_properties(guest_memory_pool_stats PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSsl>")
endif()
add_test_executable(guest_system_gesture_tests tests/GuestSystemGesture.cpp)
target_include_directories(guest_system_gesture_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_system_gesture_tests PRIVATE libSceSystemGesture libc)
configure_windows_unwind(guest_system_gesture_tests)
add_test(NAME guest_system_gesture COMMAND guest_system_gesture_tests)
if(WIN32)
set_tests_properties(guest_system_gesture PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSystemGesture>")
endif()
if(UNIX AND (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$" OR CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64"))
add_subdirectory(tests/register_context)
endif()
@ -238,4 +280,14 @@ if(BUILD_TESTING)
set_tests_properties(savedata_native_write_replacement PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceSaveData.native>")
endif()
add_test_executable(guest_posix_random_tests tests/GuestPosixRandom.cpp)
target_include_directories(guest_posix_random_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_posix_random_tests PRIVATE libScePosixForWebKit)
configure_windows_unwind(guest_posix_random_tests)
add_test(NAME guest_posix_random COMMAND guest_posix_random_tests)
if(WIN32)
set_tests_properties(guest_posix_random PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libScePosixForWebKit>")
endif()
endif()

12
core/libs/SceTypes.hpp
View File

@ -1278,6 +1278,18 @@ struct NpTitleId { char data[13]; char pad[3]; };
struct NpTitleSecret { std::uint8_t data[128]; };
struct NpContentRestriction { std::uint8_t opaque[128]; };
struct NpOnlineId { char data[17]; char pad[3]; };
struct SharePlayConnectionInfoA {
std::int32_t status;
std::int32_t mode;
NpOnlineId hostOnlineId;
NpOnlineId visitorOnlineId;
std::uint64_t hostAccountId;
std::uint64_t visitorAccountId;
std::int32_t hostUserId;
std::int32_t visitorUserId;
};
static_assert(sizeof(SharePlayConnectionInfoA) == 72 && offsetof(SharePlayConnectionInfoA, hostAccountId) == 48 && offsetof(SharePlayConnectionInfoA, hostUserId) == 64);
struct NpId { NpOnlineId online_id; std::uint8_t opaque[4]; };
struct NpCreateAsyncRequestParameter { std::uint8_t opaque[64]; };
struct NpCheckPremiumParameter { std::uint8_t opaque[64]; };

2
core/libs/prx/libSceAgc/CMakeLists.txt
View File

@ -209,7 +209,7 @@ if(BUILD_TESTING)
endif()
add_test_executable(agc_shader_fusion_tests EXCLUDE_FROM_ALL tests/ShaderFusion.cpp)
target_include_directories(agc_shader_fusion_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include ${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include)
target_include_directories(agc_shader_fusion_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR} ${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR})
target_link_libraries(agc_shader_fusion_tests PRIVATE libSceAgc libSceAgcDriver libc)
set_target_properties(agc_shader_fusion_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_shader_fusion_tests)

2
core/libs/prx/libSceAgc/Command/src/Packet.cpp
View File

@ -85,7 +85,7 @@ void ValidatePacket(const std::uint32_t* packet, std::uint32_t opcode, std::uint
}
std::uint32_t* WriteNop(CommandBuffer* buffer, std::uint32_t count, const char* function) {
const auto header = Header(0x10u, count);
const auto header = count == 1 ? 0xffff1000u : Header(0x10u, count);
auto* packet = Allocate(buffer, count, function);
packet[0] = header;
std::fill_n(packet + 1, count - 1u, 0u);

2
core/libs/prx/libSceAgc/DcbDraw/src/Instancing.cpp
View File

@ -20,7 +20,7 @@ uint32_t* APS5_VABI sceAgcDcbSetBaseIndirectArgs(CommandBuffer* buf, uint32_t sh
// SET_BASE with base index 1; header bit 1 selects the dispatch (compute) base over the draw base.
Agc::Command::CheckBits(shader_type, 1, __func__);
const auto address = reinterpret_cast<std::uintptr_t>(indirect_base_addr);
Agc::Command::CheckAddress(address, 8, __func__);
if (address != 0) Agc::Command::CheckAddress(address, 8, __func__);
Agc::Command::CheckBits(address, 0xffffffffffffull, __func__);
auto* packet = Agc::Command::Allocate(buf, 4, __func__);
packet[0] = Agc::Command::Header(0x11u, 4, shader_type != 0 ? 2u : 0u);

6
core/libs/prx/libSceAgc/DcbState/src/Registers.cpp
View File

@ -29,7 +29,7 @@ std::uint32_t* APS5_VABI sceAgcDcbSetCxRegistersIndirect(CommandBuffer* buf, con
}
std::uint32_t APS5_VABI sceAgcDcbSetCxRegistersIndirectGetSize(std::uint32_t numRegs) {
Agc::Command::CheckBits(numRegs, 0x3fffu, __func__);
static_cast<void>(numRegs);
return 20;
}
@ -46,7 +46,7 @@ std::uint32_t* APS5_VABI sceAgcDcbSetShRegistersIndirect(CommandBuffer* buf, con
}
std::uint32_t APS5_VABI sceAgcDcbSetShRegistersIndirectGetSize(std::uint32_t numRegs) {
Agc::Command::CheckBits(numRegs, 0x3fffu, __func__);
static_cast<void>(numRegs);
return 20;
}
@ -63,7 +63,7 @@ std::uint32_t* APS5_VABI sceAgcDcbSetUcRegistersIndirect(CommandBuffer* buf, con
}
std::uint32_t APS5_VABI sceAgcDcbSetUcRegistersIndirectGetSize(std::uint32_t numRegs) {
Agc::Command::CheckBits(numRegs, 0x3fffu, __func__);
static_cast<void>(numRegs);
return 20;
}

16
core/libs/prx/libSceAgc/Patch/src/Branch.cpp
View File

@ -19,6 +19,12 @@ void PatchBranchTarget(std::uint32_t* cmd, std::uint32_t field, const volatile s
cmd[field + 2u] = (cmd[field + 2u] & ~0xfffffu) | sizeInDwords;
}
void PatchBranchTarget(std::uint32_t* cmd, std::uint32_t field, std::uint8_t cachePolicy, const volatile std::uint32_t* target, std::uint32_t sizeInDwords, const char* function) {
Agc::Command::CheckBits(cachePolicy, 3, function);
PatchBranchTarget(cmd, field, target, sizeInDwords, function);
cmd[field + 2u] = (cmd[field + 2u] & ~(3u << 28u)) | (static_cast<std::uint32_t>(cachePolicy) << 28u);
}
}
extern "C" {
@ -42,4 +48,14 @@ int APS5_VABI sceAgcBranchPatchSetElseTarget(std::uint32_t* cmd, const volatile
return 0;
}
int APS5_VABI sceAgcBranchPatchSetThenTarget_0300(std::uint32_t* cmd, std::uint8_t cachePolicy, const volatile std::uint32_t* target, std::uint32_t sizeInDwords) {
PatchBranchTarget(cmd, 8, cachePolicy, target, sizeInDwords, __func__);
return 0;
}
int APS5_VABI sceAgcBranchPatchSetElseTarget_0300(std::uint32_t* cmd, std::uint8_t cachePolicy, const volatile std::uint32_t* target, std::uint32_t sizeInDwords) {
PatchBranchTarget(cmd, 11, cachePolicy, target, sizeInDwords, __func__);
return 0;
}
}

10
core/libs/prx/libSceAgc/Unimplemented.cpp
View File

@ -80,11 +80,6 @@ int APS5_VABI sceAgcCbMemsetExclusive() {
return 0;
}
int APS5_VABI sceAgcBranchPatchSetThenTarget_0300() {
NotImplemented_nid_no_patch(__func__);
return 0;
}
int APS5_VABI sceAgcGetGsPrimPayload() {
NotImplemented_nid_no_patch(__func__);
return 0;
@ -100,11 +95,6 @@ int APS5_VABI sceAgcGetShaderInstrumentation() {
return 0;
}
int APS5_VABI sceAgcBranchPatchSetElseTarget_0300() {
NotImplemented_nid_no_patch(__func__);
return 0;
}
APS5_EXPORT("vieBRwlh1Lw", sceAgcUnknown_vieBRwlh1Lw);
int APS5_VABI sceAgcUnknown_vieBRwlh1Lw(void) {
NotImplemented_nid_no_patch(__func__);

29
core/libs/prx/libSceAgc/tests/BranchPatch.cpp
View File

@ -13,6 +13,8 @@ std::uint32_t APS5_VABI sceAgcCbBranchGetSize();
int APS5_VABI sceAgcBranchPatchSetCompareAddress(std::uint32_t*, const volatile std::uint64_t*);
int APS5_VABI sceAgcBranchPatchSetThenTarget(std::uint32_t*, const volatile std::uint32_t*, std::uint32_t);
int APS5_VABI sceAgcBranchPatchSetElseTarget(std::uint32_t*, const volatile std::uint32_t*, std::uint32_t);
int APS5_VABI sceAgcBranchPatchSetThenTarget_0300(std::uint32_t*, std::uint8_t, const volatile std::uint32_t*, std::uint32_t);
int APS5_VABI sceAgcBranchPatchSetElseTarget_0300(std::uint32_t*, std::uint8_t, const volatile std::uint32_t*, std::uint32_t);
}
namespace {
@ -107,6 +109,31 @@ void testTarget(TSetter setter, std::size_t field, std::uint32_t cachePolicyBits
check(storage.words == expected, name);
}
template <typename TSetter>
void testTargetWithCachePolicy(TSetter setter, std::size_t field, const char* name) {
Storage storage;
auto expected = storage.words;
expected[field] = 0x89abcdecu;
expected[field + 1] = 0x4567u;
expected[field + 2] = 0x30000004u;
check(setter(storage.packet, 3, targetAt(0x0000456789abcdecull), 4) == 0, name);
check(storage.words == expected, name);
expected[field] = 0x1000u;
expected[field + 1] = 0;
expected[field + 2] = 0xfffffu;
check(setter(storage.packet, 0, targetAt(0x1000u), 0xfffffu) == 0, name);
check(storage.words == expected, name);
expectFailure([&] { setter(storage.packet, 4, targetAt(0x2000u), 1); });
expectFailure([&] { setter(storage.packet, 1, nullptr, 1); });
expectFailure([&] { setter(storage.packet, 1, targetAt(0x1002u), 1); });
expectFailure([&] { setter(storage.packet, 1, targetAt(0x2000u), 0x100000u); });
expectFailure([&] { setter(nullptr, 1, targetAt(0x2000u), 1); });
storage.packet[0] = 0xc00b3f00u;
expected[0] = 0xc00b3f00u;
expectFailure([&] { setter(storage.packet, 1, targetAt(0x2000u), 1); });
check(storage.words == expected, name);
}
void testMalformedPackets() {
for (const auto header : {0xc0023f00u, 0xc00b3f00u, 0xc00c3f01u, 0xc00c2200u}) {
Storage storage;
@ -128,6 +155,8 @@ int main() {
testCompareAddress();
testTarget(sceAgcBranchPatchSetThenTarget, 8, 0x10000000u, "then target setter wrote the wrong words");
testTarget(sceAgcBranchPatchSetElseTarget, 11, 0x20000000u, "else target setter wrote the wrong words");
testTargetWithCachePolicy(sceAgcBranchPatchSetThenTarget_0300, 8, "3.00 then target setter wrote the wrong words");
testTargetWithCachePolicy(sceAgcBranchPatchSetElseTarget_0300, 11, "3.00 else target setter wrote the wrong words");
testMalformedPackets();
std::puts("AGC branch patch tests passed");
return 0;

32
core/libs/prx/libSceAgc/tests/Commands.cpp
View File

@ -34,6 +34,8 @@ extern "C" std::uint32_t* APS5_VABI sceAgcAcbPushMarker(CommandBuffer* buf, cons
extern "C" std::uint32_t* APS5_VABI sceAgcAcbPopMarker(CommandBuffer* buf);
extern "C" std::uint32_t* APS5_VABI sceAgcAcbSetMarker(CommandBuffer* buf, const char* str, std::uint32_t color);
extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetIndexBuffer(CommandBuffer* buf, std::uint64_t indexAddress);
extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetBaseIndirectArgs(CommandBuffer* buf, std::uint32_t shaderType, const volatile void* indirectBaseAddr);
extern "C" std::uint32_t* APS5_VABI sceAgcSetNop(CommandBuffer* buf, std::uint32_t sizeDw);
namespace {
@ -109,6 +111,21 @@ void testPackets() {
check(noDown.cursor_up == scratch.data() + 6, "allocation past the top of a buffer without a down cursor advanced its cursor");
}
void testNop() {
Storage storage;
storage.words.fill(0xdeadbeefu);
auto* single = sceAgcSetNop(&storage.buffer, 1);
auto* pair = sceAgcSetNop(&storage.buffer, 2);
auto* triple = sceAgcSetNop(&storage.buffer, 3);
const std::array<std::uint32_t, 7> expected{0xffff1000u, 0xc0001000u, 0, 0xc0011000u, 0, 0, 0xdeadbeefu};
check(single == storage.words.data() && pair == single + 1 && triple == pair + 2, "incorrect NOP placement");
check(std::equal(expected.begin(), expected.end(), storage.words.begin()), "incorrect NOP packet");
check(storage.buffer.cursor_up == storage.words.data() + 6, "incorrect NOP cursor advance");
expectFailure([&] { sceAgcSetNop(&storage.buffer, 0); });
expectFailure([&] { sceAgcSetNop(&storage.buffer, 0x4002u); });
check(storage.buffer.cursor_up == storage.words.data() + 6, "invalid NOP advanced the cursor");
}
void testClearState() {
Storage storage;
storage.words.fill(0xdeadbeefu);
@ -205,6 +222,19 @@ void testIndexBuffer() {
throw std::runtime_error("misaligned index buffer was accepted");
}
void testIndirectBase() {
Storage storage;
alignas(8) std::uint32_t arguments[4]{};
const auto address = reinterpret_cast<std::uintptr_t>(arguments);
const auto* bound = sceAgcDcbSetBaseIndirectArgs(&storage.buffer, 0, arguments);
check(bound[1] == 1 && bound[2] == static_cast<std::uint32_t>(address) && bound[3] == static_cast<std::uint32_t>(address >> 32u), "indirect base address mismatch");
const auto* cleared = sceAgcDcbSetBaseIndirectArgs(&storage.buffer, 1, nullptr);
check(cleared == bound + 4 && (cleared[0] & 2u) != 0 && cleared[1] == 1 && cleared[2] == 0 && cleared[3] == 0, "indirect base was not cleared");
const auto before = storage.words;
expectFailure([&] { sceAgcDcbSetBaseIndirectArgs(&storage.buffer, 0, reinterpret_cast<const volatile void*>(address + 4)); });
check(storage.words == before, "misaligned indirect base modified packet memory");
}
void testContextState() {
const std::array<std::array<std::uint32_t, 6>, 4> sizes{{{5}, {5, 8, 9, 3, 2}, {3, 5, 8, 9, 2}, {5, 8, 9, 3, 2, 5}}};
const std::array<std::array<std::uint32_t, 4>, 4> reservations{{{5}, {22, 3, 2}, {3, 22, 2}, {22, 3, 2, 5}}};
@ -404,10 +434,12 @@ int main(int argc, char** argv) {
return 0;
}
testPackets();
testNop();
testClearState();
testIndexedIndirectDraws();
testMarkers();
testIndexBuffer();
testIndirectBase();
testContextState();
testFlip();
testRegisters();

4
core/libs/prx/libSceAgc/tests/RegisterCommands.cpp
View File

@ -116,8 +116,8 @@ void testIndirect() {
check(storage.words == expected, "count setter modified unrelated packet data");
}
const auto before = storage.words;
for (const auto count : {0x4000u, 0xffffffffu}) {
expectFailure([&] { sizes[i](count); });
for (const auto count : {0x4000u, 0xffffffffu, 0xf5f5039cu}) {
check(sizes[i](count) == 20, "indirect size depends on the count");
expectFailure([&] { setters[i](packet, count); });
}
expectFailure([&] { setters[i](nullptr, 0); });

214
core/libs/prx/libSceAgcDriver/CMakeLists.txt
View File

@ -161,6 +161,7 @@ set(agcDriverSources
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Synchronization/WaitMemory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Synchronization/WaitStatistics.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/WorkerSampler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/ThreadPriority.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/GuestMemory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/ShaderMemory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/BdaFeatures.cpp
@ -178,6 +179,7 @@ set(agcDriverSources
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineLibrary.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineCache.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Resources.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PlaceholderImages.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ParallelCompare.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderInputState.cpp
@ -223,8 +225,8 @@ target_include_directories(libSceAgcDriver PRIVATE
${LIBS_INCLUDE_DIR}
${agcTextureDetileGeneratedRoot}
${CMAKE_SOURCE_DIR}/core/shader/recompiler/RdnaDecoder/include
${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include
${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR}
)
file(GLOB_RECURSE agcShaderCacheGuardSources CONFIGURE_DEPENDS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
@ -243,6 +245,7 @@ target_link_libraries(libSceAgcDriver PRIVATE
SDL2-static
libc
libkernel
${CMAKE_DL_LIBS}
)
set_target_properties(libSceAgcDriver PROPERTIES CXX_EXTENSIONS OFF)
@ -286,8 +289,8 @@ add_test_executable(agc_driver_tests EXCLUDE_FROM_ALL
target_include_directories(agc_driver_tests PRIVATE
${LIBS_INCLUDE_DIR}
${CMAKE_SOURCE_DIR}/core/shader/recompiler
${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include
${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR}
)
target_link_libraries(agc_driver_tests PRIVATE
@ -299,6 +302,23 @@ target_link_libraries(agc_driver_tests PRIVATE
set_target_properties(agc_driver_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_driver_tests)
add_test_executable(agc_driver_wait_shutdown_tests EXCLUDE_FROM_ALL tests/WaitShutdown.cpp)
target_include_directories(agc_driver_wait_shutdown_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_driver_wait_shutdown_tests PRIVATE libSceAgcDriver libkernel libc)
set_target_properties(agc_driver_wait_shutdown_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_driver_wait_shutdown_tests)
if(BUILD_TESTING)
set_target_properties(agc_driver_wait_shutdown_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_driver_wait_shutdown COMMAND agc_driver_wait_shutdown_tests)
set_tests_properties(agc_driver_wait_shutdown PROPERTIES
ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_GPU_WAIT_TIMEOUT_MS=20"
TIMEOUT 10 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_driver_wait_shutdown PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_shader_memory_tests EXCLUDE_FROM_ALL
tests/ShaderMemory.cpp
Execution/src/ShaderMemory.cpp
@ -333,13 +353,36 @@ if(BUILD_TESTING)
endif()
endif()
add_test_executable(agc_thread_priority_tests EXCLUDE_FROM_ALL
tests/ThreadPriority.cpp
Execution/src/ThreadPriority.cpp
)
target_include_directories(agc_thread_priority_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_thread_priority_tests PRIVATE libc ${CMAKE_DL_LIBS})
set_target_properties(agc_thread_priority_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_thread_priority_tests)
if(BUILD_TESTING)
set_target_properties(agc_thread_priority_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_thread_priority COMMAND agc_thread_priority_tests)
if(WIN32)
set_tests_properties(agc_thread_priority PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
else()
foreach(mode rt high)
add_test(NAME agc_thread_priority_refused_${mode} COMMAND agc_thread_priority_tests refused)
set_tests_properties(agc_thread_priority_refused_${mode} PROPERTIES ENVIRONMENT
"APS5_THREAD_PRIORITY=${mode};DBUS_SYSTEM_BUS_ADDRESS=unix:path=/nonexistent/aps5-no-rtkit")
endforeach()
endif()
endif()
add_test_executable(agc_structurizer_tests EXCLUDE_FROM_ALL tests/Structurizer.cpp)
target_link_libraries(agc_structurizer_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_structurizer_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_structurizer_tests)
add_test_executable(agc_fragment_barycentric_tests EXCLUDE_FROM_ALL tests/FragmentBarycentric.cpp)
target_include_directories(agc_fragment_barycentric_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_include_directories(agc_fragment_barycentric_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
set_target_properties(agc_fragment_barycentric_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_fragment_barycentric_tests)
if(BUILD_TESTING)
@ -351,7 +394,7 @@ add_test_executable(agc_subgroup_clock_tests EXCLUDE_FROM_ALL
tests/SubgroupClock.cpp
Execution/src/SubgroupClock.cpp
)
target_include_directories(agc_subgroup_clock_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_include_directories(agc_subgroup_clock_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
set_target_properties(agc_subgroup_clock_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_subgroup_clock_tests)
if(BUILD_TESTING)
@ -406,6 +449,19 @@ if(BUILD_TESTING)
endif()
endif()
add_test_executable(agc_spirv_lds_atomic_lock_tests EXCLUDE_FROM_ALL tests/SpirvLdsAtomicLock.cpp)
target_link_libraries(agc_spirv_lds_atomic_lock_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_spirv_lds_atomic_lock_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_spirv_lds_atomic_lock_tests)
if(BUILD_TESTING)
set_target_properties(agc_spirv_lds_atomic_lock_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_spirv_lds_atomic_lock COMMAND agc_spirv_lds_atomic_lock_tests)
set_tests_properties(agc_spirv_lds_atomic_lock PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1")
if(WIN32)
set_tests_properties(agc_spirv_lds_atomic_lock PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_dpp_write_tests EXCLUDE_FROM_ALL tests/DppWrite.cpp)
target_link_libraries(agc_dpp_write_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_dpp_write_tests PROPERTIES CXX_EXTENSIONS OFF)
@ -471,13 +527,13 @@ if(BUILD_TESTING)
endif()
add_test_executable(agc_pass_hazards_tests EXCLUDE_FROM_ALL tests/PassHazards.cpp Graphics/src/PassHazards.cpp)
target_include_directories(agc_pass_hazards_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_include_directories(agc_pass_hazards_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
target_link_libraries(agc_pass_hazards_tests PRIVATE libc)
set_target_properties(agc_pass_hazards_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_pass_hazards_tests)
add_test_executable(agc_dcc_key_sharing_tests EXCLUDE_FROM_ALL tests/DccKeySharing.cpp)
target_include_directories(agc_dcc_key_sharing_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_include_directories(agc_dcc_key_sharing_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
target_link_libraries(agc_dcc_key_sharing_tests PRIVATE libc)
set_target_properties(agc_dcc_key_sharing_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_dcc_key_sharing_tests)
@ -491,7 +547,7 @@ if(BUILD_TESTING)
endif()
add_test_executable(agc_buffer_pool_tests EXCLUDE_FROM_ALL tests/BufferPool.cpp)
target_include_directories(agc_buffer_pool_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_include_directories(agc_buffer_pool_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
target_link_libraries(agc_buffer_pool_tests PRIVATE libSceAgcDriver libc)
set_target_properties(agc_buffer_pool_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_buffer_pool_tests)
@ -505,7 +561,7 @@ if(BUILD_TESTING)
endif()
add_test_executable(agc_display_buffer_dcc_tests EXCLUDE_FROM_ALL tests/DisplayBufferDcc.cpp)
target_include_directories(agc_display_buffer_dcc_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_include_directories(agc_display_buffer_dcc_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
target_link_libraries(agc_display_buffer_dcc_tests PRIVATE libSceAgcDriver libc)
set_target_properties(agc_display_buffer_dcc_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_display_buffer_dcc_tests)
@ -534,8 +590,14 @@ target_link_libraries(agc_shader_disk_cache_tests PRIVATE shader_recompiler_agc_
set_target_properties(agc_shader_disk_cache_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_shader_disk_cache_tests)
add_test_executable(agc_placeholder_images_tests EXCLUDE_FROM_ALL tests/PlaceholderImages.cpp)
target_include_directories(agc_placeholder_images_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_link_libraries(agc_placeholder_images_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_placeholder_images_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_placeholder_images_tests)
add_test_executable(agc_shader_device_profile_tests EXCLUDE_FROM_ALL tests/ShaderDeviceProfile.cpp Execution/src/ShaderDeviceProfile.cpp)
target_include_directories(agc_shader_device_profile_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_include_directories(agc_shader_device_profile_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
target_link_libraries(agc_shader_device_profile_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_shader_device_profile_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_shader_device_profile_tests)
@ -638,7 +700,7 @@ target_include_directories(agc_driver_cmask_fast_clear_tests PRIVATE
${LIBS_INCLUDE_DIR}
${agcTestShaderGeneratedRoot}
${CMAKE_SOURCE_DIR}/core/shader/recompiler
${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
)
target_link_libraries(agc_driver_cmask_fast_clear_tests PRIVATE
@ -663,7 +725,7 @@ target_include_directories(agc_driver_recorder_tests PRIVATE
${LIBS_INCLUDE_DIR}
${agcTestShaderGeneratedRoot}
${CMAKE_SOURCE_DIR}/core/shader/recompiler
${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
)
target_link_libraries(agc_driver_recorder_tests PRIVATE
@ -699,16 +761,24 @@ if(BUILD_TESTING)
set_tests_properties(agc_unchanged_cpu_stamps PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_storage_baseline_settled COMMAND agc_driver_recorder_tests --settled-baseline-only)
set_tests_properties(agc_storage_baseline_settled PROPERTIES SKIP_RETURN_CODE 77)
if(WIN32)
set_tests_properties(agc_storage_baseline_settled PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_host_import_unmap COMMAND agc_driver_recorder_tests --host-import-unmap-only)
set_tests_properties(agc_host_import_unmap PROPERTIES SKIP_RETURN_CODE 77)
add_test(NAME agc_unit_shadow COMMAND agc_driver_recorder_tests --unit-shadow-only)
set_tests_properties(agc_unit_shadow PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30 ENVIRONMENT "APS5_PIN_WAIT_MS=3000")
if(WIN32)
set_tests_properties(agc_host_import_unmap PROPERTIES ENVIRONMENT_MODIFICATION
set_tests_properties(agc_host_import_unmap agc_unit_shadow PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_recorder_kept_bytes_tests EXCLUDE_FROM_ALL tests/RecorderKeptBytes.cpp)
target_include_directories(agc_recorder_kept_bytes_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_include_directories(agc_recorder_kept_bytes_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
target_link_libraries(agc_recorder_kept_bytes_tests PRIVATE libSceAgcDriver libc)
set_target_properties(agc_recorder_kept_bytes_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_recorder_kept_bytes_tests)
@ -758,6 +828,7 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL
Graphics/src/ColorTargetLayout.cpp
Graphics/src/ColorTargetTransfer.cpp
Graphics/src/Resources.cpp
Graphics/src/PlaceholderImages.cpp
Graphics/src/ShaderResources.cpp
Graphics/src/ShaderAddressResources.cpp
Graphics/src/GuestBufferMemory.cpp
@ -794,8 +865,8 @@ target_include_directories(agc_driver_graphics_tests PRIVATE
${CMAKE_SOURCE_DIR}/core/shader/recompiler/IntermediateRepresentation/include
${CMAKE_SOURCE_DIR}/core/shader/recompiler/ControlFlow/include
${CMAKE_SOURCE_DIR}/core/shader/recompiler/RdnaDecoder/include
${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include
${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR}
)
target_link_libraries(agc_driver_graphics_tests PRIVATE
@ -814,13 +885,14 @@ add_test_executable(agc_driver_bda_device_tests EXCLUDE_FROM_ALL
tests/BdaContracts.cpp
Execution/src/BdaFeatures.cpp
Graphics/src/Resources.cpp
Graphics/src/PlaceholderImages.cpp
Graphics/src/BufferAddress.cpp
)
target_include_directories(agc_driver_bda_device_tests PRIVATE
${LIBS_INCLUDE_DIR}
${CMAKE_SOURCE_DIR}/core/shader/recompiler
${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
)
target_link_libraries(agc_driver_bda_device_tests PRIVATE SDL2-static libc shader_recompiler_agc_driver)
set_target_properties(agc_driver_bda_device_tests PROPERTIES CXX_EXTENSIONS OFF)
@ -833,10 +905,10 @@ add_test_executable(agc_driver_mesh_tests EXCLUDE_FROM_ALL
target_include_directories(agc_driver_mesh_tests PRIVATE
${LIBS_INCLUDE_DIR}
${agcTextureDetileGeneratedRoot}
${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include
${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR}
)
target_link_libraries(agc_driver_mesh_tests PRIVATE shader_recompiler_agc_driver SDL2-static libc libkernel)
target_link_libraries(agc_driver_mesh_tests PRIVATE shader_recompiler_agc_driver SDL2-static libc libkernel ${CMAKE_DL_LIBS})
set_target_properties(agc_driver_mesh_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_driver_mesh_tests)
@ -850,8 +922,8 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
target_include_directories(${agcExecutionTest} PRIVATE
${LIBS_INCLUDE_DIR}
${CMAKE_SOURCE_DIR}/core/shader/recompiler
${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include
${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR}
)
target_link_libraries(${agcExecutionTest} PRIVATE libSceAgcDriver SDL2-static libc)
set_target_properties(${agcExecutionTest} PROPERTIES CXX_EXTENSIONS OFF)
@ -859,8 +931,12 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
list(APPEND agcExecutionTests ${agcExecutionTest})
endforeach()
target_include_directories(agc_driver_alpha_to_coverage_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_dual_source_blend_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_swap_alt_blend_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_packed_unorm_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
foreach(stage IN ITEMS vert frag)
@ -888,6 +964,8 @@ target_include_directories(agc_driver_prepared_publication_tests PRIVATE $<TARGE
target_include_directories(agc_driver_prepared_graphics_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_link_libraries(agc_driver_interpolation_modes_tests PRIVATE shader_recompiler_agc_driver)
target_include_directories(agc_driver_prepared_unregistered_graphics_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_prepared_shaders_tests PRIVATE
$<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>
)
@ -908,8 +986,8 @@ if(BUILD_TESTING)
if(WIN32)
set_tests_properties(agc_shader_bindless agc_shader_frontend_pair agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_constant_bit_test_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests})
if(agcTest STREQUAL "agc_driver_constant_store_tests")
foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_constant_bit_test_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests})
if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests")
continue()
endif()
string(REGEX REPLACE "_tests$" "" agcTestName "${agcTest}")
@ -921,8 +999,35 @@ if(BUILD_TESTING)
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endforeach()
add_test(NAME agc_driver_f64_denorm_preserve_spirv COMMAND agc_driver_f64_fma_subnormals_tests --verify-spv-denorm-mode)
set_tests_properties(agc_driver_f64_denorm_preserve_spirv PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_driver_f64_denorm_preserve_spirv PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_structurizer_intersection_off COMMAND agc_structurizer_tests)
set_tests_properties(agc_structurizer_intersection_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_DOM_INTERSECTION=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_structurizer_intersection_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_gated_dispatch_capture_short_timeout COMMAND agc_driver_gated_dispatch_capture_tests)
set_tests_properties(agc_driver_gated_dispatch_capture_short_timeout PROPERTIES
ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_GPU_WAIT_TIMEOUT_MS=1;APS5_NO_PIPELINE_LTO=1"
SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_driver_gated_dispatch_capture_short_timeout PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
set_tests_properties(agc_driver_provoking_vertex PROPERTIES TIMEOUT 30)
set_tests_properties(agc_driver_flat_store PROPERTIES RUN_SERIAL TRUE)
set_target_properties(agc_driver_flat_apertures_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
foreach(flatAperturesPart RANGE 5)
add_test(NAME agc_driver_flat_apertures_${flatAperturesPart} COMMAND agc_driver_flat_apertures_tests --part ${flatAperturesPart}/6)
set_tests_properties(agc_driver_flat_apertures_${flatAperturesPart} PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_driver_flat_apertures_${flatAperturesPart} PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endforeach()
set_tests_properties(agc_driver_storage_cache_budget PROPERTIES TIMEOUT 30)
add_test(NAME agc_driver_sampled_cache_budget_memory COMMAND agc_driver_sampled_cache_budget_tests memory)
set_tests_properties(agc_driver_sampled_cache_budget_memory PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77)
@ -936,12 +1041,23 @@ if(BUILD_TESTING)
set_tests_properties(agc_driver_pm4_failure PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
if(NOT WIN32)
add_test(NAME agc_driver_pm4_priority_refused COMMAND agc_driver_pm4_tests priority-refused)
set_tests_properties(agc_driver_pm4_priority_refused PROPERTIES ENVIRONMENT
"ANYPS5_NO_SHADER_CACHE=1;APS5_THREAD_PRIORITY=high;DBUS_SYSTEM_BUS_ADDRESS=unix:path=/nonexistent/aps5-no-rtkit" SKIP_RETURN_CODE 77)
endif()
add_test(NAME agc_driver_flip_compute COMMAND agc_driver_flip_tests compute)
set_tests_properties(agc_driver_flip_compute PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77)
if(WIN32)
set_tests_properties(agc_driver_flip_compute PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_driver_draw_cache_reuse_off COMMAND agc_driver_draw_cache_reuse_tests)
set_tests_properties(agc_driver_draw_cache_reuse_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_GPL=1" SKIP_RETURN_CODE 77)
if(WIN32)
set_tests_properties(agc_driver_draw_cache_reuse_off PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
foreach(agcTest IN ITEMS agc_spirv_narrow_constant_store_tests agc_spirv_optimizer_tests agc_driver_constant_store_tests)
if(NOT TARGET ${agcTest})
@ -994,6 +1110,11 @@ if(BUILD_TESTING)
set_tests_properties(${preparedTest} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endforeach()
add_test(NAME agc_driver_compute_user_data_calls_registered COMMAND agc_driver_compute_user_data_calls_tests --registered)
set_tests_properties(agc_driver_compute_user_data_calls_registered PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_driver_compute_user_data_calls_registered PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
set_tests_properties(agc_driver_prepared_shaders
agc_driver_prepared_shaders_indirect agc_driver_prepared_shaders_uncached
agc_driver_prepared_shaders_debug agc_driver_prepared_shaders_debug_failure
@ -1040,8 +1161,8 @@ if(AGC_BUILD_VISUAL_TEST)
${CMAKE_SOURCE_DIR}/core/shader/recompiler/IntermediateRepresentation/include
${CMAKE_SOURCE_DIR}/core/shader/recompiler/ControlFlow/include
${CMAKE_SOURCE_DIR}/core/shader/recompiler/RdnaDecoder/include
${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include
${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}
${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR}
)
target_link_libraries(agc_driver_visual_test PRIVATE
@ -1103,7 +1224,7 @@ configure_windows_unwind(agc_shader_replay)
if(BUILD_TESTING)
add_test_executable(texture_format_lookup_tests ${LIBS_INCLUDE_DIR}/tests/TextureFormatLookup.cpp)
target_include_directories(texture_format_lookup_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_include_directories(texture_format_lookup_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
target_link_libraries(texture_format_lookup_tests PRIVATE libSceAgcDriver libc)
configure_windows_unwind(texture_format_lookup_tests)
add_test(NAME texture_format_lookup COMMAND texture_format_lookup_tests)
@ -1112,7 +1233,7 @@ if(BUILD_TESTING)
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test_executable(linear_display_buffer_tests ${LIBS_INCLUDE_DIR}/tests/LinearDisplayBuffer.cpp)
target_include_directories(linear_display_buffer_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_include_directories(linear_display_buffer_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR})
target_link_libraries(linear_display_buffer_tests PRIVATE libSceAgcDriver libc)
configure_windows_unwind(linear_display_buffer_tests)
add_test(NAME linear_display_buffer COMMAND linear_display_buffer_tests)
@ -1250,6 +1371,14 @@ if(BUILD_TESTING)
set_tests_properties(scalar_scratch_load PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test_executable(aperture_sources_tests ${LIBS_INCLUDE_DIR}/tests/ApertureSources.cpp)
target_link_libraries(aperture_sources_tests PRIVATE shader_recompiler_agc_driver libc)
configure_windows_unwind(aperture_sources_tests)
add_test(NAME aperture_sources COMMAND aperture_sources_tests)
if(WIN32)
set_tests_properties(aperture_sources PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test_executable(scalar_call_tests ${LIBS_INCLUDE_DIR}/tests/ScalarCall.cpp)
target_link_libraries(scalar_call_tests PRIVATE shader_recompiler_agc_driver libc)
configure_windows_unwind(scalar_call_tests)
@ -1365,6 +1494,20 @@ if(BUILD_TESTING)
endif()
endif()
add_test_executable(agc_vertex_barrier_tests EXCLUDE_FROM_ALL tests/VertexBarrier.cpp)
target_link_libraries(agc_vertex_barrier_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_vertex_barrier_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_vertex_barrier_tests)
if(BUILD_TESTING)
set_target_properties(agc_vertex_barrier_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_vertex_barrier COMMAND agc_vertex_barrier_tests)
set_tests_properties(agc_vertex_barrier PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1")
if(WIN32)
set_tests_properties(agc_vertex_barrier PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
if(BUILD_TESTING)
add_test_executable(image_by_tests ${LIBS_INCLUDE_DIR}/tests/ImageBy.cpp)
target_link_libraries(image_by_tests PRIVATE shader_recompiler_agc_driver libc)
@ -1392,3 +1535,16 @@ if(BUILD_TESTING)
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_embedded_vertex_fetch_aliasing_tests EXCLUDE_FROM_ALL tests/EmbeddedVertexFetchAliasing.cpp)
target_link_libraries(agc_embedded_vertex_fetch_aliasing_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_embedded_vertex_fetch_aliasing_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_embedded_vertex_fetch_aliasing_tests)
if(BUILD_TESTING)
set_target_properties(agc_embedded_vertex_fetch_aliasing_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_embedded_vertex_fetch_aliasing COMMAND agc_embedded_vertex_fetch_aliasing_tests)
if(WIN32)
set_tests_properties(agc_embedded_vertex_fetch_aliasing PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()

1
core/libs/prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp
View File

@ -25,6 +25,7 @@ struct DisplayBuffer {
};
std::size_t DisplayBufferSize(const DisplayBuffer& buffer);
std::size_t DisplayBufferOffset(const DisplayBuffer& buffer, std::uint32_t x, std::uint32_t y);
std::vector<std::byte> DecodeDisplayBuffer(const DisplayBuffer& buffer, std::span<const std::byte> source);
std::vector<std::byte> ReadDisplayBuffer(const DisplayBuffer& buffer);
std::array<std::byte, 4> DisplayBufferClearPixel(const DisplayBuffer& buffer, Graphics::DccKeys keys);

2
core/libs/prx/libSceAgcDriver/Execution/include/DisplayFormat.hpp
View File

@ -10,6 +10,8 @@ namespace AgcDriver {
VkFormat DisplayTexelFormat(std::uint64_t pixelFormat);
bool DisplayTenBit(std::uint64_t pixelFormat);
bool DisplayRedLow(std::uint64_t pixelFormat);
bool DisplayHalfFloat(std::uint64_t pixelFormat);
std::uint32_t DisplayTexelBytes(std::uint64_t pixelFormat);
enum class ResidentPresent : std::uint8_t { None, Blit, Convert };
ResidentPresent ResidentPresentPath(VkFormat storage, std::uint64_t pixelFormat, bool blitSource);

1
core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp
View File

@ -12,6 +12,7 @@ void Submit(const Packet* packet, std::uint32_t queue);
}
extern "C" void AgcDriverWaitIdle_nid_postfix();
extern "C" void AgcDriverRaiseWorkerThreadPriority_nid_postfix(const char* role);
extern "C" void AgcDriverLockVulkanLoader_nid_postfix();
extern "C" void AgcDriverUnlockVulkanLoader_nid_postfix();
extern "C" void AgcDriverShutdown_nid_postfix();

2
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp
View File

@ -81,6 +81,8 @@ struct StageCapture {
std::uint32_t pushOffset = 0;
};
bool DecodeProgramsCurrent(const DrawDecode& decode, const ShaderRegistry& registry);
}
#endif

2
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp
View File

@ -63,6 +63,7 @@ public:
static void FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters);
static std::optional<Graphics::IndirectDrawPath> ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr<VulkanDevice>& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect);
static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request);
DrawEntryCounters DrawCacheCounters();
private:
friend class SampledReadScope;
@ -113,6 +114,7 @@ private:
static std::size_t drawCacheEntries();
void accountDrawVariant(const DispatchVariant& variant, bool added);
void insertDrawEntry(std::uint64_t key, std::vector<std::shared_ptr<DispatchVariant>>& fresh, std::shared_ptr<const DrawDecode> decode);
void eraseDrawEntry(std::uint64_t key, const std::shared_ptr<DrawEntry>& expected);
std::shared_ptr<const DrawRecipe> findDrawRecipe(std::uint64_t key, const std::vector<std::shared_ptr<DispatchVariant>>& stages);
void attachDrawRecipe(std::uint64_t key, const std::vector<std::shared_ptr<DispatchVariant>>& stages, std::shared_ptr<const DrawRecipe> recipe);
void reportDrawCache(DrawEntryCounters& counters);

14
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp
View File

@ -10,6 +10,8 @@
#include <cstddef>
#include <cstdint>
#include <exception>
#include <functional>
#include <initializer_list>
#include <map>
#include <memory>
#include <mutex>
@ -27,7 +29,7 @@ class VulkanDevice;
namespace AgcDriver::DriverDetail {
struct ShaderSnapshot;
using ShaderRegistry = std::map<std::uint64_t, std::shared_ptr<const ShaderSnapshot>>;
using ShaderRegistry = std::map<std::uint64_t, std::vector<std::shared_ptr<const ShaderSnapshot>>>;
struct PreparedShaderState {
struct Entry {
@ -79,17 +81,25 @@ struct ShaderSnapshot {
std::vector<std::byte> header;
std::shared_ptr<PreparedShaders> prepared = std::make_shared<PreparedShaders>();
std::shared_ptr<const RegisteredShaderState> registeredState;
std::shared_ptr<std::atomic<std::uint8_t>> workgroupUse = std::make_shared<std::atomic<std::uint8_t>>(0u);
};
std::shared_ptr<const ShaderSnapshot> ReadRawComputeShader(std::uint64_t address);
std::shared_ptr<const ShaderSnapshot> ReadRawShader(std::uint64_t address);
std::shared_ptr<const ShaderSnapshot> ProgramSnapshot(const ShaderRegistry& shaders, std::uint64_t address, std::initializer_list<std::uint8_t> types);
void NoteRawGraphicsDraw();
bool RawGraphicsDrawn();
bool ProgramUsesWorkgroup(const ShaderSnapshot& snapshot, std::size_t codeOffset);
std::shared_ptr<const ShaderRecompiler::SourceHandle> PrepareShaderWithDiagnostics(const ShaderRecompiler::RecompileRequest& request, ShaderRecompiler::ShaderPreparationContext* preparation = nullptr);
std::uint64_t NullPixelProgramAddress();
ShaderSnapshot PrepareNullPixelProgram(const VulkanDevice& device);
std::optional<ShaderRecompiler::ShaderFloatMode> RegisteredFloatMode(const ShaderSnapshot& snapshot);
std::shared_ptr<const ShaderSnapshot> RegisteredProgram(const ShaderRegistry& registry, std::uint64_t address, std::initializer_list<std::uint8_t> types);
void PublishRegisteredShader(std::shared_ptr<ShaderRegistry>& registry, const std::shared_ptr<const ShaderSnapshot>& snapshot);
void SubmitRegistrationPreparation(std::function<void()> task);
void CloseRegistrationPreparation();
void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target);

2
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp
View File

@ -30,7 +30,7 @@ enum TrySite { TryLabel = 0, TryFlush, TryReap, TryPoll, TryIdle, TryCapture, Tr
extern std::atomic<std::uint64_t> triesFailed[TrySites];
extern std::atomic<std::uint64_t> storesOnGpu, storesBehindCompletions, storesOnCpu, storesDrained;
extern std::atomic<std::uint64_t> storesOnGpu, storesBehindCompletions, storesOnCpu, storesDrained, copiesToHostMemory;
extern std::map<std::uint32_t, std::uint64_t> drainCounts;

3
core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp
View File

@ -140,12 +140,15 @@ struct MemoryCopy {
std::size_t bytes;
};
std::optional<MemoryCopy> DecodeMemoryCopy(std::span<const std::uint32_t> packet);
std::uint32_t DmaSource(std::span<const std::uint32_t> packet);
std::uint32_t DmaDestination(std::span<const std::uint32_t> packet);
// A DISPATCH_INDIRECT's arguments: the guest address of its three group-count dwords (no memory
// access, so the GPU can read them in place: VulkanDevice::DispatchIndirect), the DISPATCH_DIRECT
// packet made by reading them there (through the checked guest memory path, which waits for
// recorded GPU work that writes them), and both steps in one for the packet.
std::uint64_t DispatchArgumentAddress(std::span<const std::uint32_t> packet, const QueueState& queue);
std::array<std::uint32_t, 5> ReadDispatchArguments(std::uint64_t arguments, std::uint32_t initiator);
std::array<std::uint32_t, 3> PartialGroupThreads(const std::array<std::uint32_t, 3>& groups, const std::array<std::uint32_t, 3>& numThreads);
std::array<std::uint32_t, 5> ResolveDispatch(std::span<const std::uint32_t> packet, const QueueState& queue);
DrawParameters ResolveDraw(std::span<const std::uint32_t> packet, const QueueState& queue);

7
core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp
View File

@ -139,7 +139,7 @@ private:
std::size_t entries = 0;
};
inline Registers InitialContextRegisters() {
inline Registers BuildInitialContextRegisters() {
Registers result{
{0x200, 0}, {0x201, 0}, {0x202, 0xcc0010}, {0x203, 0},
{0x204, 0}, {0x205, 0}, {0x206, 1087}, {0x207, 0},
@ -171,6 +171,11 @@ inline Registers InitialContextRegisters() {
return result;
}
inline Registers InitialContextRegisters() {
static const Registers& initial = *new Registers(BuildInitialContextRegisters());
return initial;
}
struct Predication {
std::uint64_t address = 0;
std::uint32_t operation = 0;

34
core/libs/prx/libSceAgcDriver/Execution/include/ThreadPriority.hpp Normal file
View File

@ -0,0 +1,34 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_THREADPRIORITY_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_THREADPRIORITY_HPP
#include <cstdint>
#include <optional>
#include <string>
namespace AgcDriver {
enum class ThreadPriorityMode { Off, High, Realtime };
struct RttimeLimit {
std::uint64_t soft = 0;
std::uint64_t hard = 0;
bool operator==(const RttimeLimit&) const = default;
};
struct RtkitLimits {
std::int64_t maxPriority = 0;
std::int64_t maxRttimeUs = 0;
};
struct RealtimePlan {
std::string refusal;
std::optional<RttimeLimit> limit;
};
ThreadPriorityMode ParseThreadPriorityMode(const char* text);
RealtimePlan PlanRealtime(std::uint32_t priority, const std::optional<RtkitLimits>& rtkit, RttimeLimit current, std::uint64_t requestedUs);
void RaiseWorkerThreadPriority(const char* role);
}
#endif

5
core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp
View File

@ -9,6 +9,7 @@
#include "prx/libSceAgcDriver/Execution/include/Recipe.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
#include "prx/libc/include/GuestAllocations.hpp"
#include <functional>
#include <memory>
#include <optional>
@ -47,6 +48,7 @@ public:
// three-step form below so the GPU wait happens without the mutex.
void WaitIdle();
void PrepareForReplacement();
bool ImportGuestMemory(const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice);
// Sends recorded work to the GPU without waiting for it. With `reapFirst` it first retires batches
// that already finished, so the in-flight list stays short (APS5_NO_OPPORTUNISTIC_REAP=1 skips
// that). A reap runs completion actions, and a write-back can wait for a later batch under the
@ -78,6 +80,7 @@ public:
std::optional<Graphics::Recorder::LabelHit> PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, Graphics::Recorder::LabelRefusal* refusal = nullptr) const;
bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const;
bool RecordedWritesSettled(std::uint64_t address, std::size_t bytes) const;
bool StoresPendingOver(std::uint64_t address, std::size_t bytes) const;
// Fills [address, address + bytes) of host-imported guest memory with a repeating 16-byte pattern,
// recorded behind the open batch; false when the range is not imported (the caller stores it).
// Recorded GPU stores over the range are ordered before the fill by its barrier; finished
@ -220,7 +223,7 @@ public:
// records. Null when nothing is prepared: the resource cache may serve the dispatch (its
// Revalidate stays under the mutex), or APS5_LOCKED_BUILD=1 keeps the whole build under it as
// before. `shader` and `snapshots` must outlive the dispatch.
std::shared_ptr<PreparedDispatch> PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span<const Graphics::GuestMemorySnapshot> snapshots);
std::shared_ptr<PreparedDispatch> PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t dispatchThreads = 0);
// APS5_PROFILE_DRAW: the parts of a PrepareDispatch in milliseconds, in the order key, find,
// precollect, presync, stage A (the driver's 'prepare:' rows).
static std::span<const double, 5> PreparePhaseMs(const PreparedDispatch& prepared);

50
core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp
View File

@ -3,6 +3,7 @@
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
#include "prx/libSceAgcDriver/Graphics/include/HalfFloatScanout.hpp"
#include <algorithm>
#include <array>
#include <chrono>
@ -33,6 +34,30 @@ std::uint32_t tileOffset(std::uint32_t x, std::uint32_t y) {
return ((y << 4u) & 0x0070u) ^ ((y << 5u) & 0x0f00u) ^ ((y << 9u) & 0x1000u) ^ ((y << 8u) & 0x4000u) ^ ((x << 2u) & 0x000cu) ^ ((x << 5u) & 0x0380u) ^ ((x << 4u) & 0x0400u) ^ ((x << 6u) & 0x0800u) ^ ((x << 9u) & 0xa000u);
}
std::uint32_t tileOffset64(std::uint32_t x, std::uint32_t y) {
return ((x << 3u) & 0x0008u) ^ ((x << 4u) & 0x0460u) ^ ((x << 5u) & 0x0300u) ^ ((x << 6u) & 0x0800u) ^ ((x << 9u) & 0x8000u) ^ ((x << 10u) & 0x2000u) ^ ((y << 4u) & 0x0010u) ^ ((y << 6u) & 0x0080u) ^ ((y << 5u) & 0x0f00u) ^ ((y << 10u) & 0x5000u);
}
std::size_t texelOffset(const DisplayBuffer& buffer, std::uint32_t x, std::uint32_t y, std::uint32_t texelBytes) {
if (buffer.tilingMode == 1) {
const auto pitch = buffer.pitchInPixel == 0 ? buffer.width : buffer.pitchInPixel;
return (static_cast<std::size_t>(y) * pitch + x) * texelBytes;
}
const auto blocksPerRow = (buffer.width + 127u) / 128u;
const auto blockHeight = 65536u / 128u / texelBytes;
const auto block = static_cast<std::size_t>(y / blockHeight) * blocksPerRow + x / 128u;
return block * 65536u + (texelBytes == 8u ? tileOffset64(x, y) : tileOffset(x, y));
}
void decodeHalfFloatTexel(const std::byte* texel, std::byte* pixel, bool redLow) {
std::array<std::uint16_t, 4> halves{};
std::memcpy(halves.data(), texel, sizeof(halves));
pixel[0] = static_cast<std::byte>(Graphics::HalfFloatScanoutCode(halves[redLow ? 2 : 0], false));
pixel[1] = static_cast<std::byte>(Graphics::HalfFloatScanoutCode(halves[1], false));
pixel[2] = static_cast<std::byte>(Graphics::HalfFloatScanoutCode(halves[redLow ? 0 : 2], false));
pixel[3] = static_cast<std::byte>(Graphics::HalfFloatScanoutCode(halves[3], true));
}
void decodeTexel(const std::byte* texel, std::byte* pixel, bool rgba, bool tenBit) {
if (tenBit) {
std::uint32_t word = 0;
@ -90,11 +115,13 @@ void traceDisplayKeys(const DisplayBuffer& buffer, const std::uint8_t* keys, std
std::size_t DisplayBufferSize(const DisplayBuffer& buffer) {
require(buffer.width != 0 && buffer.height != 0 && buffer.width <= 16384 && buffer.height <= 16384, "VideoOut: invalid display buffer dimensions");
if (baseFormat(buffer.pixelFormat) != PixelFormatB8G8R8A8 && baseFormat(buffer.pixelFormat) != PixelFormatR8G8B8A8) {
const bool halfFloat = DisplayHalfFloat(buffer.pixelFormat);
if (!halfFloat && baseFormat(buffer.pixelFormat) != PixelFormatB8G8R8A8 && baseFormat(buffer.pixelFormat) != PixelFormatR8G8B8A8) {
char message[96];
std::snprintf(message, sizeof(message), "VideoOut: unsupported display pixel format 0x%016llx", static_cast<unsigned long long>(buffer.pixelFormat));
throw std::runtime_error(message);
}
require(!halfFloat || buffer.dccAddress == 0, "VideoOut: DCC metadata on a 16-bit float display buffer is not implemented");
require(buffer.address != 0 && (buffer.address & 65535u) == 0, "VideoOut: display buffer requires 64 KiB alignment");
require(buffer.tilingMode <= 1, "VideoOut: unsupported display tiling mode");
require(buffer.tilingMode != 0 || buffer.pitchInPixel == 0, "VideoOut: tiled display pitch is unsupported");
@ -102,16 +129,33 @@ std::size_t DisplayBufferSize(const DisplayBuffer& buffer) {
require(buffer.dccAddress != 0 || buffer.dccClearColor == 0, "VideoOut: a DCC clear color needs DCC metadata");
const auto pitch = buffer.pitchInPixel == 0 ? buffer.width : buffer.pitchInPixel;
require(pitch >= buffer.width && pitch <= 16384, "VideoOut: invalid linear display pitch");
const auto size = buffer.tilingMode == 1 ? static_cast<std::uint64_t>(pitch) * buffer.height * 4u
: static_cast<std::uint64_t>((buffer.width + 127u) / 128u) * ((buffer.height + 127u) / 128u) * 65536u;
const auto texelBytes = DisplayTexelBytes(buffer.pixelFormat);
const auto blockHeight = 65536u / 128u / texelBytes;
const auto size = buffer.tilingMode == 1 ? static_cast<std::uint64_t>(pitch) * buffer.height * texelBytes
: static_cast<std::uint64_t>((buffer.width + 127u) / 128u) * ((buffer.height + blockHeight - 1u) / blockHeight) * 65536u;
require(size <= std::numeric_limits<std::size_t>::max() && size <= std::numeric_limits<std::uintptr_t>::max() - buffer.address, "VideoOut: display buffer range overflow");
return static_cast<std::size_t>(size);
}
std::size_t DisplayBufferOffset(const DisplayBuffer& buffer, std::uint32_t x, std::uint32_t y) {
static_cast<void>(DisplayBufferSize(buffer));
require(x < buffer.width && y < buffer.height, "VideoOut: display buffer coordinate out of range");
return texelOffset(buffer, x, y, DisplayTexelBytes(buffer.pixelFormat));
}
std::vector<std::byte> DecodeDisplayBuffer(const DisplayBuffer& buffer, std::span<const std::byte> source) {
PerformanceTimer timing("DisplayBuffer.Decode");
require(source.size() == DisplayBufferSize(buffer), "VideoOut: invalid display buffer size");
std::vector<std::byte> pixels(static_cast<std::size_t>(buffer.width) * buffer.height * 4);
if (DisplayHalfFloat(buffer.pixelFormat)) {
const bool redLow = DisplayRedLow(buffer.pixelFormat);
timing.Mark("validate_allocate");
for (std::uint32_t y = 0; y < buffer.height; ++y) {
for (std::uint32_t x = 0; x < buffer.width; ++x) decodeHalfFloatTexel(source.data() + texelOffset(buffer, x, y, 8u), pixels.data() + (static_cast<std::size_t>(y) * buffer.width + x) * 4, redLow);
}
timing.Mark("detile_convert");
return pixels;
}
const auto blocksPerRow = (buffer.width + 127u) / 128u;
const bool rgba = baseFormat(buffer.pixelFormat) == PixelFormatR8G8B8A8;
// Bit 56 marks the 10-bit variant: A2B10G10R10 with red in the low bits (the scanout draws render

14
core/libs/prx/libSceAgcDriver/Execution/src/DisplayFormat.cpp
View File

@ -5,6 +5,8 @@ namespace {
constexpr std::uint64_t PixelFormatUnormBit = 0x0100000000000000ull;
constexpr std::uint64_t PixelFormatR8G8B8A8 = 0x8000000022000000ull;
constexpr std::uint64_t PixelFormatR16G16B16A16Float = 0xc001000622000000ull;
constexpr std::uint64_t PixelFormatB16G16R16A16Float = 0xc001000600000000ull;
}
@ -13,16 +15,26 @@ bool DisplayTenBit(std::uint64_t pixelFormat) {
}
bool DisplayRedLow(std::uint64_t pixelFormat) {
return (pixelFormat & ~PixelFormatUnormBit) == PixelFormatR8G8B8A8;
return (pixelFormat & ~PixelFormatUnormBit) == PixelFormatR8G8B8A8 || pixelFormat == PixelFormatR16G16B16A16Float;
}
bool DisplayHalfFloat(std::uint64_t pixelFormat) {
return pixelFormat == PixelFormatR16G16B16A16Float || pixelFormat == PixelFormatB16G16R16A16Float;
}
std::uint32_t DisplayTexelBytes(std::uint64_t pixelFormat) {
return DisplayHalfFloat(pixelFormat) ? 8u : 4u;
}
VkFormat DisplayTexelFormat(std::uint64_t pixelFormat) {
if (DisplayHalfFloat(pixelFormat)) return VK_FORMAT_R16G16B16A16_SFLOAT;
const bool redLow = DisplayRedLow(pixelFormat);
if (DisplayTenBit(pixelFormat)) return redLow ? VK_FORMAT_A2B10G10R10_UNORM_PACK32 : VK_FORMAT_A2R10G10B10_UNORM_PACK32;
return redLow ? VK_FORMAT_R8G8B8A8_UNORM : VK_FORMAT_B8G8R8A8_UNORM;
}
ResidentPresent ResidentPresentPath(VkFormat storage, std::uint64_t pixelFormat, bool blitSource) {
if (DisplayHalfFloat(pixelFormat)) return storage == VK_FORMAT_R16G16B16A16_SFLOAT ? ResidentPresent::Convert : ResidentPresent::None;
const bool tenBit = DisplayTenBit(pixelFormat);
if (!tenBit && blitSource && storage == DisplayTexelFormat(pixelFormat)) return ResidentPresent::Blit;
switch (storage) {

26
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp
View File

@ -16,13 +16,8 @@ namespace AgcDriver::DriverDetail {
void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission, std::uint64_t indirectArguments) {
PerformanceTimer timing("Driver.Dispatch");
const auto address = (static_cast<std::uint64_t>(readRegister(queue.shader, 0x20c)) << 8u) | (static_cast<std::uint64_t>(readRegister(queue.shader, 0x20d) & 0xffu) << 40u);
auto it = submission.shaders->upper_bound(address);
std::shared_ptr<const ShaderSnapshot> registeredShader;
if (it != submission.shaders->begin()) {
--it;
if (address - it->second->codeAddress < it->second->code.size() * sizeof(std::uint32_t)) registeredShader = it->second;
}
if (!registeredShader) registeredShader = ReadRawComputeShader(address);
auto registeredShader = RegisteredProgram(*submission.shaders, address, {0});
if (!registeredShader) registeredShader = ReadRawShader(address);
const auto& snapshot = *registeredShader;
require(snapshot.type == 0, "compute program refers to a non-compute shader");
const auto userCount = (readRegister(queue.shader, 0x213) >> 1u) & 0x1fu;
@ -70,16 +65,21 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
packet = resolved;
indirectArguments = 0;
}
std::array<std::uint32_t, 3> partialGroups{};
if (indirectArguments == 0 && (packet[4] & 0x2u) != 0) {
partialGroups = Pm4::PartialGroupThreads({packet[1], packet[2], packet[3]}, {readRegister(queue.shader, 0x207), readRegister(queue.shader, 0x208), readRegister(queue.shader, 0x209)});
}
const bool partialLastGroup = partialGroups != std::array<std::uint32_t, 3>{};
if (resolving) {
if (matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return;
if (indirectArguments == 0 && matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return;
if (!partialLastGroup && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return;
if (!partialLastGroup && indirectArguments == 0 && matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return;
} else {
if (const auto htile = Graphics::HtileDepthClearAddress(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads); htile != 0) Graphics::NoteHtileDepthClear(htile);
if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) {
if (!partialLastGroup && fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) {
pendingDispatchPhases().outcome = DispatchOutcome::FillHle;
return;
}
if (indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) {
if (!partialLastGroup && indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) {
pendingDispatchPhases().outcome = DispatchOutcome::CopyHle;
return;
}
@ -90,6 +90,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
if (threads[axis] % compute.numThreads[axis] != 0) compute.partialThreads = threads;
}
}
if (partialLastGroup) compute.partialThreads = partialGroups;
ShaderRecompiler::RecompileRequest request{
{ShaderRecompiler::ShaderStage::Compute, address, std::span(snapshot.code).subspan(codeOffset), snapshot.headerAddress, snapshot.header},
{(packet[4] & 0x8000u) != 0 ? 32u : 64u, 0, userData, compute, std::nullopt, std::nullopt, memory, RegisteredFloatMode(snapshot)},
@ -321,6 +322,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
groups[axis] = (groups[axis] + threads - 1) / threads;
}
}
const std::uint64_t dispatchThreads = indirectArguments != 0 ? 0 : std::uint64_t{groups[0]} * groups[1] * groups[2] * compute.numThreads[0] * compute.numThreads[1] * compute.numThreads[2];
static const bool traceIo = std::getenv("APS5_TRACE_DISPATCH_IO") != nullptr;
if (traceIo) {
@ -363,7 +365,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
std::shared_ptr<PreparedDispatch> prepared;
if (recipeHit == nullptr || VulkanDevice::VerifyRecipes()) {
try {
prepared = localDevice->PrepareDispatch(compiled, snapshots);
prepared = localDevice->PrepareDispatch(compiled, snapshots, dispatchThreads);
} catch (const std::exception& error) {
if (rethrow(error)) return;
}

23
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp
View File

@ -82,6 +82,10 @@ void NoteLabel(std::vector<RunLabel>& labels, std::uint64_t address, std::span<c
labels.push_back({address, bytes.size(), value});
}
bool WritesThroughPointers(const ShaderRecompiler::RecompileResult& compiled) {
return std::any_of(compiled.bindings.begin(), compiled.bindings.end(), [](const ShaderRecompiler::DescriptorBinding& binding) { return binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable; });
}
bool OverlapsLabel(std::span<const ShaderRecompiler::MemoryRegion> regions, const std::vector<RunLabel>& labels) {
for (const auto& label : labels) {
for (const auto& region : regions) {
@ -104,6 +108,7 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l
QueueState scratch = live;
std::vector<RunLabel> labels;
std::vector<std::pair<std::uint64_t, std::uint64_t>> writes;
bool unknownWrites = false;
for (const auto& label : deferredLabels().labels) NoteLabel(labels, label.address, std::span<const std::byte>(label.bytes).first(label.size));
struct Resolving {
std::size_t offset;
@ -130,6 +135,21 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l
if (!Pm4::WaitSatisfiedUnchecked(packet) && !WrittenByRun(packet, labels)) break;
} else if (opcode == 0x15 || opcode == 0x16) {
if (resolved.contains(cursor)) break;
if (opcode == 0x16) {
std::uint64_t arguments = 0;
try {
arguments = Pm4::DispatchArgumentAddress(packet, scratch);
} catch (const std::exception&) {
break;
}
const auto argumentsEnd = arguments + 3u * sizeof(std::uint32_t);
const bool written = std::any_of(writes.begin(), writes.end(), [&](const auto& range) { return range.first < argumentsEnd && arguments < range.second; })
|| std::any_of(labels.begin(), labels.end(), [&](const RunLabel& label) { return label.address < argumentsEnd && arguments < label.address + label.size; });
if (unknownWrites || written) {
++stats.writtenBefore;
break;
}
}
currentPacketOffset() = cursor;
try {
if (opcode == 0x15) dispatch(scratch, packet, submission);
@ -150,6 +170,9 @@ void Driver::resolveGroupAhead(const Submission& submission, const QueueState& l
break;
}
if (found->second.compiledResult != nullptr) appendWrittenRanges(*found->second.compiledResult, writes);
if (found->second.compiledResult == nullptr || WritesThroughPointers(*found->second.compiledResult)) unknownWrites = true;
} else {
unknownWrites = true;
}
} else if (opcode == 0x37 || opcode == 0x49) {
const auto label = Pm4::DecodeLabelWrite(packet);

11
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp
View File

@ -81,6 +81,13 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
++drawEntryCounters.absent;
}
}
if (decode != nullptr && !DecodeProgramsCurrent(*decode, *submission.shaders)) {
std::lock_guard cacheLock(drawCacheMutex);
eraseDrawEntry(drawKey, entry);
++drawEntryCounters.absent;
entry = nullptr;
decode = nullptr;
}
phaseTiming.Phase(DrawRowKeyLookupValidate);
resolveDrawDecode(queue, submission, decode, registerKey, drawKey, profile);
@ -123,7 +130,9 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
std::vector<ShaderRecompiler::LinkedProgram> linked;
for (std::size_t i = 0; i < programs.size(); ++i) {
const auto& program = programs[i];
memory.insert(memory.end(), program.memory.begin(), program.memory.end());
for (const auto& region : program.memory) {
if (!region.bytes.empty()) memory.push_back(region);
}
linked.push_back({roles[i], program.binary, program.userDataBase, program.firstUserSgpr, program.userData});
}
timing.Mark("prepare");

17
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp
View File

@ -1,5 +1,6 @@
#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp"
#include <algorithm>
#include <cstdlib>
namespace AgcDriver::DriverDetail {
@ -107,6 +108,22 @@ void Driver::insertDrawEntry(std::uint64_t key, std::vector<std::shared_ptr<Disp
}
}
void Driver::eraseDrawEntry(std::uint64_t key, const std::shared_ptr<DrawEntry>& expected) {
const auto found = drawCache.find(key);
if (found == drawCache.end() || found->second != expected) return;
for (const auto& variants : found->second->stages) {
for (const auto& variant : variants) accountDrawVariant(*variant, false);
}
drawOrder.erase(found->second->order);
drawCache.erase(found);
}
bool DecodeProgramsCurrent(const DrawDecode& decode, const ShaderRegistry&) {
return std::all_of(decode.programs.begin(), decode.programs.end(), [&](const DrawProgram& program) {
return !program.snapshot->header.empty() || ReadRawShader(program.snapshot->codeAddress + program.codeOffset * sizeof(std::uint32_t)) == program.snapshot;
});
}
std::shared_ptr<const DrawRecipe> Driver::findDrawRecipe(std::uint64_t key, const std::vector<std::shared_ptr<DispatchVariant>>& stages) {
std::shared_ptr<const std::vector<DrawRecipeRecord>> records;
{

5
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCacheStatistics.cpp
View File

@ -18,4 +18,9 @@ void Driver::reportDrawCache(DrawEntryCounters& counters) {
counters = DrawEntryCounters{};
}
DrawEntryCounters Driver::DrawCacheCounters() {
std::lock_guard cacheLock(drawCacheMutex);
return drawEntryCounters;
}
}

30
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp
View File

@ -28,13 +28,11 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const
const auto prepare = [&](std::uint64_t address, std::uint8_t type, Stage stage, std::uint32_t rsrc2, std::uint32_t userDataBase) {
const bool nullPixel = stage == Stage::Fragment && (address == 0 || pixelSkipped);
if (nullPixel) address = NullPixelProgramAddress();
auto it = registry.upper_bound(address);
require(it != registry.begin(), "graphics program does not belong to a registered shader");
--it;
const auto& snapshot = *it->second;
require(address - snapshot.codeAddress < snapshot.code.size() * sizeof(std::uint32_t), "graphics program is outside registered shader code");
const auto program = ProgramSnapshot(registry, address, {type});
const auto& snapshot = *program;
require((address - snapshot.codeAddress) % sizeof(std::uint32_t) == 0, "graphics entry point is not dword aligned");
require(snapshot.type == type, "graphics program refers to an incompatible shader binary type");
require(snapshot.header.empty() || snapshot.type == type, "graphics program refers to an incompatible shader binary type");
if (!nullPixel && snapshot.header.empty()) NoteRawGraphicsDraw();
if (!nullPixel) Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, rsrc2);
const auto resources = nullPixel ? 0u : ReadGraphicsRegister(queue.shader, rsrc2);
const auto userCount = ((resources >> 1u) & 0x1fu) | (((resources >> 27u) & 1u) << 5u);
@ -46,7 +44,7 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const
8,
{},
{{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}},
it->second,
program,
codeOffset
};
for (std::uint32_t i = 0; i < userCount; ++i) {
@ -89,10 +87,8 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const
append(0x0c8, 2, Stage::TessellationEvaluation, 0x08b, 0x08c, Role::Domain);
} else if (graphics.stages.path == Graphics::ShaderPath::Geometry) {
const auto frontAddress = programAddress(0xc8);
auto snapshot = registry.upper_bound(frontAddress);
require(snapshot != registry.begin(), "geometry front program is not registered");
--snapshot;
const auto type = snapshot->second->type;
const auto front = ProgramSnapshot(registry, frontAddress, {2, 4});
const auto type = front->header.empty() ? std::uint8_t{2} : front->type;
require(type == 2 || type == 4, "invalid geometry front binary type");
append(0xc8, type, Stage::Mesh, 0x8b, 0x8c, Role::Main);
initializeMerged(programs.back(), 0x82, type == 4);
@ -113,11 +109,21 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const
std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Submission& submission) {
auto product = std::make_shared<DrawDecode>();
product->state = Graphics::DecodeState(queue);
bool vertexWorkgroup = false;
const auto stages = queue.context.find(0x2d5);
const auto low = queue.shader.find(0xc8);
const auto high = queue.shader.find(0xc9);
if (stages != queue.context.end() && (stages->second & 0x02002024u) == 0x2000u && low != queue.shader.end() && high != queue.shader.end()) {
const auto address = (static_cast<std::uint64_t>(low->second) << 8u) | (static_cast<std::uint64_t>(high->second & 0xffu) << 40u);
const auto front = ProgramSnapshot(*submission.shaders, address, {2});
vertexWorkgroup = front->header.empty() && ProgramUsesWorkgroup(*front, static_cast<std::size_t>((address - front->codeAddress) / sizeof(std::uint32_t)));
}
product->state = Graphics::DecodeState(queue, vertexWorkgroup);
DecodeGraphicsPrograms(*product, queue, *submission.shaders, false, true);
product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(product->state), Graphics::PixelProgramSkipped(queue));
product->pixel.targetExportPacking = Graphics::ExportPackings(product->state);
product->pixel.dualSourceBlend = product->state.dualSourceBlend;
product->pixel.alphaToCoverage = product->state.alphaToCoverage;
return product;
}

2
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp
View File

@ -68,6 +68,7 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr<Vulk
for (const auto value : pixel.targetExportMapping) mix(value);
for (const auto value : pixel.targetExportPacking) mix(static_cast<std::uint64_t>(value));
mix(pixel.dualSourceBlend);
mix(pixel.alphaToCoverage);
std::lock_guard cacheLock(drawCacheMutex);
++drawEntryCounters.lookups;
const auto found = drawCache.find(drawKey);
@ -89,6 +90,7 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr<Vulk
for (std::size_t i = 0; !miss && i < programs.size(); ++i) {
if (roles[i] == Role::GeometryBack) continue;
++stageValidations;
cursor = Graphics::StagePushOffset(cursor, programs[i].binary.stage, localDevice->GraphicsPipelineLibraries());
const auto& variants = entry->stages[i];
auto outcome = EntryOutcome::Differing;
bool anyLayout = false;

6
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp
View File

@ -50,8 +50,8 @@ std::uint64_t Driver::drawRegisterKey(const QueueState& queue, const ShaderRegis
continue;
}
--it;
mix(reinterpret_cast<std::uintptr_t>(it->second.get()));
mix(address - it->second->codeAddress);
for (const auto& snapshot : it->second) mix(reinterpret_cast<std::uintptr_t>(snapshot.get()));
mix(address - it->first);
}
return key;
}
@ -105,7 +105,7 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) {
const auto& y = b.programs[i];
if (x.binary.stage != y.binary.stage || x.binary.codeAddress != y.binary.codeAddress || x.userDataBase != y.userDataBase || x.firstUserSgpr != y.firstUserSgpr || x.userData != y.userData || x.snapshot != y.snapshot || x.codeOffset != y.codeOffset) return false;
}
if (p.targetExportPacking != q.targetExportPacking || p.dualSourceBlend != q.dualSourceBlend) return false;
if (p.targetExportPacking != q.targetExportPacking || p.dualSourceBlend != q.dualSourceBlend || p.alphaToCoverage != q.alphaToCoverage) return false;
return true;
}

5
core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp
View File

@ -1,3 +1,4 @@
#include "prx/libSceAgcDriver/Execution/include/ThreadPriority.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
#include <mutex>
@ -55,6 +56,10 @@ void ReportFailure(std::exception_ptr error) {
}
extern "C" void AgcDriverRaiseWorkerThreadPriority_nid_postfix(const char* role) {
AgcDriver::RaiseWorkerThreadPriority(role);
}
extern "C" void AgcDriverWaitIdle_nid_postfix() try {
AgcDriver::WaitIdle();
} catch (const ProcessShutdown&) {

15
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp
View File

@ -1,9 +1,12 @@
#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp"
#include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
#include "prx/libc/include/GuestAllocations.hpp"
#include <cstdio>
namespace AgcDriver::DriverDetail {
@ -13,6 +16,7 @@ Driver& Driver::Get() {
}
Driver::~Driver() {
GuestAllocations::GuestAllocationsSetGpuMapObserver_nid_postfix(nullptr);
stop();
}
@ -41,6 +45,7 @@ void Driver::stop() {
if (worker.thread.joinable()) worker.thread.join();
}
StopWorkerSampler();
CloseRegistrationPreparation();
std::lock_guard gpuLock(GuestMemory::GpuMutex());
device.Reset();
replacedDevices.clear();
@ -57,6 +62,16 @@ Driver::Driver() {
stop();
throw;
}
GuestAllocations::GuestAllocationsSetGpuMapObserver_nid_postfix([](const GuestAllocations::Mapped& ranges, std::uint64_t generation) {
try {
const auto current = Driver::Get().device.Load();
if (current == nullptr || current->ImportGuestMemory(ranges, generation, false)) return;
std::lock_guard lock(GuestMemory::GpuMutex());
if (Driver::Get().device.Load() == current) current->ImportGuestMemory(ranges, generation, true);
} catch (const std::exception& error) {
std::fprintf(stderr, "[gpu] import of GPU memory at its mapping failed: %s\n", error.what());
}
});
}
void Driver::WaitIdle() {

2
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp
View File

@ -5,6 +5,7 @@
#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DwordPatternFill.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include <algorithm>
#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
#include <bit>
#include <cstdlib>
#include <optional>
@ -264,6 +265,7 @@ bool Driver::fillBuffer(QueueState& queue, std::uint32_t queueId, std::span<cons
for (std::size_t done = 0; done < bytes; done += chunk) GuestMemory::Write(base + done, std::span<const std::byte>(block).first(std::min(chunk, bytes - done)), 16);
phase(FillCpu);
}
if (pattern[0] == pattern[1] && pattern[1] == pattern[2] && pattern[2] == pattern[3]) Graphics::NoteDepthMetadataFill(base, bytes, pattern[0]);
if (profile) {
const auto now = std::chrono::steady_clock::now();
const auto cover = static_cast<std::size_t>(coverage.cover);

12
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp
View File

@ -5,6 +5,9 @@
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
#include "prx/libSceAgcDriver/Eq/include/Event.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
#include <cstdlib>
#include <cstring>
#include <limits>
@ -56,6 +59,9 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
}
}
}
if (opcode == 0x50 && packet.size() >= 7) {
if (Pm4::DmaSource(packet) == 2 && Pm4::DmaDestination(packet) != 1) Graphics::NoteDepthMetadataFill(packet[4] | (static_cast<std::uint64_t>(packet[5]) << 32u), packet[6] & 0x3ffffffu, packet[2]);
}
if (!drainAll && !endOfPipeInterrupt && (opcode == 0x49 || opcode == 0x37)) {
if (const auto label = Pm4::DecodeLabelWrite(packet)) {
const auto bytes = label->Bytes();
@ -143,6 +149,12 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
if (outcome.path == 1 || outcome.path == 3) {
wroteOnGpu = true;
drained = false;
} else if (outcome.path == 2 && !Graphics::RegisteredReadableCovers(copy->destination, copy->bytes) && !localDevice->StoresPendingOver(copy->destination, copy->bytes)) {
Graphics::StorageTexture::FlushPending(copy->source, copy->bytes, nullptr, "buffer copy source", Graphics::PublishScope::Whole);
if (auto* recorder = Graphics::Recorder::Active()) recorder->SyncThrough(copy->source, copy->bytes);
++copiesToHostMemory;
orderedAlready = true;
drained = false;
}
}
}

14
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp
View File

@ -5,7 +5,9 @@
#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
#include "prx/libSceAgcDriver/Execution/include/ThreadPriority.hpp"
#include <bit>
#include <cstdio>
#include <cstdlib>
namespace AgcDriver::DriverDetail {
@ -347,7 +349,17 @@ void Driver::enqueue(Submission submission) {
for (const auto dword : submission.labelWrites) ++worker.unfinishedWrites[dword];
worker.pending.push_back(std::move(submission));
worker.queued.fetch_add(1, std::memory_order_acq_rel);
if (!worker.thread.joinable()) worker.thread = std::thread([this, queue] { run(queue); });
if (!worker.thread.joinable()) worker.thread = std::thread([this, queue] {
char role[32];
std::snprintf(role, sizeof(role), "queue 0x%x worker", queue);
try {
RaiseWorkerThreadPriority(role);
} catch (...) {
ReportFailure(std::current_exception());
return;
}
run(queue);
});
}
void Driver::noteHeldAtSubmit(Submission& submission, std::size_t cursor) {

274
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp
View File

@ -11,9 +11,11 @@
#include "CompiledVariant.hpp"
#include "Optimization/ResourceProgram.hpp"
#include <algorithm>
#include <atomic>
#include <condition_variable>
#include <cstdio>
#include <cstddef>
#include <span>
#include <cstdlib>
#include <cstring>
#include <deque>
@ -37,7 +39,7 @@ std::shared_ptr<const ShaderRecompiler::SourceHandle> PrepareShaderWithDiagnosti
}
}
std::shared_ptr<const ShaderSnapshot> ReadRawComputeShader(std::uint64_t address) {
std::shared_ptr<const ShaderSnapshot> ReadRawShader(std::uint64_t address) {
GuestMemory::CheckRange(reinterpret_cast<const void*>(address), sizeof(std::uint32_t), 256);
static std::mutex cacheMutex;
static std::list<std::shared_ptr<const ShaderSnapshot>> cache;
@ -100,7 +102,25 @@ std::shared_ptr<const ShaderSnapshot> ReadRawComputeShader(std::uint64_t address
if (snapshot.code.size() == available) break;
}
}
throw std::runtime_error("AGC driver: raw compute program has no reachable end within mapped code or the size limit");
throw std::runtime_error("AGC driver: raw shader program has no reachable end within mapped code or the size limit");
}
std::shared_ptr<const ShaderSnapshot> ProgramSnapshot(const ShaderRegistry& shaders, std::uint64_t address, std::initializer_list<std::uint8_t> types) {
if (auto registered = RegisteredProgram(shaders, address, types)) return registered;
require(shaders.empty(), "graphics program does not belong to a compatible registered shader");
return ReadRawShader(address);
}
namespace {
std::atomic<bool> rawGraphicsDrawn{false};
}
void NoteRawGraphicsDraw() {
rawGraphicsDrawn.store(true, std::memory_order_relaxed);
}
bool RawGraphicsDrawn() {
return rawGraphicsDrawn.load(std::memory_order_relaxed);
}
namespace {
@ -112,6 +132,21 @@ void AwaitRegisteredPreparation(PreparedShaders& prepared, std::unique_lock<std:
}
bool ProgramUsesWorkgroup(const ShaderSnapshot& snapshot, std::size_t codeOffset) {
constexpr std::uint8_t Unknown = 0, Unused = 1, Used = 2;
if (codeOffset == 0) {
const auto cached = snapshot.workgroupUse->load(std::memory_order_acquire);
if (cached != Unknown) return cached == Used;
}
const auto decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(std::span(snapshot.code).subspan(codeOffset));
const bool used = std::any_of(decoded.instructions.begin(), decoded.instructions.end(), [](const auto& instruction) {
return instruction.family == ShaderRecompiler::RdnaInstructionFamily::DS || instruction.op == ShaderRecompiler::RdnaOpcode::SBarrier;
});
if (codeOffset == 0) snapshot.workgroupUse->store(used ? Used : Unused, std::memory_order_release);
return used;
}
struct ShaderPreparationTransaction::State {
struct Change {
std::shared_ptr<PreparedShaders> destination;
@ -207,40 +242,85 @@ bool SameHeader(const ShaderSnapshot& snapshot, const Shader* shader) {
return std::memcmp(&used, &registered, offsetof(Shader, num_sh_registers) + sizeof(Shader::num_sh_registers)) == 0;
}
std::shared_ptr<const ShaderSnapshot> RegisteredHeader(const ShaderRegistry& registry, const Shader* shader) {
const auto found = registry.find(reinterpret_cast<std::uintptr_t>(const_cast<const void*>(shader->code)));
if (found == registry.end()) return {};
for (const auto& snapshot : found->second) {
if (snapshot->headerAddress == reinterpret_cast<std::uintptr_t>(shader)) return snapshot;
}
for (const auto& snapshot : found->second) {
if (SameHeader(*snapshot, shader)) return snapshot;
}
return {};
}
std::shared_ptr<const ShaderSnapshot> RegisteredProgram(const ShaderRegistry& registry, std::uint64_t address, std::initializer_list<std::uint8_t> types) {
auto found = registry.upper_bound(address);
if (found == registry.begin()) return {};
--found;
std::shared_ptr<const ShaderSnapshot> result;
bool registered = false;
for (const auto& snapshot : found->second) {
if (address - snapshot->codeAddress >= snapshot->code.size() * sizeof(std::uint32_t)) continue;
registered = true;
if (std::ranges::find(types, snapshot->type) == types.end()) continue;
require(result == nullptr || result->type == snapshot->type, "registered program has ambiguous shader binary types");
result = snapshot;
}
require(!registered || result != nullptr, "registered program refers to an incompatible shader binary type");
return result;
}
void PublishRegisteredShader(std::shared_ptr<ShaderRegistry>& registry, const std::shared_ptr<const ShaderSnapshot>& snapshot) {
ShaderPreparationTransaction transaction;
auto published = snapshot;
if (registry != nullptr) {
const auto found = registry->find(snapshot->codeAddress);
if (found != registry->end()) {
const auto& current = *found->second;
if (current.headerAddress == snapshot->headerAddress && current.type == snapshot->type && current.code == snapshot->code && current.header == snapshot->header) {
transaction.Commit();
return;
for (const auto& current : found->second) {
if (current->headerAddress == snapshot->headerAddress && current->type == snapshot->type && current->code == snapshot->code && current->header == snapshot->header) {
if (current == found->second.back()) {
transaction.Commit();
return;
}
published = current;
break;
}
}
}
}
auto next = registry != nullptr ? std::make_shared<ShaderRegistry>(*registry) : std::make_shared<ShaderRegistry>();
next->insert_or_assign(snapshot->codeAddress, snapshot);
auto& headers = (*next)[snapshot->codeAddress];
std::erase_if(headers, [&](const auto& current) { return current->headerAddress == snapshot->headerAddress || current->code != snapshot->code; });
headers.push_back(published);
registry = std::move(next);
transaction.Commit();
}
namespace {
bool PreparedAtUse(const ShaderSnapshot& snapshot, const ShaderRecompiler::RecompileRequest& request) {
void ReportPreparedAtUse(const ShaderSnapshot& snapshot, const ShaderRecompiler::RecompileRequest& request) {
if (snapshot.codeAddress == NullPixelProgramAddress() && request.shader.stage == ShaderRecompiler::ShaderStage::Fragment && request.context.waveSize == 32u) {
APS5_LOG_ERR("The null pixel program has no wave%u artifact for this draw; preparing it at draw", request.context.waveSize);
return true;
return;
}
if (!snapshot.header.empty()) return snapshot.prepared->deferred;
if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader");
APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast<unsigned long long>(snapshot.codeAddress));
return true;
if (!snapshot.header.empty()) {
if (!snapshot.prepared->deferred) APS5_LOG_ERR("Shader 0x%llx stage %u has no artifact prepared at registration for the state of this draw or dispatch; preparing it at use", static_cast<unsigned long long>(request.shader.codeAddress), static_cast<std::uint32_t>(request.shader.stage));
return;
}
if (request.shader.stage == ShaderRecompiler::ShaderStage::Compute) {
if (snapshot.type != 0) throw std::runtime_error("AGC driver: unregistered program is not a compute shader");
APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast<unsigned long long>(snapshot.codeAddress));
return;
}
APS5_LOG_ERR("Graphics shader 0x%llx was not registered; preparing its artifact at first use", static_cast<unsigned long long>(snapshot.codeAddress));
}
}
std::shared_ptr<const ShaderRecompiler::SourceHandle> SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request) {
require(codeOffset < snapshot.code.size(), "prepared shader code offset is outside the snapshot");
require(std::ranges::equal(request.shader.code, std::span(snapshot.code).subspan(codeOffset)), "prepared shader request does not refer to registered code");
struct PreparedKeyStorage {};
auto& key = HostThreadLocal<std::vector<std::uint64_t>, PreparedKeyStorage>();
ShaderRecompiler::BuildPreparedShaderKey(request, key);
@ -249,18 +329,10 @@ std::shared_ptr<const ShaderRecompiler::SourceHandle> SourceHandleFor(const Shad
for (const auto& entry : snapshot.prepared->entries) {
if (entry.codeOffset == codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) return entry.handle;
}
if (PreparedAtUse(snapshot, request)) {
auto handle = PrepareShaderWithDiagnostics(request);
snapshot.prepared->entries.push_back({codeOffset, handle});
return handle;
}
std::string layouts;
for (const auto& entry : snapshot.prepared->entries) {
if (entry.codeOffset != codeOffset || entry.handle == nullptr || entry.handle->artifact == nullptr) continue;
const auto& layout = entry.handle->artifact->layout;
layouts += " [" + std::to_string(layout.pushConstantOffsetBytes) + "," + std::to_string(layout.pushConstantSizeBytes) + "]";
}
throw std::runtime_error("AGC driver: prepared shader artifact is missing for the requested static ABI: address=" + std::to_string(request.shader.codeAddress) + " stage=" + std::to_string(static_cast<std::uint32_t>(request.shader.stage)) + " wave=" + std::to_string(request.context.waveSize) + " pushOffset=" + std::to_string(request.layout.pushConstantOffsetBytes) + " pushCapacity=" + std::to_string(request.layout.pushConstantSizeBytes) + " preparedLayouts=" + layouts);
ReportPreparedAtUse(snapshot, request);
auto handle = PrepareShaderWithDiagnostics(request);
snapshot.prepared->entries.push_back({codeOffset, handle});
return handle;
}
ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapshot, std::uint64_t vertexId, std::uint64_t fragmentId) {
@ -285,37 +357,85 @@ ShaderRecompiler::RectListShaders DrawRectangle(const ShaderSnapshot& front, con
return PreparedRectangle(front, vertexId, fragmentId);
}
namespace {
std::string hexText(std::uint64_t value) {
char text[24];
std::snprintf(text, sizeof(text), "0x%llx", static_cast<unsigned long long>(value));
return text;
}
std::vector<std::uint32_t> ReadUserDataCallee(std::uint64_t address, std::uint32_t linkRegister) {
constexpr std::uint64_t page = 4096u;
constexpr std::size_t limitWords = 16384u;
if (address % 4u != 0u) throw std::runtime_error("its callee address " + hexText(address) + " is not dword aligned");
std::vector<std::uint32_t> words;
while (words.size() < limitWords) {
const auto next = address + words.size() * 4u;
const auto bytes = static_cast<std::size_t>(page - next % page);
if (!GuestMemory::Accessible(reinterpret_cast<const void*>(static_cast<std::uintptr_t>(next)), bytes)) throw std::runtime_error("its callee at " + hexText(address) + " is not readable at " + hexText(next));
const auto old = words.size();
words.resize(old + bytes / 4u);
GuestMemory::Read(next, std::as_writable_bytes(std::span(words).subspan(old)));
if (const auto length = ShaderRecompiler::UserDataCalleeLength(words, linkRegister)) {
words.resize(*length);
return words;
}
}
throw std::runtime_error("its callee at " + hexText(address) + " does not return within " + std::to_string(limitWords * 4u) + " bytes");
}
std::vector<std::uint32_t> LinkUserDataCallees(const ShaderRecompiler::RecompileRequest& request) {
if (request.shader.stage != ShaderRecompiler::ShaderStage::Compute) return {};
const auto code = request.shader.code;
if (std::none_of(code.begin(), code.end(), [](std::uint32_t word) { return (word & 0xff80ff00u) == 0xbe802100u; })) return {};
const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code);
const ShaderRecompiler::SwappcInfo swappc{false, request.context.userDataBaseRegister, static_cast<std::uint32_t>(request.context.userData.size())};
const auto calls = ShaderRecompiler::FindUserDataCalls(program, swappc);
if (calls.empty()) return {};
std::vector<std::vector<std::uint32_t>> callees;
for (const auto& call : calls) {
const auto slot = call.userDataRegister - swappc.userDataBaseRegister;
const auto address = static_cast<std::uint64_t>(request.context.userData[slot]) | static_cast<std::uint64_t>(request.context.userData[slot + 1u]) << 32u;
try {
callees.push_back(ReadUserDataCallee(address, call.linkRegister));
} catch (const std::exception& error) {
throw std::runtime_error("AGC driver: compute shader " + hexText(request.shader.codeAddress) + " calls user data s[" + std::to_string(call.userDataRegister) + ":" +
std::to_string(call.userDataRegister + 1u) + "] at program counter " + hexText(call.programCounter) + ", but " + error.what());
}
}
return ShaderRecompiler::LinkUserDataCalls(program, calls, callees);
}
}
ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request) {
require(codeOffset < snapshot.code.size(), "prepared shader code offset is outside the snapshot");
const auto code = std::span(snapshot.code).subspan(codeOffset);
require(request.shader.code.data() == code.data() && request.shader.code.size() == code.size(), "prepared invocation does not refer to registered code");
auto invocationRequest = request;
const auto linked = LinkUserDataCallees(request);
auto source = request;
if (!linked.empty()) source.shader.code = linked;
auto invocationRequest = source;
struct PreparedKeyStorage {};
auto& key = HostThreadLocal<std::vector<std::uint64_t>, PreparedKeyStorage>();
ShaderRecompiler::BuildPreparedShaderKey(request, key);
ShaderRecompiler::BuildPreparedShaderKey(source, key);
std::unique_lock lock(snapshot.prepared->mutex);
AwaitRegisteredPreparation(*snapshot.prepared, lock);
for (const auto& entry : snapshot.prepared->entries) {
if (entry.codeOffset != codeOffset) continue;
invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle);
if (!linked.empty() && !std::ranges::equal(invocationRequest.shader.code, linked)) continue;
if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) return std::move(*invocation);
}
if (PreparedAtUse(snapshot, request)) {
auto handle = PrepareShaderWithDiagnostics(request);
invocationRequest = request;
invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*handle);
auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, handle, key);
if (!invocation.has_value()) throw std::runtime_error("AGC driver: raw compute artifact does not match its invocation");
snapshot.prepared->entries.push_back({codeOffset, std::move(handle)});
return std::move(*invocation);
}
std::string layouts;
for (const auto& entry : snapshot.prepared->entries) {
if (entry.codeOffset != codeOffset || entry.handle == nullptr || entry.handle->artifact == nullptr) continue;
const auto& layout = entry.handle->artifact->layout;
layouts += " [" + std::to_string(layout.pushConstantOffsetBytes) + "," + std::to_string(layout.pushConstantSizeBytes) + "]";
}
throw std::runtime_error("AGC driver: prepared shader artifact is missing for the requested static ABI: address=" + std::to_string(request.shader.codeAddress) + " stage=" + std::to_string(static_cast<std::uint32_t>(request.shader.stage)) + " wave=" + std::to_string(request.context.waveSize) + " pushOffset=" + std::to_string(request.layout.pushConstantOffsetBytes) + " pushCapacity=" + std::to_string(request.layout.pushConstantSizeBytes) + " preparedLayouts=" + layouts);
ReportPreparedAtUse(snapshot, source);
auto handle = PrepareShaderWithDiagnostics(source);
invocationRequest = source;
invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*handle);
auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, handle, key);
if (!invocation.has_value()) throw std::runtime_error("AGC driver: an artifact prepared at use does not match its invocation");
snapshot.prepared->entries.push_back({codeOffset, std::move(handle)});
return std::move(*invocation);
}
std::optional<ShaderRecompiler::ShaderFloatMode> RegisteredFloatMode(const ShaderSnapshot& snapshot) {
if (snapshot.registeredState == nullptr) return std::nullopt;
@ -526,15 +646,22 @@ public:
}
void Submit(std::function<void()> task) {
if (threads == 0) {
task();
return;
}
{
std::lock_guard lock(mutex);
tasks.push_back(std::move(task));
std::unique_lock lock(mutex);
if (threads != 0 && !closed) {
tasks.push_back(std::move(task));
lock.unlock();
ready.notify_one();
return;
}
}
ready.notify_one();
task();
}
void Close() {
std::unique_lock lock(mutex);
closed = true;
idle.wait(lock, [&] { return tasks.empty() && running == 0; });
}
private:
@ -562,16 +689,26 @@ private:
pool.ready.wait(lock, [&] { return !pool.tasks.empty(); });
task = std::move(pool.tasks.front());
pool.tasks.pop_front();
++pool.running;
}
task();
task = nullptr;
{
std::lock_guard lock(pool.mutex);
--pool.running;
if (pool.tasks.empty() && pool.running == 0) pool.idle.notify_all();
}
}
return nullptr;
}
std::mutex mutex;
std::condition_variable ready;
std::condition_variable idle;
std::deque<std::function<void()>> tasks;
unsigned threads = 0;
unsigned running = 0;
bool closed = false;
};
bool AsyncRegistrationPrepare() {
@ -580,7 +717,7 @@ bool AsyncRegistrationPrepare() {
}
void PrepareRegisteredInBackground(std::shared_ptr<const ShaderSnapshot> snapshot, std::shared_ptr<RegisteredPreparation> plan) {
RegistrationPreparePool::Instance().Submit([snapshot = std::move(snapshot), plan = std::move(plan)] {
SubmitRegistrationPreparation([snapshot = std::move(snapshot), plan = std::move(plan)] {
PerformanceContext timingContext(FrameTiming::Preparation());
std::vector<PreparedShaders::Entry> entries;
std::exception_ptr failure;
@ -606,6 +743,14 @@ void PrepareRegisteredInBackground(std::shared_ptr<const ShaderSnapshot> snapsho
}
void SubmitRegistrationPreparation(std::function<void()> task) {
RegistrationPreparePool::Instance().Submit(std::move(task));
}
void CloseRegistrationPreparation() {
RegistrationPreparePool::Instance().Close();
}
std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target) {
ShaderPreparationTransaction transaction;
require(decoded.programs.size() == decoded.roles.size(), "graphics ABI program roles are incomplete");
@ -681,9 +826,10 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
GuestMemory::CheckRange(shader, sizeof(Shader), alignof(Shader));
const auto address = reinterpret_cast<std::uintptr_t>(const_cast<const void*>(shader->code));
require(registry != nullptr && registry->contains(address), "graphics ABI refers to an unregistered shader");
if (owner == nullptr) owner = registry->at(address);
const auto& snapshot = *registry->at(address);
require(SameHeader(snapshot, shader), "graphics ABI refers to a replaced shader header");
const auto selected = RegisteredHeader(*registry, shader);
require(selected != nullptr, "graphics ABI refers to a replaced shader header");
if (owner == nullptr) owner = selected;
const auto& snapshot = *selected;
require(snapshot.registeredState != nullptr, "registered shader state is missing");
const auto& registered = *snapshot.registeredState;
for (const auto& [offset, value] : registered.shader) state.shader.insert_or_assign(offset, value);
@ -747,8 +893,8 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegist
std::lock_guard lock(mutex);
const auto address = reinterpret_cast<std::uintptr_t>(const_cast<const void*>(shader->code));
require(shaders != nullptr && shaders->contains(address), "static ABI refers to an unregistered shader");
snapshot = shaders->at(address);
require(SameHeader(*snapshot, shader), "static ABI refers to a replaced shader header");
snapshot = RegisteredHeader(*shaders, shader);
require(snapshot != nullptr, "static ABI refers to a replaced shader header");
}
require(snapshot->registeredState != nullptr, "registered shader state is missing");
QueueState state{};
@ -809,8 +955,8 @@ void Driver::ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::
const auto lookup = [&](const Shader* shader) {
const auto address = reinterpret_cast<std::uintptr_t>(const_cast<const void*>(shader->code));
require(shaders != nullptr && shaders->contains(address), "rectangle ABI refers to an unregistered shader");
const auto snapshot = shaders->at(address);
require(SameHeader(*snapshot, shader), "rectangle ABI refers to a replaced shader header");
const auto snapshot = RegisteredHeader(*shaders, shader);
require(snapshot != nullptr, "rectangle ABI refers to a replaced shader header");
return snapshot;
};
front = lookup(vertex);
@ -905,6 +1051,7 @@ void Driver::RegisterShader(const Shader* shader) {
PerformanceTimer timing("Shader.Register");
ShaderPreparationTransaction transaction;
CheckFailure();
require(!RawGraphicsDrawn(), "a shader was registered after graphics programs were drawn without registration");
GuestMemory::CheckRange(shader, sizeof(Shader), 1);
Shader fields;
std::memcpy(&fields, static_cast<const void*>(shader), sizeof(Shader));
@ -926,10 +1073,13 @@ void Driver::RegisterShader(const Shader* shader) {
if (shaders != nullptr) {
const auto found = shaders->find(snapshot.codeAddress);
if (found != shaders->end()) {
const auto& current = *found->second;
if (current.headerAddress == snapshot.headerAddress && current.type == snapshot.type && current.code == snapshot.code && current.header == snapshot.header) {
transaction.Commit();
return;
for (const auto& current : found->second) {
if (current->headerAddress == snapshot.headerAddress && current->type == snapshot.type && current->code == snapshot.code && current->header == snapshot.header) {
const auto identical = current;
PublishRegisteredShader(shaders, identical);
transaction.Commit();
return;
}
}
}
}

4
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/SynchronizationStatistics.cpp
View File

@ -21,7 +21,7 @@ std::atomic<std::uint64_t> packetLockRecordUs{0}, packetLockSubmitUs{0};
std::atomic<std::uint64_t> triesFailed[TrySites] = {};
std::atomic<std::uint64_t> storesOnGpu{0}, storesBehindCompletions{0}, storesOnCpu{0}, storesDrained{0};
std::atomic<std::uint64_t> storesOnGpu{0}, storesBehindCompletions{0}, storesOnCpu{0}, storesDrained{0}, copiesToHostMemory{0};
std::map<std::uint32_t, std::uint64_t> drainCounts;
@ -39,7 +39,7 @@ SubmissionCosts& submissionCosts(std::uint32_t queue) {
void reportSync() {
std::string report;
for (const auto& [code, count] : drainCounts) report += " " + (code == 0xffffu ? std::string("flip") : Pm4::Name(code << 8u)) + "=" + std::to_string(count);
AgcDriver::ProfilePrint_nid_no_patch("[sync] %llu device drains by packet:%s (%llu waited without the GPU mutex); %llu labels written on the GPU, %llu deferred behind completions, %llu no-op, fallbacks: idle %llu, completions %llu, not imported %llu, undecodable %llu; %llu labels queued per worker, recorded in %llu groups, %llu recorded at once; stores: %llu recorded on the GPU, %llu behind completions, %llu on the CPU (idle), %llu synced (drained)\n", static_cast<unsigned long long>(drainTotal), report.c_str(), static_cast<unsigned long long>(unlockedDrains.load()), static_cast<unsigned long long>(gpuLabels.load()), static_cast<unsigned long long>(completionLabels.load()), static_cast<unsigned long long>(noOpLabels.load()), static_cast<unsigned long long>(labelFallbacks[1].load()), static_cast<unsigned long long>(labelFallbacks[2].load()), static_cast<unsigned long long>(labelFallbacks[3].load()), static_cast<unsigned long long>(labelFallbacks[4].load()), static_cast<unsigned long long>(queuedLabels.load()), static_cast<unsigned long long>(labelGroups.load()), static_cast<unsigned long long>(immediateLabels.load()), static_cast<unsigned long long>(storesOnGpu.load()), static_cast<unsigned long long>(storesBehindCompletions.load()), static_cast<unsigned long long>(storesOnCpu.load()), static_cast<unsigned long long>(storesDrained.load()));
AgcDriver::ProfilePrint_nid_no_patch("[sync] %llu device drains by packet:%s (%llu waited without the GPU mutex); %llu labels written on the GPU, %llu deferred behind completions, %llu no-op, fallbacks: idle %llu, completions %llu, not imported %llu, undecodable %llu; %llu labels queued per worker, recorded in %llu groups, %llu recorded at once; stores: %llu recorded on the GPU, %llu behind completions, %llu on the CPU (idle), %llu synced (drained); %llu copies into host memory synced on their source\n", static_cast<unsigned long long>(drainTotal), report.c_str(), static_cast<unsigned long long>(unlockedDrains.load()), static_cast<unsigned long long>(gpuLabels.load()), static_cast<unsigned long long>(completionLabels.load()), static_cast<unsigned long long>(noOpLabels.load()), static_cast<unsigned long long>(labelFallbacks[1].load()), static_cast<unsigned long long>(labelFallbacks[2].load()), static_cast<unsigned long long>(labelFallbacks[3].load()), static_cast<unsigned long long>(labelFallbacks[4].load()), static_cast<unsigned long long>(queuedLabels.load()), static_cast<unsigned long long>(labelGroups.load()), static_cast<unsigned long long>(immediateLabels.load()), static_cast<unsigned long long>(storesOnGpu.load()), static_cast<unsigned long long>(storesBehindCompletions.load()), static_cast<unsigned long long>(storesOnCpu.load()), static_cast<unsigned long long>(storesDrained.load()), static_cast<unsigned long long>(copiesToHostMemory.load()));
}
void countLabelOutcome(int reason) {

3
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp
View File

@ -366,7 +366,7 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
static std::uint64_t timeouts = 0;
const std::lock_guard lock(reportedMutex);
if (++timeouts % 20 == 0) std::fprintf(stderr, "[gpu] %llu GPU waits have timed out\n", static_cast<unsigned long long>(timeouts));
if (!reported.insert(packet[2] | (static_cast<std::uint64_t>(packet[3]) << 32u)).second) return;
if (!reported.insert(packet[2] | (static_cast<std::uint64_t>(packet[3]) << 32u)).second) continue;
}
const bool wide = ((packet[0] >> 8u) & 0xffu) == 0x93u;
const std::uint64_t reference = wide ? packet[4] | (static_cast<std::uint64_t>(packet[5]) << 32u) : packet[4];
@ -392,7 +392,6 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
}
context.Print(stderr, "[gpu] preceding packet %s\n");
}
if (warned) return;
}
++outcomes.polled;
}

37
core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp
View File

@ -10,6 +10,8 @@
#include <chrono>
#include <atomic>
#include <algorithm>
#include <bit>
#include <emmintrin.h>
#include <functional>
#include <cstdio>
#include <cstdlib>
@ -1875,28 +1877,29 @@ void WriteChanged(std::uint64_t address, std::span<const std::byte> current, std
const TimedAccess timed(CounterChangedWrite, current.size());
auto* destination = reinterpret_cast<std::byte*>(address);
CheckRange(destination, current.size(), 1, true);
constexpr std::size_t block = 256;
const auto size = current.size();
const auto differs = [&](std::size_t at) {
const auto length = std::min(block, size - at);
return std::memcmp(current.data() + at, original.data() + at, length) != 0;
};
// Stamped like a GPU write: a collect memoized for this packet would not see the page fault.
storeOwn(address, size, [&] {
std::size_t firstChanged = size;
std::size_t lastChanged = 0;
for (std::size_t at = 0; at < size; at += block) {
if (!differs(at)) continue;
const auto blockEnd = std::min(at + block, size);
for (std::size_t run = at; run < blockEnd;) {
if (current[run] == original[run]) { ++run; continue; }
auto runEnd = run + 1;
while (runEnd < blockEnd && current[runEnd] != original[runEnd]) ++runEnd;
std::memcpy(destination + run, current.data() + run, runEnd - run);
firstChanged = std::min(firstChanged, run);
lastChanged = std::max(lastChanged, runEnd);
run = runEnd;
}
std::size_t at = 0;
for (; at + 16 <= size; at += 16) {
const auto bytes = _mm_loadu_si128(reinterpret_cast<const __m128i*>(current.data() + at));
const auto same = _mm_cmpeq_epi8(bytes, _mm_loadu_si128(reinterpret_cast<const __m128i*>(original.data() + at)));
const auto equal = static_cast<std::uint32_t>(_mm_movemask_epi8(same));
if (equal == 0xffffu) continue;
if (equal == 0) _mm_storeu_si128(reinterpret_cast<__m128i*>(destination + at), bytes);
else _mm_maskmoveu_si128(bytes, _mm_xor_si128(same, _mm_set1_epi8(-1)), reinterpret_cast<char*>(destination + at));
const auto changed = ~equal & 0xffffu;
firstChanged = std::min(firstChanged, at + static_cast<std::size_t>(std::countr_zero(changed)));
lastChanged = std::max(lastChanged, at + 32 - static_cast<std::size_t>(std::countl_zero(changed)));
}
_mm_sfence();
for (; at < size; ++at) {
if (current[at] == original[at]) continue;
destination[at] = current[at];
firstChanged = std::min(firstChanged, at);
lastChanged = std::max(lastChanged, at + 1);
}
return firstChanged < lastChanged ? std::pair<std::uint64_t, std::uint64_t>{address + firstChanged, address + lastChanged} : std::pair<std::uint64_t, std::uint64_t>{0, 0};
});

70
core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp
View File

@ -72,7 +72,7 @@ bool IndirectDrawsDisabled() {
// nowhere) or a user-data register of the vertex-side stage banks (0x8c.. for the vertex / geometry
// programs, 0x10c.. for the local / hull programs).
bool drawLocation(std::uint32_t value) {
return value == 0x280u || value - 0x8cu < 32u || value - 0x10cu < 32u;
return value == 0 || value == 0x280u || value - 0x8cu < 32u || value - 0x10cu < 32u;
}
Registers& registersFor(QueueState& queue, std::uint32_t opcode) {
@ -135,6 +135,15 @@ void copyMemory(std::uint64_t source, std::uint64_t destination, std::size_t byt
constexpr std::uint32_t CopyDataGpuClockSource = 18;
constexpr std::uint32_t CopyDataCachePolicy = (3u << 13u) | (3u << 25u);
constexpr std::uint32_t ContextRegisterBase = 0xa000;
bool copyDataToRegister(std::span<const std::uint32_t> packet) {
return ((packet[1] >> 8u) & 0xfu) == 0;
}
std::uint32_t copyDataCount(std::span<const std::uint32_t> packet) {
return (packet[1] & 0x10000u) != 0 ? 2u : 1u;
}
std::uint64_t gpuClockCount() {
return static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count() / 10);
@ -267,7 +276,7 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
switch (opcode) {
case 0x11:
size(4);
require(packet[1] == 1 && (packet[2] & 7u) == 0 && packet[3] <= 0xffffu, "unsupported indirect base index, alignment or address bits");
require(packet[1] == 1 && (packet[2] & 3u) == 0 && packet[3] <= 0xffffu, "unsupported indirect base index, alignment or address bits");
if ((header & 2u) == 0) graphics();
break;
case 0x12: graphics(); size(2); require((packet[1] & ~0xfu) == 0, "unsupported CLEAR_STATE payload bits"); break;
@ -323,10 +332,10 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
require((packet[8] & 3u) == 0 && packet[8] >= (opcode == 0x2c ? 16u : 20u), "invalid indirect draw stride");
require((packet[9] & ~0x20u) == (opcode == 0x2c ? 2u : 0u), "unsupported indirect draw initiator");
break;
case 0x15: size(5); if ((packet[4] & ~0xa024u) != 0x41u) throw std::runtime_error("dispatch modifiers 0x" + ToHex(packet[4]) + " are not implemented"); break;
case 0x15: size(5); if ((packet[4] & ~0xa066u) != 0x1u || (packet[4] & 0x22u) == 0x22u) throw std::runtime_error("dispatch modifiers 0x" + ToHex(packet[4]) + " are not implemented"); break;
case 0x16:
require(packet.size() == 3 || packet.size() == 4, "invalid indirect dispatch size");
require((packet.back() & ~0xa024u) == 0x41u, "indirect dispatch modifiers are not implemented");
require((packet.back() & ~0xa064u) == 0x1u, "indirect dispatch modifiers are not implemented");
break;
case 0x22:
size(5);
@ -453,7 +462,13 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
require((packet[1] & ~(0x40110f0fu | CopyDataCachePolicy)) == 0, "COPY_DATA engine or reserved fields are not implemented");
const auto source = ((packet[1] & 0xfu) << 1u) | ((packet[1] >> 30u) & 1u);
const auto destination = ((packet[1] >> 8u) & 0xfu) << 1u;
require(destination == 2 || destination == 4, "COPY_DATA register or GDS destination is not implemented");
if (copyDataToRegister(packet)) {
require(source == 2 || source == 4, "COPY_DATA to a register from a register, GDS, immediate or clock source is not implemented");
require((packet[2] & 3u) == 0, "misaligned COPY_DATA source");
require(packet[5] == 0 && packet[4] >= ContextRegisterBase && packet[4] - ContextRegisterBase <= 0x400u - copyDataCount(packet), "COPY_DATA to a register outside the context registers is not implemented");
break;
}
require(destination == 2 || destination == 4 || destination == 10, "COPY_DATA GDS or performance-counter destination is not implemented");
require(source == 2 || source == 4 || source == 5 || source == 10 || source == 11 || source == CopyDataGpuClockSource, "COPY_DATA register, GDS or reference-clock source is not implemented");
require(source < 10 || source == CopyDataGpuClockSource || ((packet[1] & 0x10000u) == 0 && packet[3] == 0), "64-bit immediate COPY_DATA is not implemented");
break;
@ -579,7 +594,7 @@ std::optional<StoreWrite> ResolveStore(std::span<const std::uint32_t> packet, co
const auto fits = [limit](std::uint64_t destination, std::size_t bytes) { return bytes <= limit && destination % 4 == 0 && bytes % 4 == 0; };
switch ((packet[0] >> 8u) & 0xffu) {
case 0x40: {
if (packet.size() < 6) return std::nullopt;
if (packet.size() < 6 || copyDataToRegister(packet)) return std::nullopt;
const auto source = ((packet[1] & 0xfu) << 1u) | ((packet[1] >> 30u) & 1u);
const std::size_t bytes = (packet[1] & 0x10000u) != 0 ? 8 : 4;
const auto destination = address(packet[4], packet[5]);
@ -606,6 +621,14 @@ std::optional<StoreWrite> ResolveStore(std::span<const std::uint32_t> packet, co
}
}
std::uint32_t DmaSource(std::span<const std::uint32_t> packet) {
return dmaSource(packet);
}
std::uint32_t DmaDestination(std::span<const std::uint32_t> packet) {
return dmaDestination(packet);
}
std::optional<MemoryCopy> DecodeMemoryCopy(std::span<const std::uint32_t> packet) {
if (packet.size() != 7 || ((packet[0] >> 8u) & 0xffu) != 0x50) return std::nullopt;
if (!memorySelector(dmaSource(packet)) || !memorySelector(dmaDestination(packet))) return std::nullopt;
@ -636,6 +659,26 @@ std::uint64_t DispatchArgumentAddress(std::span<const std::uint32_t> packet, con
return queue.dispatchIndirectBase + packet[1];
}
std::array<std::uint32_t, 3> PartialGroupThreads(const std::array<std::uint32_t, 3>& groups, const std::array<std::uint32_t, 3>& numThreads) {
std::array<std::uint32_t, 3> threads{};
bool partial = false;
bool empty = false;
for (std::size_t axis = 0; axis < 3; ++axis) {
const auto full = numThreads[axis] & 0xffffu;
const auto last = numThreads[axis] >> 16u;
require(last != 0 && last <= full, "COMPUTE_NUM_THREAD partial group size is zero or exceeds the full group");
partial |= last != full;
if (groups[axis] == 0) {
empty = true;
continue;
}
const auto total = static_cast<std::uint64_t>(groups[axis] - 1) * full + last;
require(total <= std::numeric_limits<std::uint32_t>::max(), "partial thread group dispatch size exceeds 32 bits");
threads[axis] = static_cast<std::uint32_t>(total);
}
return partial && !empty ? threads : std::array<std::uint32_t, 3>{};
}
std::array<std::uint32_t, 5> ReadDispatchArguments(std::uint64_t arguments, std::uint32_t initiator) {
std::array<std::uint32_t, 5> result{0xc0031500u, 0, 0, 0, initiator};
// Named for the [hooksync] attribution (the read goes through the flush hook).
@ -673,9 +716,10 @@ DrawParameters resolveIndirectDraw(std::span<const std::uint32_t> packet, const
indirect.count = multi ? packet[5] : 1;
indirect.countIndirect = multi && (packet[4] & (1u << 30u)) != 0;
indirect.countAddress = multi ? address(packet[6], packet[7]) : 0;
indirect.baseVertexLocation = packet[2];
indirect.startInstanceLocation = packet[3];
indirect.drawIndexLocation = multi ? packet[4] & 0xffffu : 0x280u;
const auto location = [](std::uint32_t value) { return value == 0 ? 0x280u : value; };
indirect.baseVertexLocation = location(packet[2]);
indirect.startInstanceLocation = location(packet[3]);
indirect.drawIndexLocation = multi ? location(packet[4] & 0xffffu) : 0x280u;
indirect.drawIndexEnabled = multi && (packet[4] >> 31u) != 0;
indirect.indxOffset = 0;
require(indirect.count <= (std::numeric_limits<std::uint64_t>::max() - indirect.arguments - indirect.recordBytes) / indirect.stride, "indirect draw record range overflow");
@ -891,6 +935,14 @@ void Execute(std::span<const std::uint32_t> packet, QueueState& queue) {
return;
}
case 0x40: {
if (copyDataToRegister(packet)) {
std::array<std::uint32_t, 2> values{};
const auto count = copyDataCount(packet);
const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Registers);
GuestMemory::Read(address(packet[2], packet[3]), std::as_writable_bytes(std::span(values).first(count)), 4);
for (std::uint32_t i = 0; i < count; ++i) writeRegister(queue, 0x69, packet[4] - ContextRegisterBase + i, values[i]);
return;
}
const auto destination = address(packet[4], packet[5]);
const auto data = copyDataSource(packet, (packet[1] & 0x10000u) != 0 ? 8 : 4);
GuestMemory::CheckRange(reinterpret_cast<void*>(destination), data.size(), 1, true);

1
core/libs/prx/libSceAgcDriver/Execution/src/ShaderDeviceProfile.cpp
View File

@ -34,7 +34,6 @@ ShaderDeviceProfile::ShaderDeviceProfile(const ShaderRecompiler::SpirvTarget& ta
Require(target.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version, "shader device profile has an incompatible BDA ABI");
Require(bda != nullptr && bda->bufferDeviceAddress == VK_TRUE && core.shaderInt64 == VK_TRUE, "shader runtime requires enabled bufferDeviceAddress and shaderInt64");
Require(bytes != nullptr && bytes->storageBuffer8BitAccess == VK_TRUE, "shader runtime requires enabled storageBuffer8BitAccess");
Require(nullDescriptors, "shader runtime requires enabled nullDescriptor");
Require(core.vertexPipelineStoresAndAtomics == VK_TRUE && core.fragmentStoresAndAtomics == VK_TRUE, "shader runtime requires enabled graphics stores and atomics");
Require(limits.maxPushConstantsSize >= ShaderRecompiler::RuntimeAbi::PushConstantDwords * sizeof(std::uint32_t) && limits.maxBoundDescriptorSets > ShaderRecompiler::RuntimeAbi::DescriptorSet, "shader runtime ABI exceeds device limits");
Require(limits.maxStorageBufferRange >= sizeof(ShaderRecompiler::RuntimeAbi::ShaderData), "shader runtime ShaderData exceeds device limits");

309
core/libs/prx/libSceAgcDriver/Execution/src/ThreadPriority.cpp Normal file
View File

@ -0,0 +1,309 @@
#include "prx/libSceAgcDriver/Execution/include/ThreadPriority.hpp"
#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <mutex>
#include <set>
#include <stdexcept>
#include <string>
#include <type_traits>
#ifdef _WIN32
#include <windows.h>
#else
#include <dlfcn.h>
#include <sched.h>
#include <sys/resource.h>
#include <sys/syscall.h>
#include <unistd.h>
#endif
namespace AgcDriver {
namespace {
ThreadPriorityMode RequestedMode() {
static const ThreadPriorityMode mode = ParseThreadPriorityMode(std::getenv("APS5_THREAD_PRIORITY"));
return mode;
}
int EnvInt(const char* name, int fallback) {
const char* text = std::getenv(name);
return text != nullptr && *text != '\0' ? std::atoi(text) : fallback;
}
void LogOnce(const char* role, const std::string& text) {
static std::mutex mutex;
static std::set<std::string> seen;
std::lock_guard lock(mutex);
if (!seen.insert(std::string(role) + "|" + text).second) return;
std::fprintf(stderr, "[thread-priority] %s: %s\n", role, text.c_str());
}
#ifndef _WIN32
struct DBusConnection;
struct DBusMessage;
struct DBusError {
const char* name;
const char* message;
unsigned int dummy1 : 1, dummy2 : 1, dummy3 : 1, dummy4 : 1, dummy5 : 1;
void* padding1;
};
constexpr int DbusBusSystem = 1;
constexpr int DbusTypeInvalid = 0;
constexpr int DbusTypeInt32 = static_cast<int>('i');
constexpr int DbusTypeUint32 = static_cast<int>('u');
constexpr int DbusTypeUint64 = static_cast<int>('t');
constexpr int DbusTypeInt64 = static_cast<int>('x');
constexpr int DbusTypeString = static_cast<int>('s');
constexpr int DbusTypeVariant = static_cast<int>('v');
struct DBusMessageIter {
void* dummy1;
void* dummy2;
std::uint32_t dummy3;
int dummy4, dummy5, dummy6, dummy7, dummy8, dummy9, dummy10, dummy11;
int pad1;
void* pad2;
void* pad3;
};
struct Dbus {
void* handle = nullptr;
DBusConnection* connection = nullptr;
int (*threadsInitDefault)() = nullptr;
void (*errorInit)(DBusError*) = nullptr;
void (*errorFree)(DBusError*) = nullptr;
int (*errorIsSet)(const DBusError*) = nullptr;
DBusConnection* (*busGet)(int, DBusError*) = nullptr;
DBusMessage* (*messageNewMethodCall)(const char*, const char*, const char*, const char*) = nullptr;
int (*messageAppendArgs)(DBusMessage*, int, ...) = nullptr;
DBusMessage* (*sendWithReplyAndBlock)(DBusConnection*, DBusMessage*, int, DBusError*) = nullptr;
void (*messageUnref)(DBusMessage*) = nullptr;
int (*iterInit)(DBusMessage*, DBusMessageIter*) = nullptr;
int (*iterGetArgType)(DBusMessageIter*) = nullptr;
void (*iterRecurse)(DBusMessageIter*, DBusMessageIter*) = nullptr;
void (*iterGetBasic)(DBusMessageIter*, void*) = nullptr;
std::string failure;
};
Dbus& LoadDbus() {
static Dbus dbus = [] {
Dbus loaded;
loaded.handle = dlopen("libdbus-1.so.3", RTLD_NOW | RTLD_LOCAL);
if (loaded.handle == nullptr) {
loaded.failure = "libdbus-1.so.3 is not available";
return loaded;
}
const auto resolve = [&](auto& function, const char* name) {
function = reinterpret_cast<std::remove_reference_t<decltype(function)>>(dlsym(loaded.handle, name));
if (function == nullptr && loaded.failure.empty()) loaded.failure = std::string("libdbus lacks ") + name;
};
resolve(loaded.threadsInitDefault, "dbus_threads_init_default");
resolve(loaded.errorInit, "dbus_error_init");
resolve(loaded.errorFree, "dbus_error_free");
resolve(loaded.errorIsSet, "dbus_error_is_set");
resolve(loaded.busGet, "dbus_bus_get");
resolve(loaded.messageNewMethodCall, "dbus_message_new_method_call");
resolve(loaded.messageAppendArgs, "dbus_message_append_args");
resolve(loaded.sendWithReplyAndBlock, "dbus_connection_send_with_reply_and_block");
resolve(loaded.messageUnref, "dbus_message_unref");
resolve(loaded.iterInit, "dbus_message_iter_init");
resolve(loaded.iterGetArgType, "dbus_message_iter_get_arg_type");
resolve(loaded.iterRecurse, "dbus_message_iter_recurse");
resolve(loaded.iterGetBasic, "dbus_message_iter_get_basic");
if (!loaded.failure.empty()) return loaded;
loaded.threadsInitDefault();
DBusError error{};
loaded.errorInit(&error);
loaded.connection = loaded.busGet(DbusBusSystem, &error);
if (loaded.connection == nullptr) {
loaded.failure = std::string("system bus: ") + (loaded.errorIsSet(&error) && error.message != nullptr ? error.message : "unavailable");
loaded.errorFree(&error);
}
return loaded;
}();
return dbus;
}
DBusMessage* RtkitSend(DBusMessage* message, const char* method, std::string& failure) {
auto& dbus = LoadDbus();
DBusError error{};
dbus.errorInit(&error);
auto* reply = dbus.sendWithReplyAndBlock(dbus.connection, message, 5000, &error);
dbus.messageUnref(message);
if (reply == nullptr) {
failure = std::string(method) + ": " + (dbus.errorIsSet(&error) && error.message != nullptr ? error.message : "no reply");
dbus.errorFree(&error);
}
return reply;
}
bool RtkitCall(const char* method, std::uint64_t tid, int type, const void* value, std::string& failure) {
auto& dbus = LoadDbus();
if (dbus.connection == nullptr) {
failure = dbus.failure;
return false;
}
static std::mutex mutex;
std::lock_guard lock(mutex);
auto* message = dbus.messageNewMethodCall("org.freedesktop.RealtimeKit1", "/org/freedesktop/RealtimeKit1", "org.freedesktop.RealtimeKit1", method);
if (message == nullptr) {
failure = "dbus message allocation failed";
return false;
}
const std::uint64_t pid = static_cast<std::uint64_t>(getpid());
bool ok = false;
if (type == DbusTypeInt32) ok = dbus.messageAppendArgs(message, DbusTypeUint64, &pid, DbusTypeUint64, &tid, DbusTypeInt32, value, DbusTypeInvalid) != 0;
else ok = dbus.messageAppendArgs(message, DbusTypeUint64, &pid, DbusTypeUint64, &tid, DbusTypeUint32, value, DbusTypeInvalid) != 0;
if (!ok) {
dbus.messageUnref(message);
failure = "dbus argument append failed";
return false;
}
auto* reply = RtkitSend(message, method, failure);
if (reply == nullptr) return false;
dbus.messageUnref(reply);
return true;
}
bool RtkitProperty(const char* name, std::int64_t& value, std::string& failure) {
auto& dbus = LoadDbus();
if (dbus.connection == nullptr) {
failure = dbus.failure;
return false;
}
auto* message = dbus.messageNewMethodCall("org.freedesktop.RealtimeKit1", "/org/freedesktop/RealtimeKit1", "org.freedesktop.DBus.Properties", "Get");
if (message == nullptr) {
failure = "dbus message allocation failed";
return false;
}
const char* interfaceName = "org.freedesktop.RealtimeKit1";
if (dbus.messageAppendArgs(message, DbusTypeString, &interfaceName, DbusTypeString, &name, DbusTypeInvalid) == 0) {
dbus.messageUnref(message);
failure = "dbus argument append failed";
return false;
}
auto* reply = RtkitSend(message, name, failure);
if (reply == nullptr) return false;
DBusMessageIter iter{};
DBusMessageIter variant{};
bool ok = false;
if (dbus.iterInit(reply, &iter) != 0 && dbus.iterGetArgType(&iter) == DbusTypeVariant) {
dbus.iterRecurse(&iter, &variant);
const int type = dbus.iterGetArgType(&variant);
if (type == DbusTypeInt32) {
std::int32_t number = 0;
dbus.iterGetBasic(&variant, &number);
value = number;
ok = true;
} else if (type == DbusTypeInt64) {
std::int64_t number = 0;
dbus.iterGetBasic(&variant, &number);
value = number;
ok = true;
}
}
dbus.messageUnref(reply);
if (!ok) failure = std::string(name) + ": unexpected reply type";
return ok;
}
bool PrepareRealtime(std::uint32_t priority, const char* role, std::string& failure) {
std::int64_t maxPriority = 0;
std::int64_t maxRttime = 0;
std::string rtkitFailure;
std::optional<RtkitLimits> rtkit;
if (RtkitProperty("MaxRealtimePriority", maxPriority, rtkitFailure) && RtkitProperty("RTTimeUSecMax", maxRttime, rtkitFailure)) rtkit = RtkitLimits{maxPriority, maxRttime};
static std::mutex mutex;
std::lock_guard lock(mutex);
rlimit limit{};
if (getrlimit(RLIMIT_RTTIME, &limit) != 0) {
failure = "getrlimit(RLIMIT_RTTIME) failed";
return false;
}
const auto requested = static_cast<std::uint64_t>(std::max(1, EnvInt("APS5_THREAD_RTTIME_US", 200000)));
const auto plan = PlanRealtime(priority, rtkit, {limit.rlim_cur, limit.rlim_max}, requested);
if (!plan.refusal.empty()) {
failure = rtkit.has_value() ? plan.refusal : rtkitFailure;
return false;
}
sched_param param{};
if (sched_getparam(0, &param) != 0 || sched_setscheduler(0, sched_getscheduler(0) | SCHED_RESET_ON_FORK, &param) != 0) {
failure = "SCHED_RESET_ON_FORK could not be set";
return false;
}
if (!plan.limit.has_value()) return true;
const rlimit wanted{plan.limit->soft, plan.limit->hard};
if (setrlimit(RLIMIT_RTTIME, &wanted) != 0) {
failure = "setrlimit(RLIMIT_RTTIME) failed";
return false;
}
if (wanted.rlim_max != limit.rlim_max) {
const auto text = [](rlim_t value) { return value == RLIM_INFINITY ? std::string("unlimited") : std::to_string(value); };
LogOnce(role, "RLIMIT_RTTIME hard limit lowered from " + text(limit.rlim_max) + " to " + text(wanted.rlim_max) + " us for the process");
}
return true;
}
void Raise(const char* role) {
const auto tid = static_cast<std::uint64_t>(syscall(SYS_gettid));
std::string failure;
if (RequestedMode() == ThreadPriorityMode::Realtime) {
const std::uint32_t priority = static_cast<std::uint32_t>(std::max(1, std::min(99, EnvInt("APS5_THREAD_RT", 10))));
if (!PrepareRealtime(priority, role, failure) || !RtkitCall("MakeThreadRealtimeWithPID", tid, DbusTypeUint32, &priority, failure)) {
throw std::runtime_error(std::string("APS5_THREAD_PRIORITY=rt: ") + role + " realtime refused: " + failure);
}
const int policy = sched_getscheduler(0) & ~SCHED_RESET_ON_FORK;
LogOnce(role, std::string(policy == SCHED_FIFO ? "SCHED_FIFO " : "SCHED_RR ") + std::to_string(priority) + " via rtkit");
return;
}
const std::int32_t nice = static_cast<std::int32_t>(std::max(-20, std::min(19, EnvInt("APS5_THREAD_NICE", -10))));
if (!RtkitCall("MakeThreadHighPriorityWithPID", tid, DbusTypeInt32, &nice, failure)) {
throw std::runtime_error(std::string("APS5_THREAD_PRIORITY=high: ") + role + " nice " + std::to_string(nice) + " refused: " + failure);
}
LogOnce(role, "nice " + std::to_string(nice) + " via rtkit");
}
#else
void Raise(const char* role) {
const int priority = RequestedMode() == ThreadPriorityMode::Realtime ? THREAD_PRIORITY_TIME_CRITICAL : THREAD_PRIORITY_HIGHEST;
if (!SetThreadPriority(GetCurrentThread(), priority)) {
throw std::runtime_error(std::string("APS5_THREAD_PRIORITY: ") + role + " SetThreadPriority failed (error " + std::to_string(GetLastError()) + ")");
}
LogOnce(role, priority == THREAD_PRIORITY_TIME_CRITICAL ? "time-critical" : "highest");
}
#endif
}
ThreadPriorityMode ParseThreadPriorityMode(const char* text) {
if (text == nullptr || *text == '\0' || std::strcmp(text, "off") == 0) return ThreadPriorityMode::Off;
if (std::strcmp(text, "high") == 0) return ThreadPriorityMode::High;
if (std::strcmp(text, "rt") == 0) return ThreadPriorityMode::Realtime;
throw std::invalid_argument(std::string("APS5_THREAD_PRIORITY=") + text + " is not off, high or rt");
}
RealtimePlan PlanRealtime(std::uint32_t priority, const std::optional<RtkitLimits>& rtkit, RttimeLimit current, std::uint64_t requestedUs) {
if (!rtkit.has_value()) return {"rtkit's limits are unknown", std::nullopt};
if (static_cast<std::int64_t>(priority) > rtkit->maxPriority) return {"priority " + std::to_string(priority) + " is above rtkit's MaxRealtimePriority " + std::to_string(rtkit->maxPriority), std::nullopt};
if (rtkit->maxRttimeUs <= 0) return {"rtkit reports RTTimeUSecMax " + std::to_string(rtkit->maxRttimeUs), std::nullopt};
const auto ceiling = std::min(requestedUs, static_cast<std::uint64_t>(rtkit->maxRttimeUs));
auto wanted = current;
if (wanted.hard > ceiling) wanted.hard = ceiling;
if (wanted.soft > wanted.hard / 2) wanted.soft = wanted.hard / 2;
if (wanted == current) return {};
return {{}, wanted};
}
void RaiseWorkerThreadPriority(const char* role) {
if (RequestedMode() == ThreadPriorityMode::Off) return;
Raise(role);
}
}

102
core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp
View File

@ -11,6 +11,7 @@
#include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp"
#include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
#include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp"
#include "prx/libSceAgcDriver/Execution/include/DisplayFormat.hpp"
#include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp"
@ -68,6 +69,12 @@ void require(bool condition, const char* reason) {
if (!condition) throw std::runtime_error(std::string("Vulkan presentation: ") + reason);
}
std::string workgroupLimitError(std::uint32_t x, std::uint32_t y, std::uint32_t z, const std::uint32_t* limit) {
char text[160];
std::snprintf(text, sizeof(text), "Vulkan dispatch: workgroup count %ux%ux%u exceeds device limits %ux%ux%u", x, y, z, limit[0], limit[1], limit[2]);
return text;
}
// The ShaderResources content cache dispatches share with recorded draws: Graphics::ResourceCache,
// one process-wide instance (see SharedResourceCache) that the State references so this file keeps
// its Find/Insert/Remove/Clear calls; the device clears it at teardown before its descriptor caches
@ -213,6 +220,7 @@ struct VulkanDevice::State {
bool sampleRateShading = false;
bool shaderClock = false;
bool narrowSubgroupClock = false;
bool float64DenormPreserve = false;
// VK_EXT_descriptor_indexing with non-uniform image array indexing (bindless image tables in
// graphics stages, and compute workgroups wider than a wave).
bool descriptorIndexing = false;
@ -248,6 +256,7 @@ struct VulkanDevice::State {
// VK_KHR_timeline_semaphore enabled: the recorder's unlocked waits are available.
bool timelineSemaphores = false;
bool computeWave32 = false;
bool meshWave32 = false;
std::uint32_t maxComputeSubgroupSize = 0;
// Indirect draw features enabled (see Graphics::Context).
bool drawIndirectFirstInstance = false;
@ -275,6 +284,7 @@ struct VulkanDevice::State {
std::unique_ptr<Graphics::GpuColorTransfer> colorTransfer;
std::shared_ptr<Graphics::BufferPool> bufferPool;
std::unique_ptr<Graphics::Buffer> emptyBuffer;
std::unique_ptr<Graphics::PlaceholderImages> placeholderImages;
std::unique_ptr<Graphics::TextureCache> textureCache;
std::unique_ptr<Graphics::PipelineCache> pipelineCache;
std::unique_ptr<Graphics::DescriptorCache> descriptorCache;
@ -608,6 +618,7 @@ struct VulkanDevice::State {
if (workgroupMemoryLayout != VK_NULL_HANDLE) reinterpret_cast<PFN_vkDestroyDescriptorSetLayout>(deviceProc(device, "vkDestroyDescriptorSetLayout"))(device, workgroupMemoryLayout, nullptr);
descriptorCache.reset();
emptyBuffer.reset();
placeholderImages.reset();
samplerCache.reset();
textureCache.reset();
detiler.reset();
@ -1071,6 +1082,9 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
enabled.shaderInt64 = VK_TRUE;
enabled.shaderFloat64 = available.shaderFloat64 && floatControls.shaderSignedZeroInfNanPreserveFloat64;
if (enabled.shaderFloat64) state->capabilities.push_back(spv::CapabilityFloat64);
state->float64DenormPreserve = enabled.shaderFloat64 && floatControls.shaderDenormPreserveFloat64 == VK_TRUE &&
floatControls.denormBehaviorIndependence == VK_SHADER_FLOAT_CONTROLS_INDEPENDENCE_ALL;
if (state->float64DenormPreserve && std::find(state->capabilities.begin(), state->capabilities.end(), static_cast<std::uint32_t>(spv::CapabilityDenormPreserve)) == state->capabilities.end()) state->capabilities.push_back(spv::CapabilityDenormPreserve);
enabled.vertexPipelineStoresAndAtomics = VK_TRUE;
enabled.fragmentStoresAndAtomics = VK_TRUE;
enabled.tessellationShader = available.tessellationShader;
@ -1235,9 +1249,11 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &sizeProperties);
state->computeWave32 = subgroupSizeFeatures.subgroupSizeControl == VK_TRUE && subgroupSize.minSubgroupSize <= 32u && subgroupSize.maxSubgroupSize >= 32u &&
(subgroupSize.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0;
state->meshWave32 = state->meshShader && subgroupSizeFeatures.subgroupSizeControl == VK_TRUE && subgroupSize.minSubgroupSize <= 32u && subgroupSize.maxSubgroupSize >= 32u &&
(subgroupSize.requiredSubgroupSizeStages & VK_SHADER_STAGE_MESH_BIT_EXT) != 0;
state->maxComputeSubgroupSize = subgroupSize.maxSubgroupSize;
}
if (state->computeWave32) {
if (state->computeWave32 || state->meshWave32) {
subgroupSizeFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT};
subgroupSizeFeatures.subgroupSizeControl = VK_TRUE;
deviceExtensions.push_back(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME);
@ -1245,7 +1261,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
deviceInfo.ppEnabledExtensionNames = deviceExtensions.data();
subgroupSizeFeatures.pNext = byteFeatures.pNext;
byteFeatures.pNext = &subgroupSizeFeatures;
APS5_LOG_OUT("Compute wave32 programs run on subgroups of %u", 32u);
APS5_LOG_OUT("Wave32 programs run on subgroups of %u: compute %d, mesh %d", 32u, state->computeWave32, state->meshWave32);
}
timelineFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES_KHR};
timelineFeatures.timelineSemaphore = VK_TRUE;
@ -1274,17 +1290,19 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
}
bdaFeatures.pNext = &byteFeatures;
VkPhysicalDeviceRobustness2FeaturesEXT robustness2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT};
Graphics::Require(hasExtension(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME), "shader runtime requires VK_EXT_robustness2");
VkPhysicalDeviceFeatures2 queried{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &robustness2};
state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &queried);
Graphics::Require(robustness2.nullDescriptor == VK_TRUE, "shader runtime requires nullDescriptor");
robustness2 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT};
robustness2.nullDescriptor = VK_TRUE;
robustness2.pNext = bdaFeatures.pNext;
bdaFeatures.pNext = &robustness2;
deviceExtensions.push_back(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
deviceInfo.enabledExtensionCount = static_cast<std::uint32_t>(deviceExtensions.size());
deviceInfo.ppEnabledExtensionNames = deviceExtensions.data();
if (hasExtension(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME)) {
VkPhysicalDeviceFeatures2 queried{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &robustness2};
state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &queried);
}
if (robustness2.nullDescriptor == VK_TRUE) {
robustness2 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT};
robustness2.nullDescriptor = VK_TRUE;
robustness2.pNext = bdaFeatures.pNext;
bdaFeatures.pNext = &robustness2;
deviceExtensions.push_back(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME);
deviceInfo.enabledExtensionCount = static_cast<std::uint32_t>(deviceExtensions.size());
deviceInfo.ppEnabledExtensionNames = deviceExtensions.data();
}
deviceInfo.pNext = &bdaFeatures;
auto shaderProfile = std::make_unique<const ShaderDeviceProfile>(buildTarget(), deviceInfo, state->properties.limits);
check(state->InstanceFunction<PFN_vkCreateDevice>("vkCreateDevice")(selected, &deviceInfo, nullptr, &state->device), "vkCreateDevice");
@ -1310,6 +1328,11 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
state->descriptorCache = std::make_unique<Graphics::DescriptorCache>(graphicsContext());
state->samplerCache = std::make_unique<Graphics::SamplerCache>();
state->CreateWorkgroupMemoryLayout();
if (!state->shaderProfile->NullDescriptors()) {
std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock);
if (!GuestMemory::GpuMutex().HeldByThisThread()) gpuLock.lock();
state->placeholderImages = std::make_unique<Graphics::PlaceholderImages>(graphicsContext());
}
state->recorder = std::make_unique<Graphics::Recorder>(graphicsContext(), state->timelineSemaphores);
state->recorder->Activate();
state->context = buildContext();
@ -1381,6 +1404,11 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
VulkanDevice::~VulkanDevice() = default;
bool VulkanDevice::ImportGuestMemory(const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice) {
if (adoptDevice) GuestMemory::AssertGpuLockHeld("VulkanDevice::ImportGuestMemory");
return Graphics::ImportMappedRanges(graphicsContext(), ranges, generation, adoptDevice);
}
void VulkanDevice::PrepareForReplacement() {
GuestMemory::AssertGpuLockHeld("VulkanDevice::PrepareForReplacement");
require(state->recorder != nullptr && Graphics::Recorder::Active() == state->recorder.get(), "device replacement requires its active recorder");
@ -1465,6 +1493,12 @@ bool VulkanDevice::RecordedWritesSettled(std::uint64_t address, std::size_t byte
return Graphics::Recorder::PendingCompletionLabels() == 0 || !recorder.CompletionLabelIn(address, bytes);
}
bool VulkanDevice::StoresPendingOver(std::uint64_t address, std::size_t bytes) const {
const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> range{{{address, address + bytes}}};
if (Graphics::StorageTexture::AnyPendingOverlaps(range) || Graphics::AnyShadowedOverlaps(range) || state->CopiedWriterOverlaps(address, bytes)) return true;
return state->recorder && (state->recorder->PendingWriteOverlaps(address, bytes) || state->recorder->PendingLabelIn(address, bytes));
}
int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) {
if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4;
auto& recorder = *state->recorder;
@ -1984,7 +2018,6 @@ void VulkanDevice::Resize(std::uint32_t width, std::uint32_t height) {
return;
}
if (state->extent.width == width && state->extent.height == height) return;
WaitIdle();
VkSurfaceCapabilitiesKHR surface{};
check(state->InstanceFunction<PFN_vkGetPhysicalDeviceSurfaceCapabilitiesKHR>("vkGetPhysicalDeviceSurfaceCapabilitiesKHR")(state->physical, state->surface, &surface), "vkGetPhysicalDeviceSurfaceCapabilitiesKHR resize");
// The window's drawable size and the surface's extent can disagree for a moment while the window
@ -1993,11 +2026,13 @@ void VulkanDevice::Resize(std::uint32_t width, std::uint32_t height) {
width = surface.currentExtent.width;
height = surface.currentExtent.height;
if (width == 0 || height == 0) {
WaitIdle();
state->extent = {0, 0};
return;
}
if (state->extent.width == width && state->extent.height == height) return;
}
WaitIdle();
require(width >= surface.minImageExtent.width && width <= surface.maxImageExtent.width && height >= surface.minImageExtent.height && height <= surface.maxImageExtent.height, "unsupported resized output extent");
require((surface.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT) != 0 && (surface.supportedCompositeAlpha & VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR) != 0 && (surface.supportedTransforms & VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR) != 0, "resized surface capabilities are unsupported");
VkSwapchainCreateInfoKHR create{VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR};
@ -2336,7 +2371,7 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
require(!slot.inFlight, "presentation slot is still in flight");
if (display != nullptr && resident == nullptr) {
static_cast<void>(DisplayBufferSize(*display));
state->colorTransfer->Upload(display->address, width, height, Graphics::ColorTileMode::RenderTarget);
state->colorTransfer->Upload(display->address, width, height, Graphics::ColorTileMode::RenderTarget, DisplayTexelBytes(display->pixelFormat));
}
if (!pixels.empty()) state->Upload(pixels);
// The frame's recorded work (and a refresh of the resident image) must reach the queue before the
@ -2405,12 +2440,12 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu
pipelineBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &residentBarrier);
Graphics::Recorder::CountBarriers(CommandClass::PresentBlit);
if (residentConvert) {
state->colorTransfer->DetileImage(commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, width, height, Graphics::ColorTileMode::RenderTarget, DisplayRedLow(display->pixelFormat), DisplayTenBit(display->pixelFormat));
state->colorTransfer->DetileImage(commands, resident->Image(), VK_IMAGE_LAYOUT_GENERAL, width, height, Graphics::ColorTileMode::RenderTarget, DisplayRedLow(display->pixelFormat), DisplayTenBit(display->pixelFormat), DisplayHalfFloat(display->pixelFormat));
state->scaler->RecordUpload(commands, state->colorTransfer->LinearBuffer());
}
} else {
if (display != nullptr) {
state->colorTransfer->Detile(commands, DisplayRedLow(display->pixelFormat), DisplayTenBit(display->pixelFormat));
state->colorTransfer->Detile(commands, DisplayRedLow(display->pixelFormat), DisplayTenBit(display->pixelFormat), DisplayHalfFloat(display->pixelFormat));
}
if (uniform != nullptr) state->scaler->RecordClear(commands, *uniform);
else state->scaler->RecordUpload(commands, display != nullptr ? state->colorTransfer->LinearBuffer() : state->uploadBuffer);
@ -2625,6 +2660,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const {
target.fragmentShaderBarycentricEnabled = state->fragmentShaderBarycentric;
target.nonConstantImageOffsets = state->maintenance8;
target.narrowSubgroupClock = state->narrowSubgroupClock;
target.float64DenormPreserve = state->float64DenormPreserve;
target.fixedPushSlots = state->graphicsPipelineLibrary;
target.srgbDecodeFormats = state->srgbDecodeFormats;
if (state->meshShader) {
@ -2716,6 +2752,7 @@ Graphics::Context VulkanDevice::buildContext() const {
context.samplerFilterMinmax = state->samplerFilterMinmax;
context.nonSeamlessCubeMap = state->nonSeamlessCubeMap;
context.conservativeRasterization = state->conservativeRasterization;
context.meshWave32 = state->meshWave32;
context.provokingVertexLast = state->provokingVertexLast;
context.graphicsPipelineLibrary = state->graphicsPipelineLibrary;
context.provokingVertexModePerPipeline = state->provokingVertexModePerPipeline;
@ -2731,6 +2768,7 @@ Graphics::Context VulkanDevice::buildContext() const {
context.sampleRateShading = state->sampleRateShading;
context.nullDescriptors = state->shaderProfile != nullptr && state->shaderProfile->NullDescriptors();
context.bufferInt64Atomics = state->bufferInt64Atomics;
context.placeholderImages = state->placeholderImages.get();
context.primitiveListRestart = state->primitiveListRestart;
context.imageViewMinLod = state->imageViewMinLod;
context.pipelineExecutableInfo = state->pipelineExecutableInfo;
@ -2823,6 +2861,7 @@ struct PreparedDispatch {
// APS5_PROFILE_DRAW: stage A's time, added to the [dispatch] phase totals by the dispatch, and
// the prepare's parts for the driver's 'prepare:' rows (PreparePhase order).
double prepareMs = 0;
std::uint64_t dispatchThreads = 0;
enum PreparePhase : std::size_t { PrepareKey, PrepareFind, PreparePrecollect, PreparePresync, PrepareStageA, PreparePhaseCount };
std::array<double, PreparePhaseCount> phaseMs{};
// The stage-A pre-sync (see PrepareDispatch): the newest recorder serial waited for without the
@ -3266,7 +3305,7 @@ std::uint64_t VulkanDevice::presync(std::span<const std::pair<std::uint64_t, std
return serial;
}
std::shared_ptr<PreparedDispatch> VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span<const Graphics::GuestMemorySnapshot> snapshots) {
std::shared_ptr<PreparedDispatch> VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t dispatchThreads) {
ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion);
// APS5_LOCKED_BUILD=1: the whole build under the mutex, as before the split.
static const bool lockedBuild = std::getenv("APS5_LOCKED_BUILD") != nullptr;
@ -3276,6 +3315,7 @@ std::shared_ptr<PreparedDispatch> VulkanDevice::PrepareDispatch(const ShaderReco
const Graphics::CompiledShader compute{ShaderRecompiler::ShaderStage::Compute, &shader, 0};
const auto context = graphicsContext();
auto prepared = std::make_shared<PreparedDispatch>();
prepared->dispatchThreads = dispatchThreads;
// A cached build (the map locks itself) is revalidated under the mutex by the dispatch, which
// looks it up again by the key made here; the object stays local, only its surfaces matter
// below. Anything inserted between here and the dispatch is simply replaced by this build.
@ -3295,7 +3335,7 @@ std::shared_ptr<PreparedDispatch> VulkanDevice::PrepareDispatch(const ShaderReco
phase(PreparedDispatch::PrepareFind);
}
if (cached == nullptr) {
prepared->resources = std::make_shared<Graphics::ShaderResources>(context, compute, snapshots, true);
prepared->resources = std::make_shared<Graphics::ShaderResources>(context, compute, snapshots, true, dispatchThreads);
phase(PreparedDispatch::PrepareStageA);
}
if (cached != nullptr && CachedPrecollect()) {
@ -3436,7 +3476,11 @@ void VulkanDevice::decideIndirect(RecordedDispatch& record, IndirectOutcome& out
record.arguments = 0;
timer.indirect = false;
std::snprintf(groupsText, 40, "%ux%ux%u", record.x, record.y, record.z);
if (record.x > limit[0] || record.y > limit[1] || record.z > limit[2]) throw std::runtime_error("Vulkan dispatch: workgroup count exceeds device limits");
if (record.x > limit[0] || record.y > limit[1] || record.z > limit[2]) {
char where[96];
std::snprintf(where, sizeof(where), " (indirect arguments at 0x%llx read by the CPU, reason %d)", static_cast<unsigned long long>(arguments), outcome.cpuReason);
throw std::runtime_error(workgroupLimitError(record.x, record.y, record.z, limit) + where);
}
}
void VulkanDevice::recordDispatch(RecordedDispatch& record) {
@ -3478,6 +3522,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
recordStep(PhaseRecordCommands);
recorder.Keep(record.objects);
recorder.Keep(record.resources);
if (record.resources != nullptr) recorder.KeepBytes(record.resources.get(), record.resources->CopiedBytes());
recordStep(PhaseRecordKeeps);
if (record.dataRefresh != RecordedDispatch::DataRefresh::None) {
// The template's data buffers take this dispatch's words: a transfer write the pre-dispatch
@ -3551,7 +3596,10 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
recordStep(PhaseRecordMarks);
// Only copied written buffers (and BDA fault checks) need work once the GPU is done; without them
// the batch can signal its labels from the GPU.
if (resources.NeedsCompletion()) {
if (resources.NeedsCompletion() && resources.Reusable()) {
auto kept = record.resources;
recorder.OnComplete([kept] { kept->CheckFaults(); });
} else if (resources.NeedsCompletion()) {
// Listed for DispatchIndirect until the write-back ran (a non-reusable object, so once).
auto writers = state->copiedWriters;
auto kept = record.resources;
@ -3561,12 +3609,13 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
kept->WriteBackBuffers();
});
// Listed after the registration: a throw there leaves nothing that would pin the CPU path forever.
writers->push_back(kept);
if (resources.HasCopiedWrites()) writers->push_back(kept);
}
recordStep(PhaseRecordCompletion);
}
VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t programAddress, std::shared_ptr<PreparedDispatch> prepared, std::shared_ptr<const Recipe>* recipeOut) {
state->recorder->BoundKeptBytes();
ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion);
PerformanceTimer timing("Vulkan.Dispatch");
if (recipeOut != nullptr) *recipeOut = nullptr;
@ -3587,7 +3636,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
const auto context = graphicsContext();
const auto* limit = state->properties.limits.maxComputeWorkGroupCount;
if (arguments == 0 && (x > limit[0] || y > limit[1] || z > limit[2])) {
throw std::runtime_error("Vulkan dispatch: workgroup count exceeds device limits");
throw std::runtime_error(workgroupLimitError(x, y, z, limit));
}
// Pipelines are shared by dispatches of one compiled variant; descriptor set layouts built from the
// same bindings are compatible, so the pipeline layout of the first dispatch serves them all.
@ -3706,7 +3755,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
}
if (resources == nullptr) {
const auto buildStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
resources = std::make_shared<Graphics::ShaderResources>(context, shaders[0], snapshots);
resources = std::make_shared<Graphics::ShaderResources>(context, shaders[0], snapshots, false, prepared != nullptr ? prepared->dispatchThreads : 0);
if (profile) timer.add(PhaseResourcesFullBuild, std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - buildStart).count());
if (cacheable) {
++d.cacheMisses;
@ -3839,7 +3888,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
// object the resource cache serves under this content key (reusable: no lease, no copied
// writes, every direct region import- or mirror-served), so a hit's proof is the template's
// Revalidate and nothing needs completion work.
if (recipeOut != nullptr && DispatchRecipes() && cacheable && resources->Reusable() && objects != nullptr && !contentKey.empty()) {
if (recipeOut != nullptr && DispatchRecipes() && cacheable && resources->Reusable() && !resources->NeedsCompletion() && objects != nullptr && !contentKey.empty()) {
auto recipe = std::make_shared<Recipe>();
recipe->device = state->device;
recipe->templateRef = resources;
@ -3861,6 +3910,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
}
RecipeOutcome VulkanDevice::DispatchRecipe(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::uint64_t programAddress, const std::shared_ptr<RecipeHit>& hit, IndirectOutcome& outcome, const std::shared_ptr<PreparedDispatch>& verify, bool refreshByWords) {
state->recorder->BoundKeptBytes();
ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion);
PerformanceTimer timing("Vulkan.DispatchRecipe");
outcome = {0, 0};
@ -3874,7 +3924,7 @@ RecipeOutcome VulkanDevice::DispatchRecipe(const ShaderRecompiler::RecompileResu
const auto context = graphicsContext();
const auto* limit = state->properties.limits.maxComputeWorkGroupCount;
if (arguments == 0 && (x > limit[0] || y > limit[1] || z > limit[2])) {
throw std::runtime_error("Vulkan dispatch: workgroup count exceeds device limits");
throw std::runtime_error(workgroupLimitError(x, y, z, limit));
}
if (recipe.needsCompletion || recipe.holdsLease || !hit->resources->Reusable()) throw std::runtime_error("Vulkan dispatch: recipe over a non-reusable template");
// Without the refresh the template holds the words it was keyed with: a data-only hit takes

2
core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp
View File

@ -18,12 +18,12 @@ struct BufferAllocation {
void* mapping;
VkDeviceAddress address;
VkDeviceSize allocationBytes;
// Size the VkBuffer was created with (see BufferPool::Capacity), not the size a user asked for.
std::size_t bytes;
VkBufferUsageFlags usage;
VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
VkDeviceSize offset = 0;
bool slab = false;
std::size_t bufferBytes = 0;
};
struct SlabSlot {

3
core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp
View File

@ -24,6 +24,7 @@ class Recorder;
class DescriptorCache;
class SamplerCache;
class ShaderResources;
class PlaceholderImages;
inline void Require(bool condition, const std::string& reason) {
if (!condition) throw std::runtime_error("AGC graphics: " + reason);
@ -142,6 +143,7 @@ struct Context {
bool samplerFilterMinmax = false;
bool nonSeamlessCubeMap = false;
bool conservativeRasterization = false;
bool meshWave32 = false;
VkBuffer emptyBuffer = VK_NULL_HANDLE;
// The device's list of recorded dispatches whose copied written buffers await a CPU write-back
// (VulkanDevice's State::copiedWriters; the draw counterpart is DrawCopiedWriters): an indirect
@ -158,6 +160,7 @@ struct Context {
bool sampleRateShading = false;
bool nullDescriptors = false;
bool bufferInt64Atomics = false;
const PlaceholderImages* placeholderImages = nullptr;
bool primitiveListRestart = false;
bool imageViewMinLod = false;
bool pipelineExecutableInfo = false;

2
core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp
View File

@ -19,7 +19,7 @@ struct Context;
// uncompressed here, so the keys that matter are the fast-clear codes a title writes into the metadata
// (the surface then reads as a constant whatever its texels hold) and "uncompressed", which the driver
// stores after it writes a surface so later reads see the texels.
enum class DccKeys { Uncompressed, Clear0000, Clear0001, Clear1110, Clear1111, ClearRegister, Mixed, Unreadable };
enum class DccKeys { Uncompressed, Clear0000, Clear0001, Clear1110, Clear1111, ClearRegister, ClearSingle, Mixed, Unreadable };
const char* DccKeysName(DccKeys keys);
std::size_t DccKeyBytes(std::uint64_t surfaceBytes);

5
core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp
View File

@ -18,6 +18,11 @@ void ClearDepthSurfaces(VkDevice device);
bool DepthSurfaceAt(std::uint64_t address);
std::uint64_t HtileDepthClearAddress(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const std::array<std::uint32_t, 3>& numThreads);
void NoteHtileDepthClear(std::uint64_t htileAddress);
bool DepthSurfaceHolds(const Context& context, const GuestTextureResource& resource, const Texture* texture);
void NoteDepthMetadataFill(std::uint64_t address, std::size_t bytes, std::uint32_t pattern);
VkImageAspectFlags HtileFillClears(std::uint32_t pattern, bool stencilInHtile);
std::uint64_t HtileMetadataBytes(VkExtent2D extent);
bool HtileFillCovers(std::uint64_t htile, VkExtent2D extent, std::uint64_t address, std::size_t bytes);
std::shared_ptr<Texture> DepthSurfaceTexture(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource, VkComponentMapping components);
}

14
core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp
View File

@ -12,18 +12,18 @@ public:
~GpuColorTransfer();
GpuColorTransfer(const GpuColorTransfer&) = delete;
GpuColorTransfer& operator=(const GpuColorTransfer&) = delete;
void Upload(std::uint64_t address, std::uint32_t width, std::uint32_t height, ColorTileMode mode);
void Detile(VkCommandBuffer commands, bool swapRedBlue = false, bool tenBit = false);
void DetileImage(VkCommandBuffer commands, VkImage image, VkImageLayout layout, std::uint32_t width, std::uint32_t height, ColorTileMode mode, bool swapRedBlue, bool tenBit);
void Upload(std::uint64_t address, std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t elementBytes = 4);
void Detile(VkCommandBuffer commands, bool swapRedBlue = false, bool tenBit = false, bool halfFloat = false);
void DetileImage(VkCommandBuffer commands, VkImage image, VkImageLayout layout, std::uint32_t width, std::uint32_t height, ColorTileMode mode, bool swapRedBlue, bool tenBit, bool halfFloat = false);
void Tile(VkCommandBuffer commands);
void WriteBack(std::uint64_t address);
VkBuffer LinearBuffer() const;
RenderTarget& Target(const ColorTarget& color, bool blending);
private:
void prepare(std::uint32_t width, std::uint32_t height, ColorTileMode mode);
void convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit = false);
void dispatch(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit, bool tiledSource);
void prepare(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t elementBytes);
void convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit = false, bool halfFloat = false);
void dispatch(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit, bool tiledSource, bool halfFloat);
void release() noexcept;
Context context;
VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE;
@ -35,9 +35,11 @@ private:
std::unique_ptr<Buffer> tiled;
std::unique_ptr<DeviceBuffer> tiledDevice;
std::unique_ptr<DeviceBuffer> linear;
std::unique_ptr<Buffer> thresholds;
std::uint32_t width = 0;
std::uint32_t height = 0;
ColorTileMode mode = ColorTileMode::Linear;
std::uint32_t elementBytes = 4;
std::unique_ptr<RenderTarget> target;
VkExtent2D targetExtent{};
VkFormat targetFormat = VK_FORMAT_UNDEFINED;

22
core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp
View File

@ -28,6 +28,7 @@ struct HostImport {
VkDeviceAddress address;
void* alias = nullptr;
std::shared_ptr<const GuestAllocations::Range> range {};
std::vector<std::shared_ptr<const GuestAllocations::Range>> ranges {};
// Identity for the life of this import (see HostImportSerial); 0 until first asked for.
std::uint64_t serial = 0;
bool unwatched = false;
@ -44,6 +45,7 @@ struct ImportProbe {
std::uint32_t writtenAtImport = 0;
std::uint32_t writtenAfterSubmit = 0;
std::uint32_t writtenByCpu = 0;
std::uint32_t writtenAfterFill = 0;
};
ImportProbe ProbeImportWriteProtection(const Context& context);
@ -55,6 +57,7 @@ void SetImportWatch(const Context& context, ImportWatch watch);
// (alignment and budget permitting), or null. Bytes at `address` are at `address - import->base` in
// the import's buffer.
const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes);
bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice);
// Whether an existing import covers [address, address + bytes), without reconciling the imports
// with the registry or making one (HostImportFor may take a registry lease): a hint for choices
// made outside the device lock (a sampled texture's path, a dispatch's pre-sync); the path taken
@ -63,6 +66,7 @@ bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t
// Whether a readable registered allocation contains [address, address + bytes) right now (one
// registry lease): a storage image whose memory was freed or re-registered has nothing to store to.
bool RegisteredReadableCovers(std::uint64_t address, std::size_t bytes);
std::uint64_t RegisteredReadableEnd(std::uint64_t address);
// A persistent device copy of one registered range of the main guest image (which cannot be host
// imported); see GuestBufferMemory.cpp.
@ -153,6 +157,7 @@ struct MirrorStats {
std::uint64_t heapRefills = 0;
std::uint64_t sweeps = 0;
std::uint64_t heapChecks = 0;
std::uint64_t sparseBinds = 0;
};
MirrorStats MirrorCounters();
void ClearImageMirrors(VkDevice device);
@ -183,7 +188,7 @@ public:
// (DescriptorBinding::bufferWritten): the CPU never has to wait for it. `atomic` marks an
// element the shader updates atomically (DescriptorBinding::bufferAtomic), staged in device
// memory whatever its size (see AllowDeviceStaging).
void AddWritable(std::uint64_t address, std::size_t bytes, bool atomic = false);
void AddWritable(std::uint64_t address, std::size_t bytes, bool atomic = false, bool swept = false);
void AddReadable(std::uint64_t address, std::size_t bytes);
// Device-local staging of written and atomic elements inside host imports (see
// GuestBufferMemory.cpp): allowed only for a build whose every use records its work and then
@ -243,6 +248,11 @@ public:
// Whether registered allocations are pinned until write-back (address-based shaders): by this
// build's own lease, or by the cached address space it holds.
bool HoldsLease() const { return !lease.empty() || space != nullptr; }
std::size_t CopiedBytes() const {
std::size_t bytes = 0;
for (const auto& region : regions) if (region.buffer != nullptr) bytes += static_cast<std::size_t>((region.bufferEnd != 0 ? region.bufferEnd : region.end) - region.begin);
return bytes;
}
// Every uploaded region as [begin, end) when all of them are served by host imports, in place or
// through a device-local staging copy of the import (nothing was copied through the CPU, so the
// upload can serve a later identical build), else nothing.
@ -273,7 +283,10 @@ private:
// Set when the region covers uncommitted pages: only the `backed` parts (possibly none) are guest
// memory; the rest reads as zeros and is never stored.
bool sparse = false;
bool sparseHeap = false;
std::vector<std::pair<std::uint64_t, std::uint64_t>> backed {};
std::uint64_t lastView = 0;
std::uint64_t bufferEnd = 0;
// Set when the region is served by an image mirror: nothing is copied; writable mirrors are
// written back by comparing with the mirror's shadow. Kept alive here for recorded work.
std::shared_ptr<ImageMirror> mirror {};
@ -295,6 +308,7 @@ private:
bool copiedBack = false;
// An element the shader updates atomically lies inside (AddWritable's `atomic`).
bool atomic = false;
bool swept = false;
// The gpuCopy buffer is a device-local staging shadow (see stagingEligible): no host
// mapping, so nothing is ever stored from it by the CPU, and the region is taken even when
// the import could bind it in place.
@ -324,11 +338,15 @@ private:
void addCopiedRange(const CopiedRange& range);
// The BDA table entry of an uploaded region.
static ShaderRecompiler::BdaAbi::Range addressRange(const Region& region);
static std::uint64_t servedEnd(const Region& region);
void validate(std::uint64_t address, std::size_t bytes) const;
// The region of a descriptor-bound range (AddWritable/AddReadable), committed pages only. A
// range inside a base region of the space adds nothing: the region serves it, as today's merge
// of the two did.
void addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic);
void addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic, bool swept);
static bool servesView(const Region& region, std::uint64_t view);
static bool sparseHeapCandidate(const Region& region);
bool bindSparseMirror(Region& region, const GuestAllocations::Lease& lease);
// Gives a region a buffer of its own with its bytes (guest memory for host-backed and writable
// ranges, plus the write-back's reference copy for a range a descriptor writes; else its snapshot).
void copyRegion(Region& region, bool addressable);

2
core/libs/prx/libSceAgcDriver/Graphics/include/GuestSamplerResource.hpp
View File

@ -33,7 +33,7 @@ struct GuestSamplerResource {
bool nonSeamlessCube = false;
};
GuestSamplerResource DecodeSamplerResource(std::span<const std::uint32_t> words, bool unnormalizedProven = false, bool forceDegammaPaired = false);
GuestSamplerResource DecodeSamplerResource(std::span<const std::uint32_t> words, bool unnormalizedProven = false, bool forceDegammaPaired = false, bool integerBorder = false);
std::optional<std::array<std::uint32_t, 4>> SingleLevelSamplerWords(std::span<const std::uint32_t, 4> words, bool singleLevelImage, bool mipmappedImage);
}

60
core/libs/prx/libSceAgcDriver/Graphics/include/HalfFloatScanout.hpp Normal file
View File

@ -0,0 +1,60 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_HALFFLOATSCANOUT_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_HALFFLOATSCANOUT_HPP
#include <array>
#include <cmath>
#include <cstdint>
namespace AgcDriver::Graphics {
inline constexpr std::uint32_t HalfFloatScanoutCodes = 255;
inline constexpr std::uint32_t HalfFloatPositiveInfinity = 0x7c00u;
inline const std::array<std::uint32_t, 2 * HalfFloatScanoutCodes>& HalfFloatScanoutThresholds() {
static const auto thresholds = [] {
std::array<std::uint32_t, 2 * HalfFloatScanoutCodes> table{};
const auto finiteValue = [](std::uint32_t bits) {
const auto exponent = static_cast<int>(bits >> 10u);
const auto mantissa = static_cast<double>(bits & 0x3ffu);
return exponent == 0 ? std::ldexp(mantissa, -24) : std::ldexp(1024.0 + mantissa, exponent - 25);
};
const auto firstAtLeast = [&](double value) {
std::uint32_t low = 0;
std::uint32_t high = HalfFloatPositiveInfinity;
while (low < high) {
const auto middle = (low + high) / 2u;
if (finiteValue(middle) >= value) high = middle;
else low = middle + 1u;
}
return low;
};
for (std::uint32_t code = 1; code <= HalfFloatScanoutCodes; ++code) {
const double encoded = (code - 0.5) / 255.0;
table[code - 1u] = firstAtLeast(encoded <= 0.04045 ? encoded / 12.92 : std::pow((encoded + 0.055) / 1.055, 2.4));
table[HalfFloatScanoutCodes + code - 1u] = firstAtLeast(encoded);
}
return table;
}();
return thresholds;
}
inline std::uint8_t HalfFloatScanoutCode(std::uint16_t bits, bool alpha) {
static std::array<std::array<std::uint8_t, 65536>, 2> codes{};
static const bool filled = [] {
const auto& thresholds = HalfFloatScanoutThresholds();
for (std::uint32_t kind = 0; kind < 2u; ++kind) {
for (std::uint32_t half = 0; half <= HalfFloatPositiveInfinity; ++half) {
std::uint32_t code = 0;
while (code < HalfFloatScanoutCodes && thresholds[kind * HalfFloatScanoutCodes + code] <= half) ++code;
codes[kind][half] = static_cast<std::uint8_t>(code);
}
}
return true;
}();
static_cast<void>(filled);
return codes[alpha ? 1u : 0u][bits];
}
}
#endif

3
core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp
View File

@ -54,6 +54,8 @@ public:
bool SplitsFaces() const { return backFaces != VK_NULL_HANDLE; }
bool Optimized() const { return optimized != nullptr && optimized->handle.load() != VK_NULL_HANDLE; }
void ContinueBackFaces(VkCommandBuffer commands, const State& state) const;
bool SplitsSecondChannel() const { return secondChannel != VK_NULL_HANDLE; }
void ContinueSecondChannel(VkCommandBuffer commands) const;
void PushConstants(VkCommandBuffer commands, VkShaderStageFlags stages, std::span<const std::byte, PipelinePushConstantBytes> bytes) const;
// Forgets the Vulkan objects without destroying them: for entries of a device that is already gone.
void Abandon() noexcept;
@ -73,6 +75,7 @@ private:
VkRenderPass renderPass = VK_NULL_HANDLE;
VkPipeline pipeline = VK_NULL_HANDLE;
VkPipeline backFaces = VK_NULL_HANDLE;
VkPipeline secondChannel = VK_NULL_HANDLE;
std::size_t attachments = 0;
std::size_t colorAttachments = 0;
bool depthBounds = false;

2
core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp
View File

@ -31,7 +31,7 @@ struct OptimizedPipeline {
void Release() noexcept;
};
VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr<OptimizedPipeline>* optimized = nullptr);
VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr<OptimizedPipeline>* optimized = nullptr, std::span<VkShaderModule> modules = {});
void WaitForOptimizedPipelines(VkDevice device);

75
core/libs/prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp Normal file
View File

@ -0,0 +1,75 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PLACEHOLDERIMAGES_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PLACEHOLDERIMAGES_HPP
#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
#include "RuntimeAbi.hpp"
#include <array>
#include <cstdint>
namespace AgcDriver::Graphics {
inline constexpr std::uint32_t PlaceholderDimensionsPerClass = (ShaderRecompiler::RuntimeAbi::FirstComparisonImageBinding - ShaderRecompiler::RuntimeAbi::FirstImageBinding) / 3u;
static_assert(ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding == ShaderRecompiler::RuntimeAbi::FirstComparisonImageBinding + PlaceholderDimensionsPerClass);
static_assert(ShaderRecompiler::RuntimeAbi::ImageBindingCount == 8u * PlaceholderDimensionsPerClass && PlaceholderDimensionsPerClass == 7u);
struct PlaceholderImageShape {
VkFormat format;
VkFormatFeatureFlags features;
VkImageUsageFlags usage;
VkImageAspectFlags aspect;
VkImageType type;
VkImageViewType viewType;
VkSampleCountFlagBits samples;
};
constexpr PlaceholderImageShape PlaceholderImageShapeOf(std::uint32_t heap) {
constexpr std::array<VkFormat, 8> formats{VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_R32_UINT, VK_FORMAT_R32_SINT, VK_FORMAT_D16_UNORM,
VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_R32_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R64_UINT};
constexpr std::array<VkImageType, PlaceholderDimensionsPerClass> types{VK_IMAGE_TYPE_1D, VK_IMAGE_TYPE_1D, VK_IMAGE_TYPE_2D, VK_IMAGE_TYPE_2D,
VK_IMAGE_TYPE_2D, VK_IMAGE_TYPE_2D, VK_IMAGE_TYPE_3D};
constexpr std::array<VkImageViewType, PlaceholderDimensionsPerClass> views{VK_IMAGE_VIEW_TYPE_1D, VK_IMAGE_VIEW_TYPE_1D_ARRAY, VK_IMAGE_VIEW_TYPE_2D,
VK_IMAGE_VIEW_TYPE_2D_ARRAY, VK_IMAGE_VIEW_TYPE_2D, VK_IMAGE_VIEW_TYPE_2D_ARRAY,
VK_IMAGE_VIEW_TYPE_3D};
constexpr std::array<VkSampleCountFlagBits, PlaceholderDimensionsPerClass> samples{VK_SAMPLE_COUNT_1_BIT, VK_SAMPLE_COUNT_1_BIT, VK_SAMPLE_COUNT_1_BIT,
VK_SAMPLE_COUNT_1_BIT, VK_SAMPLE_COUNT_4_BIT, VK_SAMPLE_COUNT_4_BIT,
VK_SAMPLE_COUNT_1_BIT};
const auto index = heap - ShaderRecompiler::RuntimeAbi::FirstImageBinding;
const auto kind = index / PlaceholderDimensionsPerClass;
const auto format = formats[kind];
const auto dimension = index % PlaceholderDimensionsPerClass;
const bool storage = heap >= ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding;
const bool atomic = kind >= 6u;
const VkFormatFeatureFlags access = atomic ? VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT | VK_FORMAT_FEATURE_STORAGE_IMAGE_ATOMIC_BIT
: storage ? VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT
: VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT;
return {format,
static_cast<VkFormatFeatureFlags>(access | VK_FORMAT_FEATURE_TRANSFER_DST_BIT),
static_cast<VkImageUsageFlags>((storage ? VK_IMAGE_USAGE_STORAGE_BIT : VK_IMAGE_USAGE_SAMPLED_BIT) | VK_IMAGE_USAGE_TRANSFER_DST_BIT),
static_cast<VkImageAspectFlags>(format == VK_FORMAT_D16_UNORM ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT),
types[dimension],
views[dimension],
samples[dimension]};
}
class PlaceholderImages {
public:
explicit PlaceholderImages(const Context& context);
~PlaceholderImages();
PlaceholderImages(const PlaceholderImages&) = delete;
PlaceholderImages& operator=(const PlaceholderImages&) = delete;
VkImageView View(std::uint32_t heap) const;
private:
struct Image {
VkImage image = VK_NULL_HANDLE;
VkDeviceMemory memory = VK_NULL_HANDLE;
VkImageView view = VK_NULL_HANDLE;
};
void release() noexcept;
Context context;
std::array<Image, ShaderRecompiler::RuntimeAbi::ImageBindingCount> images{};
};
}
#endif

6
core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp
View File

@ -17,6 +17,7 @@
#include <optional>
#include <span>
#include <tuple>
#include <unordered_set>
#include <utility>
#include <vector>
@ -85,6 +86,9 @@ public:
// Whether recorded work still has completion actions (write-backs the CPU must see) to run.
bool HasCompletions() const;
void Keep(std::shared_ptr<void> object, std::size_t bytes = 0);
void KeepBytes(const void* owner, std::size_t bytes);
std::size_t OpenKeptBytes() const { return open != nullptr ? open->keptBytes : 0; }
void FlushDeferredReleases();
static constexpr std::size_t KeptBytesBudget = std::size_t{512} << 20u;
void BoundKeptBytes();
std::size_t InFlightKeptBytes() const { return inFlightKeptBytes; }
@ -379,6 +383,7 @@ public:
// The recorder of the current device, for code that only has guest addresses (the flush hook)
// and for helpers that must not recycle resources the recorded work still uses.
static Recorder* Active();
VkDevice Device() const { return context.device; }
// Whether the calling thread runs inside a batch's completion action (finish()): a store made
// there must not have a publish recorded for it (UnitShadow), see SyncThrough.
static bool InCompletion();
@ -511,6 +516,7 @@ private:
VkFence fence = VK_NULL_HANDLE;
std::vector<std::shared_ptr<void>> kept;
std::size_t keptBytes = 0;
std::unordered_set<const void*> keptOwners;
std::vector<std::function<void()>> completions;
std::vector<std::pair<std::uint64_t, std::uint64_t>> writes;
std::vector<std::uint64_t> writeNotes;

1
core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp
View File

@ -29,7 +29,6 @@ private:
VkDeviceMemory memory = VK_NULL_HANDLE;
void* mapping = nullptr;
std::size_t size;
// The VkBuffer's own size, `size` rounded up to its pool class (see BufferPool::Capacity).
std::size_t capacity;
// Fully made (or taken from the pool), so release returns it to the pool instead of destroying
// the handles a failed construction left.

2
core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp
View File

@ -47,7 +47,7 @@ public:
explicit SamplerCache(std::size_t capacity = 1024);
SamplerCache(const SamplerCache&) = delete;
SamplerCache& operator=(const SamplerCache&) = delete;
std::shared_ptr<Sampler> Get(const Context& context, std::span<const std::uint32_t> words, bool compareEnable, bool unnormalizedProven = false, bool forceDegammaPaired = false);
std::shared_ptr<Sampler> Get(const Context& context, std::span<const std::uint32_t> words, bool compareEnable, bool unnormalizedProven = false, bool forceDegammaPaired = false, bool integerBorder = false);
// APS5_PROFILE_DRAW counters: lookups served by an existing sampler, and samplers created.
std::uint64_t Hits() const { return hits; }
std::uint64_t Misses() const { return misses; }

17
core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp
View File

@ -109,7 +109,7 @@ public:
// then no other member may be used. `compute` and `snapshots` must outlive Complete(). An
// address-based shader (BDA tables) builds entirely in Complete(): its lease acquisition
// reconciles imports and refreshes mirrors, which needs the lock.
ShaderResources(const Context& context, const CompiledShader& compute, std::span<const GuestMemorySnapshot> snapshots, bool deferred);
ShaderResources(const Context& context, const CompiledShader& compute, std::span<const GuestMemorySnapshot> snapshots, bool deferred, std::uint64_t dispatchThreads = 0);
void Complete();
bool Completed() const { return completed; }
~ShaderResources();
@ -149,6 +149,8 @@ public:
// Whether a written buffer was copied (its results reach guest memory by the CPU write-back).
bool HasCopiedWrites() const { return guestMemory.HasCopiedWrites(); }
bool HoldsLease() const { return guestMemory.HoldsLease(); }
bool UsesAddressTable() const { return usesBda; }
std::size_t CopiedBytes() const { return guestMemory.CopiedBytes(); }
bool WritesOverlap(std::uint64_t address, std::size_t bytes) const { return guestMemory.WritesOverlap(address, bytes); }
// Whether a region the recorded work reads in place through a host import overlaps the range.
bool ReadsOverlap(std::uint64_t address, std::size_t bytes) const;
@ -186,6 +188,8 @@ public:
bool Reusable() const { return reusable; }
static bool NeverReusable(std::span<const CompiledShader> shaders);
const std::vector<std::shared_ptr<Texture>>& SampledTextures() const { return textures; }
bool ChecksFaults() const { return bda != nullptr; }
void CheckFaults() const;
// `shaders` are the stages the object was built from, in build order (a recorded draw's vertex
// and fragment stages, or one compute stage): their bindings are walked like the build did.
// How a Revalidate proved (or refused) the object, for the [recipe] line: the proof path taken
@ -213,7 +217,7 @@ public:
bool DataWordsDiffer(const CompiledShader& shader) const;
// Why the fast proof of a Revalidate left the object to the full walk (the [rescache]
// revalidate line's reasons); Count: it did not.
enum class FastFail : std::size_t { NoRecord, Collect, Pending, Evicted, Changed, Keys, ClearedView, StorageKeys, Count };
enum class FastFail : std::size_t { NoRecord, Collect, Pending, Evicted, Changed, Keys, ClearedView, StorageKeys, Depth, Count };
// Why a Pending failure was left to the full walk instead of the own-object refresh (T1, see
// refreshOwnObjects); Count: it was not.
enum class OwnRefreshFallback : std::size_t { Disabled, Snapshot, Keys, ForeignView, SurfaceKey, NotImported, Uncached, Rerun, Count };
@ -302,7 +306,7 @@ private:
// `written` is the element's DescriptorBinding::bufferWritten: a read-only element binds the
// same way but is left out of the write set (no write-back, no pending-write note). `atomic`
// is its bufferAtomic (see GuestBufferMemory::AddWritable).
std::size_t addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic);
std::size_t addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read);
std::size_t addDataBuffer(std::span<const std::uint32_t> words);
// Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache
// locks itself); stage B looks the sampled textures and storage images up (resolveImageBinding).
@ -371,6 +375,7 @@ private:
std::uint64_t collected = 0;
// Storage-sourced textures: the image the view follows (kept alive by the texture).
const StorageTexture* source = nullptr;
bool depth = false;
bool valid = false;
};
void captureValidation();
@ -421,11 +426,14 @@ private:
// The cache pool the set was allocated from, freed back to it on release.
VkDescriptorPool cachePool = VK_NULL_HANDLE;
std::vector<Allocation> allocations;
std::vector<VkDescriptorBufferInfo> allocationDescriptors;
std::vector<VkDescriptorBufferInfo> descriptorBuffers;
std::vector<VkDescriptorImageInfo> descriptorImages;
std::vector<VkWriteDescriptorSet> descriptorWrites;
// Guest buffer elements bound read-only: each use of this object skips that many pending-write
// notes (counted in MarkGpuWrites for the [buffers] line).
std::size_t readOnlyBuffers = 0;
bool drawBuild = false;
std::uint64_t dispatchThreads = 0;
std::vector<std::shared_ptr<Texture>> textures;
std::vector<bool> textureFirstLayer;
std::vector<std::shared_ptr<StorageTexture>> storageTextures;
@ -441,6 +449,7 @@ private:
std::array<std::uint32_t, 4> words{};
bool compareEnable = false;
bool unnormalized = false;
bool integerBorder = false;
bool singleLevelImage = false;
bool mipmappedImage = false;
};

7
core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp
View File

@ -67,6 +67,7 @@ struct DepthTarget {
float clearDepth;
std::uint8_t clearStencil;
std::uint64_t htileAddress = 0;
bool htileStencil = false;
};
struct State {
@ -100,6 +101,7 @@ struct State {
VkViewport viewport;
bool negativeOneToOne;
bool depthClamp = false;
bool alphaToCoverage = false;
VkConservativeRasterizationModeEXT conservativeRasterization = VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT;
VkRect2D scissor;
VkCullModeFlags cullMode;
@ -109,8 +111,8 @@ struct State {
std::array<float, 4> blendConstants;
};
ShaderStages DecodeShaderStages(const QueueState& queue);
State DecodeState(const QueueState& queue);
ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup = false);
State DecodeState(const QueueState& queue, bool vertexWorkgroup = false);
std::array<std::uint8_t, 8> ExportMappings(const State& state);
std::array<ShaderRecompiler::ColorExportPacking, 8> ExportPackings(const State& state);
ColorTarget DecodeColorBuffer(const Registers& context, std::uint32_t slot);
@ -179,6 +181,7 @@ inline constexpr std::array<DrawKeyRange, 47> DrawKeyRegisters{{
// GE_PRIM_TYPE, GE_MULTI_PRIM_IB_RESET_EN, the geometry subgroup sizes.
{RegisterBank::UserConfig, 0x242, 1}, {RegisterBank::UserConfig, 0x24b, 1}, {RegisterBank::UserConfig, 0x25b, 1},
}};
bool SplitsAlternateAlpha(const State& state);
// Whether DrawKeyRegisters holds the read.
bool DrawKeyCovers(RegisterRead read);

18
core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp
View File

@ -61,6 +61,17 @@ public:
// CanCopyFrom says whether the two descriptors address the same surface compatibly.
Texture(const Context& context, const std::shared_ptr<StorageTexture>& source, const GuestTextureResource& descriptor, VkComponentMapping components);
Texture(const Context& context, VkImage depthImage, VkFormat depthFormat, VkImageAspectFlags aspect, VkComponentMapping components);
struct DepthCopyTag {};
Texture(const Context& context, const std::shared_ptr<StorageTexture>& source, const GuestTextureResource& descriptor, VkComponentMapping components, DepthCopyTag);
static VkFormat DepthCopyFormat(VkFormat colorFormat);
static bool CanDepthCopyFrom(const StorageTexture& source, const GuestTextureResource& descriptor);
bool IsDepthCopy() const { return depthCopy; }
bool RefreshDepthCopy();
struct DepthCopyCounts {
std::uint64_t textures;
std::uint64_t refreshes;
};
static DepthCopyCounts TakeDepthCopyCounts();
static bool CanCopyFrom(const StorageTexture& source, const GuestTextureResource& descriptor);
~Texture();
Texture(const Texture&) = delete;
@ -108,6 +119,11 @@ private:
std::shared_ptr<ResidentColor> source;
std::shared_ptr<StorageTexture> storageSource;
std::unique_ptr<CommandBatch> upload;
bool depthCopy = false;
bool copiedOnce = false;
std::uint64_t copiedVersion = 0;
VkFormat depthFormat = VK_FORMAT_UNDEFINED;
std::uint32_t copyWidth = 0, copyHeight = 0, copyLevels = 0, copyLayers = 0, copyElementBytes = 0;
};
VkFormat AttachmentProxyFormat(const Context& context, VkFormat format);
@ -288,6 +304,7 @@ public:
// GuestMemory::GpuMutex only, as Refresh is; never from a build's stage A.
DccKeys UploadedKeys() const { return uploadedKeys; }
DccKeys FilledKeys() const { return filledKeys; }
bool GuestSnapshotValid() const { return originalValid; }
DccKeyProof& KeyProof() const { return keyProof; }
DccRangeProof& TargetKeyProof() const { return targetKeyProof; }
DccKeys ProvedKeys() const;
@ -317,6 +334,7 @@ private:
void writeBack(std::uint64_t address, std::size_t bytes);
void writeBackLayers(const std::vector<bool>& layers);
bool unchangedSinceBaseline(std::uint64_t from, std::uint64_t to) const;
bool guestBytesSettled() const;
// Tracked units as 64 KiB write-stamp blocks (`blockUnits`: a thin tiled surface at a 64 KiB
// aligned base; APS5_NO_BLOCK_TRACKING=1 tracks array layers as above instead): a fill of one
// layer, a CPU write or another image's store then costs the blocks it touched, moved through

1
core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp
View File

@ -13,6 +13,7 @@ namespace AgcDriver::Graphics {
struct Context;
VkFormat ResolveTextureFormat(std::uint32_t guestFormat);
VkComponentSwizzle TextureComponentChannel(std::uint32_t guestFormat, VkComponentSwizzle component);
std::uint32_t SrgbDecodeFormats(PFN_vkGetPhysicalDeviceFormatProperties formatProperties, VkPhysicalDevice physical);
VkFormat SampledTextureFormat(const Context& context, std::uint32_t guestFormat);
bool IsSrgbTextureFormat(std::uint32_t guestFormat);

25
core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp
View File

@ -246,6 +246,31 @@ inline std::optional<std::uint32_t> HighestDrawIndex(std::span<const std::byte>
return highest;
}
inline std::vector<std::uint32_t> MeshRestartTable(std::span<const std::byte> indices, std::uint32_t indexSize) {
Require(indexSize == 2 || indexSize == 4, "unsupported index size");
const auto restartIndex = indexSize == 2 ? 0xffffu : 0xffffffffu;
std::vector<std::uint32_t> starts(indices.size() / indexSize);
std::uint32_t stripStart = 0;
bool restarted = false;
for (std::size_t position = 0; position < starts.size(); ++position) {
std::uint32_t index = 0;
if (indexSize == 2) {
std::uint16_t value = 0;
std::memcpy(&value, indices.data() + position * 2u, sizeof(value));
index = value;
} else {
std::memcpy(&index, indices.data() + position * 4u, sizeof(index));
}
if (index == restartIndex) {
restarted = true;
stripStart = static_cast<std::uint32_t>(position) + 1u;
}
starts[position] = stripStart;
}
if (!restarted) starts.clear();
return starts;
}
inline std::vector<std::size_t> SoloZeroPaddedFetchIndices(const std::vector<VertexFetch>& fetches, const std::vector<std::size_t>& fetchValid) {
Require(fetchValid.size() == fetches.size(), "fetch validity does not match the fetch count");
std::vector<std::size_t> solo;

25
core/libs/prx/libSceAgcDriver/Graphics/shaders/ColorTransfer.comp
View File

@ -3,6 +3,7 @@
layout(local_size_x = 8, local_size_y = 8) in;
layout(set = 0, binding = 0, std430) buffer TiledPixels { uint tiled[]; };
layout(set = 0, binding = 1, std430) buffer LinearPixels { uint linear[]; };
layout(set = 0, binding = 2, std430) readonly buffer ScanoutThresholds { uint thresholds[]; };
layout(push_constant) uniform Parameters {
uint width;
uint height;
@ -10,6 +11,15 @@ layout(push_constant) uniform Parameters {
uint mode;
} parameters;
uint scanoutCode(uint bits, uint base) {
if (bits > 0x7c00u) return 0u;
uint code = 0u;
for (uint step = 128u; step != 0u; step >>= 1u) {
if (code + step <= 255u && thresholds[base + code + step - 1u] <= bits) code += step;
}
return code;
}
void main() {
uint x = gl_GlobalInvocationID.x;
uint y = gl_GlobalInvocationID.y;
@ -21,7 +31,20 @@ void main() {
^ ((x << 2u) & 0x000cu) ^ ((x << 5u) & 0x0380u) ^ ((x << 4u) & 0x0400u) ^ ((x << 6u) & 0x0800u) ^ ((x << 9u) & 0xa000u);
tiledIndex = ((y / 128u) * parameters.blocksPerRow + x / 128u) * 16384u + offset / 4u;
}
uint pixel = (parameters.mode & 1u) != 0u ? linear[linearIndex] : tiled[tiledIndex];
uint pixel;
if ((parameters.mode & 16u) != 0u) {
uint texel = linearIndex * 2u;
if ((parameters.mode & 4u) != 0u) {
uint offset = ((x << 3u) & 0x0008u) ^ ((x << 4u) & 0x0460u) ^ ((x << 5u) & 0x0300u) ^ ((x << 6u) & 0x0800u) ^ ((x << 9u) & 0x8000u) ^ ((x << 10u) & 0x2000u)
^ ((y << 4u) & 0x0010u) ^ ((y << 6u) & 0x0080u) ^ ((y << 5u) & 0x0f00u) ^ ((y << 10u) & 0x5000u);
texel = (((y / 64u) * parameters.blocksPerRow + x / 128u) * 65536u + offset) / 4u;
}
uint first = tiled[texel];
uint second = tiled[texel + 1u];
pixel = scanoutCode(first & 0xffffu, 0u) | (scanoutCode(first >> 16u, 0u) << 8u) | (scanoutCode(second & 0xffffu, 0u) << 16u) | (scanoutCode(second >> 16u, 255u) << 24u);
} else {
pixel = (parameters.mode & 1u) != 0u ? linear[linearIndex] : tiled[tiledIndex];
}
// Mode 8: A2B10G10R10 source (red in the low bits) presented as 8-bit RGBA.
if ((parameters.mode & 8u) != 0u) pixel = ((pixel >> 2u) & 0xffu) | (((pixel >> 12u) & 0xffu) << 8u) | (((pixel >> 22u) & 0xffu) << 16u) | 0xff000000u;
if ((parameters.mode & 2u) != 0u) pixel = (pixel & 0xff00ff00u) | ((pixel & 0xffu) << 16u) | ((pixel >> 16u) & 0xffu);

20
core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp
View File

@ -124,7 +124,7 @@ BdaResources::BdaResources(const Context& context) {
static_assert(std::endian::native == std::endian::little);
Require(ShaderRecompiler::BdaAbi::FaultBufferBytes <= context.limits.maxStorageBufferRange, "BDA fault buffer exceeds storage buffer range limit");
fault = std::make_unique<Buffer>(context, ShaderRecompiler::BdaAbi::FaultBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
std::memset(fault->Bytes().data(), 0, fault->Bytes().size());
std::memset(fault->Bytes().data(), 0, ShaderRecompiler::BdaAbi::WrittenListWord * sizeof(std::uint32_t));
}
BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memory) : BdaResources(context) {
@ -192,6 +192,7 @@ BdaResources::BdaResources(const Context& context, const GuestBufferMemory& memo
++it;
}
}
for (const auto& range : ranges) Require((range.permissions & ShaderRecompiler::BdaAbi::Write) == 0 || range.end <= ShaderRecompiler::BdaAbi::WrittenAddressLimit, "a writable BDA range ends past the addresses the written-page record can hold");
table = std::make_shared<Buffer>(context, tableBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
const ShaderRecompiler::BdaAbi::Header header{ShaderRecompiler::BdaAbi::Version, static_cast<std::uint32_t>(ranges.size()), sizeof(ShaderRecompiler::BdaAbi::Range), 0};
std::memcpy(table->Bytes().data(), &header, sizeof(header));
@ -276,15 +277,16 @@ namespace AgcDriver::Graphics {
void BdaResources::markWrittenPages() const {
namespace Abi = ShaderRecompiler::BdaAbi;
auto* words = reinterpret_cast<std::uint32_t*>(fault->Bytes().data());
Require(words[Abi::WrittenOverflowWord] == 0, "more than " + std::to_string(Abi::WrittenPageSlots) + " pages stored to through the BDA table in one use are not implemented");
bool any = false;
for (std::uint32_t slot = 0; slot < Abi::WrittenPageSlots; ++slot) {
const auto page = words[Abi::WrittenSlotsWord + slot];
if (page == 0) continue;
GuestMemory::MarkWritten(static_cast<std::uint64_t>(page - 1u) << Abi::WrittenPageShift, std::size_t{1} << Abi::WrittenPageShift);
any = true;
Require(words[Abi::WrittenOverflowWord] == 0, "more than " + std::to_string(Abi::WrittenPageSlots) + " chunks of " + std::to_string(Abi::WrittenChunkPages) + " pages stored to through the BDA table in one use are not implemented");
const auto count = words[Abi::WrittenCountWord];
if (count == 0) return;
Require(Abi::ForEachWrittenRange(words, [](std::uint64_t address, std::size_t bytes) { GuestMemory::MarkWritten(address, bytes); }), "invalid BDA written-page record");
for (std::uint32_t claim = 0; claim < count; ++claim) {
const auto slot = words[Abi::WrittenListWord + claim];
words[Abi::WrittenSlotsWord + 2 * slot] = 0;
words[Abi::WrittenSlotsWord + 2 * slot + 1] = 0;
}
if (any) std::memset(words + Abi::WrittenSlotsWord, 0, Abi::WrittenPageSlots * sizeof(std::uint32_t));
words[Abi::WrittenCountWord] = 0;
}
}

2
core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp
View File

@ -51,6 +51,7 @@ DccKeys ByteKeys(std::uint8_t key) {
case 0x80: return DccKeys::Clear1110;
case 0xc0: return DccKeys::Clear1111;
case 0x20: return DccKeys::ClearRegister;
case 0x10: return DccKeys::ClearSingle;
case 0xff: return DccKeys::Uncompressed;
default: return DccKeys::Mixed;
}
@ -384,6 +385,7 @@ const char* DccKeysName(DccKeys keys) {
case DccKeys::Clear1110: return "1110";
case DccKeys::Clear1111: return "1111";
case DccKeys::ClearRegister: return "register";
case DccKeys::ClearSingle: return "single";
case DccKeys::Mixed: return "mixed";
case DccKeys::Unreadable: return "unreadable";
}

98
core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp
View File

@ -17,6 +17,11 @@
namespace AgcDriver::Graphics {
namespace {
Recorder* RecorderFor(const Context& context) {
auto* recorder = Recorder::Active();
return recorder != nullptr && recorder->Device() == context.device ? recorder : nullptr;
}
class DepthSurface {
public:
DepthSurface(const Context& context, const DepthTarget& target) : context(context), target(target) {
@ -53,7 +58,7 @@ public:
viewInfo.format = target.format;
viewInfo.subresourceRange = {aspects, 0, 1, 0, 1};
Check(context.Function<PFN_vkCreateImageView>("vkCreateImageView")(context.device, &viewInfo, nullptr, &view), "vkCreateImageView depth");
auto* recorder = Recorder::Active();
auto* recorder = RecorderFor(context);
std::unique_ptr<CommandBatch> batch;
if (recorder == nullptr) batch = std::make_unique<CommandBatch>(context);
const auto commands = recorder != nullptr ? recorder->Commands() : batch->Handle();
@ -78,23 +83,28 @@ public:
}
}
void ClearDepth(float value) {
auto* recorder = Recorder::Active();
std::unique_ptr<CommandBatch> batch;
if (recorder == nullptr) batch = std::make_unique<CommandBatch>(context);
const auto commands = recorder != nullptr ? recorder->Commands() : batch->Handle();
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
const VkClearDepthStencilValue clear{value, 0};
const VkImageSubresourceRange range{VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, 1};
context.Function<PFN_vkCmdClearDepthStencilImage>("vkCmdClearDepthStencilImage")(commands, image, VK_IMAGE_LAYOUT_GENERAL, &clear, 1, &range);
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT);
if (batch) batch->SubmitAndWait();
else Recorder::CountBarriers(Recorder::CommandClass::Draw, 2);
}
~DepthSurface() { release(); }
DepthSurface(const DepthSurface&) = delete;
DepthSurface& operator=(const DepthSurface&) = delete;
void ApplyFastClear() {
const auto aspects = pendingClear & (VK_IMAGE_ASPECT_DEPTH_BIT | (target.stencilAddress != 0 ? VK_IMAGE_ASPECT_STENCIL_BIT : 0u));
pendingClear = 0;
if (aspects == 0) return;
auto* recorder = RecorderFor(context);
std::unique_ptr<CommandBatch> batch;
if (recorder == nullptr) batch = std::make_unique<CommandBatch>(context);
const auto commands = recorder != nullptr ? recorder->Commands() : batch->Handle();
const VkImageSubresourceRange range{aspects, 0, 1, 0, 1};
constexpr VkAccessFlags access = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, access, VK_ACCESS_TRANSFER_WRITE_BIT);
const VkClearDepthStencilValue clear{clearDepth, clearStencil};
context.Function<PFN_vkCmdClearDepthStencilImage>("vkCmdClearDepthStencilImage")(commands, image, VK_IMAGE_LAYOUT_GENERAL, &clear, 1, &range);
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, access);
if (batch) batch->SubmitAndWait();
else Recorder::CountBarriers(Recorder::CommandClass::Draw, 2);
}
std::shared_ptr<Texture> Sampled(std::span<const std::uint32_t> words, const GuestTextureResource& resource, VkComponentMapping components) {
std::array<std::uint32_t, 12> key{};
std::copy_n(words.begin(), std::min<std::size_t>(words.size(), 8), key.begin());
@ -120,6 +130,13 @@ public:
return texture;
}
bool Holds(const Texture* texture) const {
return std::any_of(textures.begin(), textures.end(), [&](const auto& entry) { return entry.second.get() == texture; });
}
VkImageAspectFlags pendingClear = 0;
float clearDepth = 0.0f;
std::uint8_t clearStencil = 0;
const Context context;
const DepthTarget target;
VkImage image = VK_NULL_HANDLE;
@ -174,7 +191,10 @@ VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target)
const bool cleared = target.htileAddress != 0 && clearedHtiles().erase(target.htileAddress) != 0;
for (const auto& surface : surfaces()) {
if (surface->context.device != context.device || !sameSurface(surface->target, target)) continue;
if (cleared) surface->ClearDepth(target.clearDepth);
surface->clearDepth = target.clearDepth;
surface->clearStencil = target.clearStencil;
if (cleared) surface->pendingClear |= VK_IMAGE_ASPECT_DEPTH_BIT;
surface->ApplyFastClear();
return surface->view;
}
surfaces().push_back(std::make_unique<DepthSurface>(context, target));
@ -191,7 +211,13 @@ std::uint64_t HtileDepthClearAddress(std::span<const std::uint32_t> code, std::s
void NoteHtileDepthClear(std::uint64_t htileAddress) {
std::lock_guard lock(surfacesMutex());
clearedHtiles().insert(htileAddress);
bool existing = false;
for (const auto& surface : surfaces()) {
if (surface->target.htileAddress != htileAddress) continue;
surface->pendingClear |= VK_IMAGE_ASPECT_DEPTH_BIT;
existing = true;
}
if (!existing) clearedHtiles().insert(htileAddress);
}
void ClearDepthSurfaces(VkDevice device) {
@ -213,10 +239,50 @@ std::shared_ptr<Texture> DepthSurfaceTexture(const Context& context, std::span<c
const bool d16 = target.format == VK_FORMAT_D16_UNORM || target.format == VK_FORMAT_D16_UNORM_S8_UINT;
const bool depthBits = words.size() >= 4 && ShaderRecompiler::DepthBitsTextureWidth(words[1], words[3]) == (d16 ? 16u : 32u);
if (ResolveTextureFormat(resource.format) != (d16 ? VK_FORMAT_R16_UNORM : VK_FORMAT_R32_SFLOAT) && !depthBits) return nullptr;
} else if (BytesPerElement(resource.format) != 1u) {
return nullptr;
}
(*found)->ApplyFastClear();
return (*found)->Sampled(words, resource, components);
}
bool DepthSurfaceHolds(const Context& context, const GuestTextureResource& resource, const Texture* texture) {
if (resource.dimension != TextureDimension::k2D || resource.baseArray != 0) return false;
const auto address = resource.baseAddress;
std::lock_guard lock(surfacesMutex());
const auto& list = surfaces();
const auto found = std::find_if(list.rbegin(), list.rend(), [&](const auto& surface) {
return surface->context.device == context.device && (surface->target.address == address || (surface->target.stencilAddress != 0 && surface->target.stencilAddress == address));
});
return found != list.rend() && (*found)->Holds(texture);
}
VkImageAspectFlags HtileFillClears(std::uint32_t pattern, bool stencilInHtile) {
VkImageAspectFlags cleared = (pattern & 0xfu) == 0 ? VK_IMAGE_ASPECT_DEPTH_BIT : 0u;
if (stencilInHtile && ((pattern >> 8u) & 0x3u) == 0) cleared |= VK_IMAGE_ASPECT_STENCIL_BIT;
return cleared;
}
std::uint64_t HtileMetadataBytes(VkExtent2D extent) {
const auto blocks = static_cast<std::uint64_t>((extent.width + 1023u) / 1024u) * ((extent.height + 511u) / 512u);
return blocks * 32768u;
}
bool HtileFillCovers(std::uint64_t htile, VkExtent2D extent, std::uint64_t address, std::size_t bytes) {
if (htile == 0 || htile < address || htile >= address + bytes) return false;
return address + bytes - htile >= HtileMetadataBytes(extent);
}
void NoteDepthMetadataFill(std::uint64_t address, std::size_t bytes, std::uint32_t pattern) {
std::lock_guard lock(surfacesMutex());
for (const auto& surface : surfaces()) {
const auto& target = surface->target;
if (!HtileFillCovers(target.htileAddress, target.extent, address, bytes)) continue;
const VkImageAspectFlags written = VK_IMAGE_ASPECT_DEPTH_BIT | (target.htileStencil ? VK_IMAGE_ASPECT_STENCIL_BIT : 0u);
surface->pendingClear = (surface->pendingClear & ~written) | HtileFillClears(pattern, target.htileStencil);
}
}
bool DepthSurfaceAt(std::uint64_t address) {
std::lock_guard lock(surfacesMutex());
return std::any_of(surfaces().begin(), surfaces().end(), [&](const auto& surface) { return surface->target.address == address || surface->target.stencilAddress == address; });

98
core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp
View File

@ -132,6 +132,25 @@ void storeClearTexels(const Context& context, const ColorTarget& color, const st
MarkDccUncompressed(context, color.dccAddress, color.bytes, colorKeyCount(color, color.bytes));
}
void storeSingleTexels(const Context& context, const ColorTarget& color) {
Require(color.tileMode != ColorTileMode::Linear, "comp-to-single DCC keys of a linear color target");
const ColorTargetLayout surface(color.extent.width, color.extent.height, color.tileMode, color.elementBytes, color.pipeBankXor);
Require(surface.Bytes() == color.bytes, "comp-to-single DCC keys over a color target whose layout differs from its surface");
StorageTexture::FlushPending(color.address, color.bytes, nullptr, "comp-to-single expansion");
std::vector<std::byte> texels(color.bytes);
GuestMemory::Read(color.address, texels, surface.Alignment());
constexpr std::size_t BlockBytes = 256;
for (std::uint32_t y = 0; y < color.extent.height; ++y) {
for (std::uint32_t x = 0; x < color.extent.width; ++x) {
const auto offset = surface.Offset(x, y);
const auto first = offset / BlockBytes * BlockBytes;
if (offset != first) std::memcpy(texels.data() + offset, texels.data() + first, color.elementBytes);
}
}
GuestMemory::Write(color.address, texels, surface.Alignment());
MarkDccUncompressed(context, color.dccAddress, color.bytes, colorKeyCount(color, color.bytes));
}
void materializeRegisterClear(const Context& context, const ColorTarget& color, StorageTexture& resident) {
if (color.dccAddress == 0 || resident.Descriptor().dccAddress != color.dccAddress) return;
if (ProvedCurrentDccKeys(color.dccAddress, color.bytes, resident.TargetKeyProof()) != DccKeys::ClearRegister) return;
@ -961,6 +980,7 @@ struct DrawInputs {
std::set<std::uint32_t> fragmentOutputs;
VkPipelineStageFlags shaderStages = 0;
std::uint32_t meshGroups = 0;
std::vector<std::uint32_t> restartTable;
};
std::shared_ptr<Buffer> zeroVertexBuffer(const Context& context, const ShaderRecompiler::VertexAttribute& attribute) {
@ -1062,6 +1082,10 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P
Require(*highest <= context.limits.maxDrawIndexedIndexValue, "index exceeds the device's indexed draw limit");
Require(!fanGeometry || !state.primitiveRestart || *highest != (draw.indexSize == 2 ? 0xffffu : 0xffffffffu), "primitive restart in a triangle fan geometry draw is unsupported");
inputs.maxIndex = *highest;
if (skipRestart && state.stages.mesh && state.stages.mesh->inputPrimitive == 6) {
const auto indexSpan = import != nullptr ? std::span<const std::byte>(reinterpret_cast<const std::byte*>(draw.indexAddress), static_cast<std::size_t>(indexBytes)) : copy.buffer->Bytes().first(static_cast<std::size_t>(indexBytes));
inputs.restartTable = MeshRestartTable(indexSpan, draw.indexSize);
}
if (import != nullptr) {
inputs.indexHandle = import->buffer;
inputs.indexOffset = draw.indexAddress - import->base;
@ -1363,10 +1387,30 @@ bool recordIndirectArguments(const Context& context, VkCommandBuffer commands, R
return true;
}
std::shared_ptr<Buffer> recordMeshArguments(const Context& context, VkCommandBuffer commands, Recorder* recorder, bool recorded, const State& state, const Pm4::DrawParameters& draw, const IndirectRecord& indirect, const std::function<void(std::uint32_t)>& countBarrier) {
std::shared_ptr<Buffer> meshArgumentBuffer(const Context& context, std::span<const std::uint32_t> restartTable) {
const auto tableBytes = restartTable.empty() ? 0u : sizeof(std::uint32_t) * (restartTable.size() + 1u);
auto arguments = std::make_shared<Buffer>(context, ShaderRecompiler::MeshArgumentBytes + tableBytes, VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
if (!restartTable.empty()) {
Require(restartTable.size() <= std::numeric_limits<std::uint32_t>::max(), "mesh restart table is too large");
const auto length = static_cast<std::uint32_t>(restartTable.size());
auto* bytes = arguments->Bytes().data() + ShaderRecompiler::MeshArgumentBytes;
std::memcpy(bytes, &length, sizeof(length));
std::memcpy(bytes + sizeof(length), restartTable.data(), restartTable.size_bytes());
}
return arguments;
}
std::shared_ptr<Buffer> directMeshArguments(const Context& context, const Pm4::DrawParameters& draw, std::span<const std::uint32_t> restartTable) {
auto arguments = meshArgumentBuffer(context, restartTable);
const MeshArguments resolved{0u, 0u, 0u, draw.indexCount, 0u};
std::memcpy(arguments->Bytes().data(), &resolved, sizeof(resolved));
return arguments;
}
std::shared_ptr<Buffer> recordMeshArguments(const Context& context, VkCommandBuffer commands, Recorder* recorder, bool recorded, const State& state, const Pm4::DrawParameters& draw, const IndirectRecord& indirect, const std::function<void(std::uint32_t)>& countBarrier, std::span<const std::uint32_t> restartTable) {
const auto* args = indirect.args;
const auto rules = MeshArgumentRulesFor(context, *state.stages.mesh, draw.indexCount);
auto arguments = std::make_shared<Buffer>(context, ShaderRecompiler::MeshArgumentBytes, VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
auto arguments = meshArgumentBuffer(context, restartTable);
if (indirect.path == IndirectDrawPath::Gpu && recorder != nullptr && indirect.argumentImport->address != 0) {
const std::array<std::uint32_t, 7> words{rules.indexCount, rules.inputSize, rules.step, rules.primitivesPerGroup, rules.maxGroups, rules.maxInstances, rules.maxTotal};
if (recorder->RecordMeshArguments(commands, indirect.argumentImport->address + (args->arguments - indirect.argumentImport->base), arguments->DeviceAddress(), words)) {
@ -1433,6 +1477,7 @@ void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>
kept->targets = std::move(targets);
const auto& residents = kept->targets;
recorder.Keep(kept);
if (resources != nullptr) recorder.KeepBytes(resources.get(), resources->CopiedBytes());
if (checkRecords) recorder.OnComplete(std::move(checkRecords));
// Counted for the [address-sync] line: a lease released by the completion (the pin waiter
// finishes the recorder up to the open batch, which holds the kept resources and gets the
@ -1442,7 +1487,7 @@ void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>
resources->MarkGpuWrites(recorder);
// The write-back (fault check, copied buffers) runs when the batch completed; a fault is
// reported by the recorder ("deferred write-back failed") instead of thrown out of the draw.
if (listed) {
if (listed && resources->HasCopiedWrites()) {
// As VulkanDevice::dispatch lists its copied writers: delisted before the write-back
// (one that fails must not keep indirect dispatches on the CPU), listed after the
// registration (a throw there leaves nothing behind).
@ -1453,7 +1498,8 @@ void keepRecordedDraw(Recorder& recorder, const std::shared_ptr<ShaderResources>
});
writers->push_back(resources);
} else if (completion) {
recorder.OnComplete([resources] { resources->WriteBackBuffers(); });
if (resources->Reusable()) recorder.OnComplete([resources] { resources->CheckFaults(); });
else recorder.OnComplete([resources] { resources->WriteBackBuffers(); });
}
for (const auto& resident : residents) {
if (resident != nullptr) resident->MarkDirty();
@ -1481,7 +1527,7 @@ struct RecordedDraw {
VkShaderStageFlags pushStages = 0;
};
void pushDrawConstants(const Pipeline& pipeline, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, std::span<const CompiledShader> shaders, const ShaderResources& resources, const std::array<std::byte, PipelinePushConstantBytes>* bytes, VkShaderStageFlags stages, VkDeviceAddress meshArguments = 0) {
void pushDrawConstants(const Pipeline& pipeline, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, std::span<const CompiledShader> shaders, const ShaderResources& resources, const std::array<std::byte, PipelinePushConstantBytes>* bytes, VkShaderStageFlags stages, VkDeviceAddress meshArguments = 0, bool restartTable = false) {
auto block = bytes != nullptr ? *bytes : AssemblePushConstants(shaders);
if (bytes == nullptr) {
resources.PatchPushConstants(block);
@ -1493,7 +1539,7 @@ void pushDrawConstants(const Pipeline& pipeline, VkCommandBuffer commands, const
}
const auto firstVertex = draw.indirect ? draw.indirect->vertexConstant : draw.firstVertex;
const auto firstInstance = draw.indirect ? draw.indirect->instanceConstant : draw.firstInstance;
const std::array<std::uint32_t, ShaderRecompiler::MeshDrawPushBytes / 4> words{draw.indexCount, firstVertex, firstInstance, draw.indexed ? draw.indexSize : 0u, static_cast<std::uint32_t>(meshArguments), static_cast<std::uint32_t>(meshArguments >> 32u)};
const std::array<std::uint32_t, ShaderRecompiler::MeshDrawPushBytes / 4> words{draw.indexCount, firstVertex, firstInstance, draw.indexed ? draw.indexSize | (restartTable ? ShaderRecompiler::MeshIndexRestartTable : 0u) : 0u, static_cast<std::uint32_t>(meshArguments), static_cast<std::uint32_t>(meshArguments >> 32u)};
static_assert(ShaderRecompiler::MeshDrawPushOffsetBytes + ShaderRecompiler::MeshDrawPushBytes == PipelinePushSlotBytes);
std::memcpy(block.data() + ShaderRecompiler::MeshDrawPushOffsetBytes, words.data(), sizeof(words));
pipeline.PushConstants(commands, stages | VK_SHADER_STAGE_MESH_BIT_EXT, block);
@ -1623,7 +1669,8 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
for (const auto& target : record.targets) {
if (target != nullptr) passAttachments.emplace_back(target->Image(), true);
}
const PassAccess passAccess{passReads, resources.GpuWrites(), passImages, passAttachments, addressBased || resources.NeedsCompletion(), addressBased || resources.NeedsCompletion()};
const bool unknownMemory = addressBased || resources.UsesAddressTable() || resources.HasCopiedWrites();
const PassAccess passAccess{passReads, resources.GpuWrites(), passImages, passAttachments, unknownMemory, unknownMemory};
const bool fenced = state.depth.has_value() || !record.proxies.empty();
const auto start = recorder->StartDrawPass(passKey, capture || readsTarget || gpuIndirect || meshIndirect, fenced, passAccess);
const bool continued = start.continued;
@ -1651,6 +1698,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
VkBuffer argumentBuffer = VK_NULL_HANDLE;
VkDeviceSize argumentOffset = 0;
bool rewritten = false;
if (!meshIndirect && !inputs.restartTable.empty()) meshArguments = directMeshArguments(context, draw, inputs.restartTable);
if (continued) {
record.pipeline->Continue(commands, state);
} else {
@ -1665,7 +1713,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
}
APS5_LOG_CHARS_OUT_DEBUG("Upload barrier recorded");
if (meshIndirect) {
meshArguments = recordMeshArguments(context, commands, recorder, true, state, draw, *record.indirect, countBarrier);
meshArguments = recordMeshArguments(context, commands, recorder, true, state, draw, *record.indirect, countBarrier, inputs.restartTable);
argumentBuffer = meshArguments->Handle();
} else if (gpuIndirect) {
rewritten = recordIndirectArguments(context, commands, recorder, true, *record.indirect, scratch, argumentBuffer, argumentOffset, countBarrier);
@ -1683,13 +1731,17 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam
resources.Bind(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, record.pipeline->Layout());
}
APS5_LOG_CHARS_OUT_DEBUG("Resources bound");
pushDrawConstants(*record.pipeline, commands, state, draw, shaders, resources, record.pushBytes, record.pushStages, meshArguments != nullptr ? meshArguments->DeviceAddress() : 0);
pushDrawConstants(*record.pipeline, commands, state, draw, shaders, resources, record.pushBytes, record.pushStages, meshArguments != nullptr ? meshArguments->DeviceAddress() : 0, !inputs.restartTable.empty());
APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded");
recordDrawCommands(context, commands, state, draw, inputs, record.indirect, argumentBuffer, argumentOffset);
if (record.pipeline->SplitsFaces()) {
record.pipeline->ContinueBackFaces(commands, state);
recordDrawCommands(context, commands, state, draw, inputs, record.indirect, argumentBuffer, argumentOffset);
}
if (record.pipeline->SplitsSecondChannel()) {
record.pipeline->ContinueSecondChannel(commands);
recordDrawCommands(context, commands, state, draw, inputs, record.indirect, argumentBuffer, argumentOffset);
}
if (meshArguments != nullptr) recorder->Keep(meshArguments);
if (args != nullptr) CountIndirectDraw(record.indirect->path, record.indirect->readMs, rewritten);
auto checkRecords = indirectRecordCheck(record.indirect);
@ -1761,6 +1813,7 @@ std::optional<std::string> KnownValidationFailure(const Context& context, std::s
}
void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span<const CompiledShader> shaders, std::span<const GuestMemorySnapshot> snapshots, std::shared_ptr<const DrawRecipe>* recipeOut) {
if (context.recorder != nullptr) context.recorder->BoundKeptBytes();
PerformanceTimer timing("Graphics.Draw");
// APS5_PROFILE_DRAW prints the time of each phase of the draw (microseconds) and the [draws] totals.
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
@ -2117,8 +2170,10 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters&
APS5_LOG_CHARS_OUT_DEBUG("Upload barrier recorded");
std::shared_ptr<Buffer> meshArguments;
if (state.stages.mesh && args != nullptr) {
meshArguments = recordMeshArguments(context, commands, recorder, recorded, state, draw, indirect, countBarrier);
meshArguments = recordMeshArguments(context, commands, recorder, recorded, state, draw, indirect, countBarrier, inputs.restartTable);
argumentBuffer = meshArguments->Handle();
} else if (!inputs.restartTable.empty()) {
meshArguments = directMeshArguments(context, draw, inputs.restartTable);
} else if (gpuIndirect) {
rewritten = recordIndirectArguments(context, commands, recorder, recorded, indirect, scratch, argumentBuffer, argumentOffset, countBarrier);
}
@ -2153,13 +2208,17 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters&
APS5_LOG_CHARS_OUT_DEBUG("Pipeline Begin OK");
resources->Bind(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->Layout());
APS5_LOG_CHARS_OUT_DEBUG("Resources bound");
pushDrawConstants(*pipeline, commands, state, draw, shaders, *resources, nullptr, 0, meshArguments != nullptr ? meshArguments->DeviceAddress() : 0);
pushDrawConstants(*pipeline, commands, state, draw, shaders, *resources, nullptr, 0, meshArguments != nullptr ? meshArguments->DeviceAddress() : 0, !inputs.restartTable.empty());
APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded");
recordDrawCommands(context, commands, state, draw, inputs, args != nullptr ? &indirect : nullptr, argumentBuffer, argumentOffset);
if (pipeline->SplitsFaces()) {
pipeline->ContinueBackFaces(commands, state);
recordDrawCommands(context, commands, state, draw, inputs, args != nullptr ? &indirect : nullptr, argumentBuffer, argumentOffset);
}
if (pipeline->SplitsSecondChannel()) {
pipeline->ContinueSecondChannel(commands);
recordDrawCommands(context, commands, state, draw, inputs, args != nullptr ? &indirect : nullptr, argumentBuffer, argumentOffset);
}
if (meshArguments != nullptr && recorded) recorder->Keep(meshArguments);
if (args != nullptr) CountIndirectDraw(indirect.path, indirect.readMs, rewritten);
auto checkRecords = indirectRecordCheck(args != nullptr ? &indirect : nullptr);
@ -2304,6 +2363,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters&
}
DrawRecipeOutcome DrawWithRecipe(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span<const CompiledShader> shaders, std::span<const GuestMemorySnapshot> snapshots, const DrawRecipe& recipe) {
if (context.recorder != nullptr) context.recorder->BoundKeptBytes();
static_cast<void>(snapshots);
PerformanceTimer timing("Graphics.DrawWithRecipe");
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
@ -2372,7 +2432,7 @@ DrawRecipeOutcome DrawWithRecipe(const Context& context, const State& state, con
SharedResourceCache().Touch(recipe.key);
countCache(&DrawProfile::cacheHits);
timer.phase(PhaseLookup);
outcome.completion = false;
outcome.completion = resources->ChecksFaults();
outcome.recorded = true;
// The pipeline is the recipe's while the pipeline store still holds it (its device's teardown
// or its eviction bound drops it: a miss). The framebuffer is the recipe's while the pipeline's
@ -2386,6 +2446,7 @@ DrawRecipeOutcome DrawWithRecipe(const Context& context, const State& state, con
const auto recordStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
RecordedDraw record;
record.recorder = recorder;
record.completion = outcome.completion;
record.resources = std::move(resources);
record.pipeline = std::move(pipeline);
record.framebuffer = std::move(framebuffer);
@ -2410,12 +2471,7 @@ namespace {
std::shared_ptr<StorageTexture> metadataPassResident(const Context& context, const ColorTarget& color) {
if (color.tileMode == ColorTileMode::Linear || context.detiler == nullptr) return nullptr;
std::shared_ptr<StorageTexture> resident;
try {
resident = CachedStorageSurface(context, SurfaceForTarget(color));
} catch (const std::exception&) {
return nullptr;
}
const auto resident = CachedStorageSurface(context, SurfaceForTarget(color));
return resident != nullptr && resident->GuestBytes() == color.bytes ? resident : nullptr;
}
@ -2430,6 +2486,12 @@ void RunColorMetadataPass(const Context& context, const ColorMetadataPass& pass)
if (color.dccAddress == 0) continue;
auto keys = CurrentDccKeys(color.dccAddress, color.bytes);
if (keys == DccKeys::Uncompressed) continue;
if (keys == DccKeys::ClearSingle) {
Require(pass.mode == ColorMetadataPass::Mode::DccDecompress, "CB fast-clear eliminate over comp-to-single DCC keys (whether it expands them is not modeled)");
storeSingleTexels(context, color);
if (const auto resident = metadataPassResident(context, color)) resident->Refresh();
continue;
}
Require(IsDccClear(keys), std::string("CB metadata pass over DCC keys that are ") + DccKeysName(keys) + " (per-block metadata is not modeled)");
const auto texel = clearTexel(color, keys);
const auto resident = metadataPassResident(context, color);

63
core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp
View File

@ -1,14 +1,16 @@
#include "prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp"
#include "prx/libSceAgcDriver/Graphics/shaders/ColorTransfer_spv.h"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Graphics/include/HalfFloatScanout.hpp"
#include <array>
#include <cstring>
namespace AgcDriver::Graphics {
GpuColorTransfer::GpuColorTransfer(const Context& context) : context(context) {
VkShaderModule module = VK_NULL_HANDLE;
try {
std::array<VkDescriptorSetLayoutBinding, 2> bindings{};
std::array<VkDescriptorSetLayoutBinding, 3> bindings{};
for (std::uint32_t i = 0; i < bindings.size(); ++i) {
bindings[i] = {i, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr};
}
@ -36,7 +38,7 @@ GpuColorTransfer::GpuColorTransfer(const Context& context) : context(context) {
Check(context.Function<PFN_vkCreateComputePipelines>("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateComputePipelines color transfer");
context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, module, nullptr);
module = VK_NULL_HANDLE;
const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2};
const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 3};
VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
poolInfo.maxSets = 1;
poolInfo.poolSizeCount = 1;
@ -47,6 +49,17 @@ GpuColorTransfer::GpuColorTransfer(const Context& context) : context(context) {
allocation.descriptorSetCount = 1;
allocation.pSetLayouts = &descriptorLayout;
Check(context.Function<PFN_vkAllocateDescriptorSets>("vkAllocateDescriptorSets")(context.device, &allocation, &descriptorSet), "vkAllocateDescriptorSets color transfer");
const auto& table = HalfFloatScanoutThresholds();
thresholds = std::make_unique<Buffer>(context, sizeof(table), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
std::memcpy(thresholds->Bytes().data(), table.data(), sizeof(table));
const VkDescriptorBufferInfo thresholdInfo{thresholds->Handle(), 0, sizeof(table)};
VkWriteDescriptorSet thresholdWrite{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
thresholdWrite.dstSet = descriptorSet;
thresholdWrite.dstBinding = 2;
thresholdWrite.descriptorCount = 1;
thresholdWrite.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
thresholdWrite.pBufferInfo = &thresholdInfo;
context.Function<PFN_vkUpdateDescriptorSets>("vkUpdateDescriptorSets")(context.device, 1, &thresholdWrite, 0, nullptr);
} catch (...) {
if (module) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, module, nullptr);
release();
@ -65,16 +78,18 @@ void GpuColorTransfer::release() noexcept {
if (descriptorLayout) context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(context.device, descriptorLayout, nullptr);
}
void GpuColorTransfer::prepare(std::uint32_t newWidth, std::uint32_t newHeight, ColorTileMode newMode) {
const ColorTargetLayout layout(newWidth, newHeight, newMode);
Require(layout.Bytes() <= context.limits.maxStorageBufferRange && layout.LinearBytes() <= context.limits.maxStorageBufferRange, "color transfer exceeds storage buffer limits");
void GpuColorTransfer::prepare(std::uint32_t newWidth, std::uint32_t newHeight, ColorTileMode newMode, std::uint32_t newElementBytes) {
Require(newElementBytes == 4u || (newElementBytes == 8u && newMode == ColorTileMode::RenderTarget), "color transfer supports 8-byte texels only in the render target layout");
const ColorTargetLayout layout(newWidth, newHeight, newMode, newElementBytes);
const auto linearBytes = static_cast<std::size_t>(newWidth) * newHeight * 4u;
Require(layout.Bytes() <= context.limits.maxStorageBufferRange && linearBytes <= context.limits.maxStorageBufferRange, "color transfer exceeds storage buffer limits");
Require((newWidth + 7u) / 8u <= context.limits.maxComputeWorkGroupCount[0] && (newHeight + 7u) / 8u <= context.limits.maxComputeWorkGroupCount[1], "color transfer exceeds workgroup limits");
if (tiled && width == newWidth && height == newHeight && mode == newMode) return;
if (tiled && width == newWidth && height == newHeight && mode == newMode && elementBytes == newElementBytes) return;
constexpr VkBufferUsageFlags copies = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
auto newTiled = std::make_unique<Buffer>(context, layout.Bytes(), copies);
auto newTiledDevice = std::make_unique<DeviceBuffer>(context, layout.Bytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
auto newLinear = std::make_unique<DeviceBuffer>(context, layout.LinearBytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
const std::array<VkDescriptorBufferInfo, 2> buffers{{{newTiledDevice->Handle(), 0, layout.Bytes()}, {newLinear->Handle(), 0, layout.LinearBytes()}}};
auto newLinear = std::make_unique<DeviceBuffer>(context, linearBytes, copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
const std::array<VkDescriptorBufferInfo, 2> buffers{{{newTiledDevice->Handle(), 0, layout.Bytes()}, {newLinear->Handle(), 0, linearBytes}}};
std::array<VkWriteDescriptorSet, 2> writes{};
for (std::uint32_t i = 0; i < writes.size(); ++i) {
writes[i].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;
@ -91,33 +106,36 @@ void GpuColorTransfer::prepare(std::uint32_t newWidth, std::uint32_t newHeight,
width = newWidth;
height = newHeight;
mode = newMode;
elementBytes = newElementBytes;
}
void GpuColorTransfer::Upload(std::uint64_t address, std::uint32_t newWidth, std::uint32_t newHeight, ColorTileMode newMode) {
prepare(newWidth, newHeight, newMode);
const ColorTargetLayout layout(width, height, mode);
void GpuColorTransfer::Upload(std::uint64_t address, std::uint32_t newWidth, std::uint32_t newHeight, ColorTileMode newMode, std::uint32_t newElementBytes) {
prepare(newWidth, newHeight, newMode, newElementBytes);
const ColorTargetLayout layout(width, height, mode, elementBytes);
GuestMemory::Read(address, tiled->Bytes(), layout.Alignment());
}
void GpuColorTransfer::convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit) {
void GpuColorTransfer::convert(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit, bool halfFloat) {
Require(commands != VK_NULL_HANDLE && tiled && tiledDevice && linear, "color transfer is not prepared");
Require(halfFloat == (elementBytes == 8u), "color transfer source does not match the prepared texel size");
Require(!toTiled || !halfFloat, "color transfer cannot write 16-bit float texels back to the guest");
// The guest bytes move to video memory with one DMA copy; the shader's scattered accesses stay local.
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
CopyBuffer(context, commands, tiled->Handle(), 0, tiledDevice->Handle(), 0, tiledDevice->Size());
dispatch(commands, toTiled, swapRedBlue, tenBit, mode == ColorTileMode::RenderTarget);
dispatch(commands, toTiled, swapRedBlue, tenBit, mode == ColorTileMode::RenderTarget, halfFloat);
if (toTiled) {
CopyBuffer(context, commands, tiledDevice->Handle(), 0, tiled->Handle(), 0, tiledDevice->Size());
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT);
}
}
void GpuColorTransfer::dispatch(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit, bool tiledSource) {
void GpuColorTransfer::dispatch(VkCommandBuffer commands, bool toTiled, bool swapRedBlue, bool tenBit, bool tiledSource, bool halfFloat) {
VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
before.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
before.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
const auto barrier = context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier");
barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr);
const std::array<std::uint32_t, 4> push{width, height, (width + 127u) / 128u, (toTiled ? 1u : 0u) | (swapRedBlue ? 2u : 0u) | (tiledSource ? 4u : 0u) | (tenBit ? 8u : 0u)};
const std::array<std::uint32_t, 4> push{width, height, (width + 127u) / 128u, (toTiled ? 1u : 0u) | (swapRedBlue ? 2u : 0u) | (tiledSource ? 4u : 0u) | (tenBit ? 8u : 0u) | (halfFloat ? 16u : 0u)};
context.Function<PFN_vkCmdBindPipeline>("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
context.Function<PFN_vkCmdBindDescriptorSets>("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &descriptorSet, 0, nullptr);
context.Function<PFN_vkCmdPushConstants>("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data());
@ -128,20 +146,20 @@ void GpuColorTransfer::dispatch(VkCommandBuffer commands, bool toTiled, bool swa
barrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &after, 0, nullptr, 0, nullptr);
}
void GpuColorTransfer::DetileImage(VkCommandBuffer commands, VkImage image, VkImageLayout layout, std::uint32_t newWidth, std::uint32_t newHeight, ColorTileMode newMode, bool swapRedBlue, bool tenBit) {
void GpuColorTransfer::DetileImage(VkCommandBuffer commands, VkImage image, VkImageLayout layout, std::uint32_t newWidth, std::uint32_t newHeight, ColorTileMode newMode, bool swapRedBlue, bool tenBit, bool halfFloat) {
Require(commands != VK_NULL_HANDLE && image != VK_NULL_HANDLE, "color transfer image source is unavailable");
prepare(newWidth, newHeight, newMode);
Require(tiledDevice->Size() >= static_cast<VkDeviceSize>(width) * height * 4u, "color transfer buffer is smaller than the image");
prepare(newWidth, newHeight, newMode, halfFloat ? 8u : 4u);
Require(tiledDevice->Size() >= static_cast<VkDeviceSize>(width) * height * elementBytes, "color transfer buffer is smaller than the image");
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
VkBufferImageCopy copy{};
copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
copy.imageExtent = {width, height, 1};
context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, image, layout, tiledDevice->Handle(), 1, &copy);
dispatch(commands, false, swapRedBlue, tenBit, false);
dispatch(commands, false, swapRedBlue, tenBit, false, halfFloat);
}
void GpuColorTransfer::Detile(VkCommandBuffer commands, bool swapRedBlue, bool tenBit) {
convert(commands, false, swapRedBlue, tenBit);
void GpuColorTransfer::Detile(VkCommandBuffer commands, bool swapRedBlue, bool tenBit, bool halfFloat) {
convert(commands, false, swapRedBlue, tenBit, halfFloat);
}
void GpuColorTransfer::Tile(VkCommandBuffer commands) {
@ -150,7 +168,8 @@ void GpuColorTransfer::Tile(VkCommandBuffer commands) {
void GpuColorTransfer::WriteBack(std::uint64_t address) {
Require(tiled != nullptr, "color transfer is not prepared for writeback");
const ColorTargetLayout layout(width, height, mode);
Require(elementBytes == 4u, "color transfer cannot write 16-bit float texels back to the guest");
const ColorTargetLayout layout(width, height, mode, elementBytes);
GuestMemory::Write(address, tiled->Bytes(), layout.Alignment());
}

546
core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp
View File

@ -20,6 +20,7 @@
#include <sys/stat.h>
#include <unistd.h>
#endif
#include <array>
#include <atomic>
#include <tuple>
#include <functional>
@ -57,6 +58,9 @@ struct ImageMirror {
std::uint64_t bytes = 0;
bool writable = false;
bool heap = false;
bool sparse = false;
std::uint64_t reserved = 0;
std::vector<std::pair<std::uint64_t, std::uint64_t>> committed;
std::vector<std::uint64_t> generations;
std::weak_ptr<const GuestAllocations::Range> range;
std::shared_ptr<Buffer> buffer;
@ -98,6 +102,7 @@ struct GuestBufferMemory::AddressSpace {
std::vector<std::size_t> writableMirrors;
std::vector<ImageMirror*> heapMirrors;
std::vector<HeapRun> heapRuns;
std::vector<ImageMirror*> sparseMirrors;
std::unique_ptr<std::atomic<std::uint64_t>[]> verified;
std::vector<CopiedRange> copied;
// The BDA table entries of `base`, in its order.
@ -143,7 +148,9 @@ struct HostImports {
PFN_vkDestroyBuffer destroyBuffer = nullptr;
PFN_vkFreeMemory freeMemory = nullptr;
std::map<std::uint64_t, HostImport> imports;
std::uint64_t liveBytes = 0;
std::set<std::uint64_t> failed;
std::set<std::uint64_t> unaliasedSpans;
// Registry generation the imports were last reconciled with.
std::uint64_t refreshedGeneration = 0;
// Bumped whenever an import is dropped, so HostImport pointers taken under the lock earlier can be
@ -191,15 +198,17 @@ struct RetiredImport {
// so the objects live until the batch open now completed; with no batches in flight all GPU work that
// used it was synchronous and it is destroyed at once.
const GuestAllocations::Range* containingRange(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end);
bool runCovers(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end);
void retireImport(VkDevice device, HostImports& state, std::map<std::uint64_t, HostImport>::iterator it, const GuestAllocations::Lease& lease) {
// Results shadowed for the import reach its (old) buffer first where the memory is still a
// readable registered range (the import retires because its registration vanished or changed
// size); the holder below outlives the batch that copies them.
RetireShadow(device, it->second, [&lease](std::uint64_t begin, std::uint64_t end) { return containingRange(lease, begin, end) != nullptr; });
RetireShadow(device, it->second, [&lease](std::uint64_t begin, std::uint64_t end) { return runCovers(lease, begin, end); });
auto holder = std::make_shared<RetiredImport>(state.device, state.destroyBuffer, state.freeMemory, it->second);
if (auto* recorder = Recorder::Active(); recorder != nullptr && !recorder->Idle()) recorder->Keep(std::move(holder));
++state.epoch;
state.liveBytes -= it->second.bytes;
state.imports.erase(it);
}
@ -497,8 +506,8 @@ void decideImportWatch(const Context& context, HostImports& state) {
state.unwatchImports = true;
std::fprintf(stderr, "[write-watch] host imports resolve write protection: unknown (probe failed at %s, %d); imported ranges are compared\n", probe.failure, static_cast<int>(probe.result));
} else {
state.unwatchImports = probe.writtenAfterSubmit != 0;
std::fprintf(stderr, "[write-watch] host imports resolve write protection: %s (%u of %u scratch pages written after a GPU read, %u after the import); imported ranges %s\n", state.unwatchImports ? "yes" : "no", probe.writtenAfterSubmit, probe.pages, probe.writtenAtImport, state.unwatchImports ? "are compared" : "stay watched");
state.unwatchImports = probe.writtenAfterSubmit != 0 || probe.writtenAfterFill != 0;
std::fprintf(stderr, "[write-watch] host imports resolve write protection: %s (%u of %u scratch pages written after a GPU read, %u after a GPU write, %u after the import); imported ranges %s\n", state.unwatchImports ? "yes" : "no", probe.writtenAfterSubmit, probe.pages, probe.writtenAfterFill, probe.writtenAtImport, state.unwatchImports ? "are compared" : "stay watched");
}
state.unwatchDmaBufImports = state.unwatchImports;
if (!context.dmaBufImport) return;
@ -508,8 +517,8 @@ void decideImportWatch(const Context& context, HostImports& state) {
std::fprintf(stderr, "[write-watch] dma-buf imports keep write protection: unknown (probe failed at %s, %d); imported ranges are compared\n", dmaBuf.failure, static_cast<int>(dmaBuf.result));
return;
}
state.unwatchDmaBufImports = dmaBuf.writtenAfterSubmit != 0 || dmaBuf.writtenByCpu == 0;
std::fprintf(stderr, "[write-watch] dma-buf imports keep write protection: %s (%u of %u scratch pages written after a GPU read, %u after the import, %u seen after a CPU store); imported ranges %s\n", state.unwatchDmaBufImports ? "no" : "yes", dmaBuf.writtenAfterSubmit, dmaBuf.pages, dmaBuf.writtenAtImport, dmaBuf.writtenByCpu, state.unwatchDmaBufImports ? "are compared" : "stay watched");
state.unwatchDmaBufImports = dmaBuf.writtenAfterSubmit != 0 || dmaBuf.writtenAfterFill != 0 || dmaBuf.writtenByCpu == 0;
std::fprintf(stderr, "[write-watch] dma-buf imports keep write protection: %s (%u of %u scratch pages written after a GPU read, %u after a GPU write, %u after the import, %u seen after a CPU store); imported ranges %s\n", state.unwatchDmaBufImports ? "no" : "yes", dmaBuf.writtenAfterSubmit, dmaBuf.pages, dmaBuf.writtenAfterFill, dmaBuf.writtenAtImport, dmaBuf.writtenByCpu, state.unwatchDmaBufImports ? "are compared" : "stay watched");
#endif
}
@ -518,16 +527,75 @@ std::shared_ptr<const GuestAllocations::Range> leasedRangeOwner(const GuestAlloc
return found != lease.end() && (*found)->address == base ? *found : nullptr;
}
bool sameRange(const HostImport& entry, const GuestAllocations::Lease& lease) {
const auto current = leasedRangeOwner(lease, entry.base);
return current != nullptr && !entry.range.owner_before(current) && !current.owner_before(entry.range);
struct ImportRun {
std::uint64_t base = 0;
std::uint64_t bytes = 0;
std::vector<std::shared_ptr<const GuestAllocations::Range>> ranges;
};
bool continuesRun(const GuestAllocations::Range& before, const GuestAllocations::Range& after) {
return before.address + before.bytes == after.address && before.readable && after.readable && before.writable == after.writable && before.gpu == after.gpu;
}
const HostImport* importAllocation(const Context& context, HostImports& state, std::uint64_t base, std::uint64_t bytes, const GuestAllocations::Lease& lease) {
std::size_t leasedIndexContaining(const GuestAllocations::Lease& lease, std::uint64_t address) {
const auto found = std::upper_bound(lease.begin(), lease.end(), address, [](std::uint64_t value, const auto& range) { return value < range->address; });
if (found == lease.begin()) return lease.size();
const auto index = static_cast<std::size_t>(std::prev(found) - lease.begin());
const auto& range = *lease[index];
return range.readable && address < range.address + range.bytes ? index : lease.size();
}
ImportRun importRun(const GuestAllocations::Lease& lease, std::uint64_t address) {
const auto index = leasedIndexContaining(lease, address);
if (index == lease.size()) return {};
auto first = index;
while (first > 0 && continuesRun(*lease[first - 1], *lease[first])) --first;
auto last = index;
while (last + 1 < lease.size() && continuesRun(*lease[last], *lease[last + 1])) ++last;
if (first != last && Imports().unaliasedSpans.contains(lease[first]->address)) return {lease[index]->address, lease[index]->bytes, {lease[index]}};
ImportRun run{lease[first]->address, lease[last]->address + lease[last]->bytes - lease[first]->address, {}};
run.ranges.assign(lease.begin() + static_cast<std::ptrdiff_t>(first), lease.begin() + static_cast<std::ptrdiff_t>(last) + 1);
return run;
}
bool runCovers(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end) {
const auto run = importRun(lease, begin);
return !run.ranges.empty() && end <= run.base + run.bytes;
}
bool sameRun(const HostImport& entry, const ImportRun& run) {
if (run.base != entry.base || run.bytes != entry.bytes || run.ranges.size() != entry.ranges.size()) return false;
for (std::size_t index = 0; index < run.ranges.size(); ++index) {
if (entry.ranges[index].owner_before(run.ranges[index]) || run.ranges[index].owner_before(entry.ranges[index])) return false;
}
return true;
}
bool sameRange(const HostImport& entry, const GuestAllocations::Lease& lease) {
return sameRun(entry, importRun(lease, entry.base));
}
const HostImport* findImport(HostImports& state, std::uint64_t begin, std::uint64_t end);
bool importsStale(const Context& context, const HostImports& state);
const HostImport* importAllocation(const Context& context, HostImports& state, std::uint64_t address, const GuestAllocations::Lease& lease) {
if (!importsStale(context, state)) {
if (const auto* current = findImport(state, address, address + 1)) return current;
}
auto run = importRun(lease, address);
if (run.ranges.empty()) return nullptr;
const auto base = run.base;
const auto bytes = run.bytes;
if (const auto found = state.imports.find(base); found != state.imports.end()) {
if (found->second.bytes == bytes && sameRange(found->second, lease)) return &found->second;
if (sameRun(found->second, run)) return &found->second;
retireImport(context.device, state, found, lease);
}
for (auto it = state.imports.lower_bound(base); it != state.imports.end() && it->first < base + bytes;) {
const auto next = std::next(it);
retireImport(context.device, state, it, lease);
it = next;
}
if (auto it = state.imports.lower_bound(base); it != state.imports.begin() && std::prev(it)->second.base + std::prev(it)->second.bytes > base) retireImport(context.device, state, std::prev(it), lease);
const auto alignment = context.hostImportAlignment;
if (alignment == 0 || base % alignment != 0 || bytes % alignment != 0 || state.failed.contains(base)) return nullptr;
// Pinned imports count against the driver's system memory budget; past it ordinary host
@ -537,8 +605,7 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
const char* value = std::getenv("APS5_HOST_IMPORT_MIB");
return (value ? std::strtoull(value, nullptr, 10) : 6144ull) << 20u;
}();
std::uint64_t live = 0;
for (const auto& [address, existing] : state.imports) live += existing.bytes;
const auto live = state.liveBytes;
if (live + bytes > budget) {
// A refused import turns every later use of the range into CPU copies, so say so.
static std::uint64_t refused = 0, refusedBytes = 0;
@ -552,7 +619,8 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
return nullptr;
}
HostImport entry{base, bytes, VK_NULL_HANDLE, VK_NULL_HANDLE, 0};
entry.range = leasedRangeOwner(lease, base);
entry.range = run.ranges.front();
entry.ranges = std::move(run.ranges);
#ifdef _WIN32
// Drivers pin imported pages, so every page must be committed and accessible.
bool writable = true;
@ -582,7 +650,13 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
std::snprintf(text, sizeof(text), "AGC graphics: host import of 0x%llx+0x%llx: memory at 0x%llx (type 0x%lx, protection 0x%lx) is neither read-write, read-only nor a shared mapping", static_cast<unsigned long long>(base), static_cast<unsigned long long>(bytes), reinterpret_cast<unsigned long long>(refused.BaseAddress), refused.Type, refused.Protect);
throw std::runtime_error(text);
}
entry.alias = GuestArena::GuestArenaMapAlias_nid_postfix(static_cast<std::uintptr_t>(base), static_cast<std::size_t>(bytes));
try {
entry.alias = GuestArena::GuestArenaMapAlias_nid_postfix(static_cast<std::uintptr_t>(base), static_cast<std::size_t>(bytes));
} catch (const std::runtime_error&) {
if (entry.ranges.size() == 1) throw;
state.unaliasedSpans.insert(base);
return importAllocation(context, state, address, lease);
}
}
#else
if (!GuestMemory::Accessible(reinterpret_cast<const void*>(base), static_cast<std::size_t>(bytes), true)) {
@ -635,9 +709,8 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s
static std::uint64_t importedBytes = 0;
importedBytes += bytes;
static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr;
std::uint64_t liveBytes = bytes;
for (const auto& [address, existing] : state.imports) liveBytes += existing.bytes;
if (trace) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx ok (%zu live, %.1f MiB live, %.1f MiB ever)\n", static_cast<unsigned long long>(base), static_cast<unsigned long long>(bytes), state.imports.size() + 1, liveBytes / 1048576.0, importedBytes / 1048576.0);
state.liveBytes += bytes;
if (trace) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx ok (%zu live, %.1f MiB live, %.1f MiB ever)\n", static_cast<unsigned long long>(base), static_cast<unsigned long long>(bytes), state.imports.size() + 1, state.liveBytes / 1048576.0, importedBytes / 1048576.0);
return &state.imports.emplace(base, entry).first->second;
}
@ -669,6 +742,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
#endif
}
state.imports.clear();
state.liveBytes = 0;
state.failed.clear();
state.device = context.device;
state.destroyBuffer = context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer");
@ -683,7 +757,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc
state.refreshedGeneration = generation;
for (auto it = state.imports.begin(); it != state.imports.end();) {
const auto* range = leasedRangeAt(lease, it->first);
if (range != nullptr && range->bytes == it->second.bytes && sameRange(it->second, lease)) {
if (range != nullptr && sameRange(it->second, lease)) {
if (it->second.unwatched) GuestMemory::Unwatch(it->first, it->second.bytes);
++it;
continue;
@ -727,6 +801,7 @@ struct ImageMirrors {
std::uint64_t heapImported = 0;
std::uint64_t heapImportedBytes = 0;
std::uint64_t subranges = 0;
std::uint64_t sparseBinds = 0;
std::uint64_t rebuilds = 0;
std::uint64_t refreshes = 0;
std::uint64_t blocksCompared = 0;
@ -822,12 +897,22 @@ struct RefreshBlock {
std::uint64_t address;
std::size_t length;
std::uint64_t generation = 0;
std::uint8_t fill = 0;
};
constexpr std::uint8_t SparseCopy = 1;
constexpr std::uint8_t SparseZero = 2;
// Compares a block's guest bytes with the mirror's shadow and copies a changed block into the device
bool compareBlock(const RefreshBlock& block) {
const auto offset = static_cast<std::size_t>(block.address - block.mirror->base);
const auto* guest = reinterpret_cast<const std::byte*>(block.address);
if (block.fill != 0) {
auto* target = block.mirror->buffer->Bytes().data() + offset;
if (block.fill == SparseZero) std::memset(target, 0, block.length);
else std::memcpy(target, guest, block.length);
return true;
}
if (block.mirror->heap) {
std::memcpy(block.mirror->buffer->Bytes().data() + offset, guest, block.length);
block.mirror->generations[static_cast<std::size_t>(block.address / 65536 - block.mirror->base / 65536)] = block.generation;
@ -936,10 +1021,7 @@ private:
// completes first, so its results reach the guest and the shadow before the compare and no batch
// writes a block being replaced; GPU reads of the mirror by earlier recorded work (an address-based
// dispatch whose lease is released at completion) see the newer bytes, as they would on hardware.
void prepareRange(std::uint64_t address, std::uint64_t bytes) {
// The compare reads the pages directly; a mirror is only reused while its Range exists, but a
// lease of a build in flight can keep a replaced Range alive past a protection change.
Require(GuestMemory::Accessible(reinterpret_cast<const void*>(address), static_cast<std::size_t>(bytes)), "image mirror range became inaccessible");
void syncRange(std::uint64_t address, std::uint64_t bytes) {
// Named for the [hooksync] attribution: the flush goes through the hook, which waits when
// recorded work notes a write inside the range (a descriptor written inside a writable exe
// range; the lease itself maps the range read-only). The access is the whole range.
@ -954,6 +1036,11 @@ void prepareRange(std::uint64_t address, std::uint64_t bytes) {
++state.refreshes;
}
void prepareRange(std::uint64_t address, std::uint64_t bytes) {
Require(GuestMemory::Accessible(reinterpret_cast<const void*>(address), static_cast<std::size_t>(bytes)), "image mirror range became inaccessible");
syncRange(address, bytes);
}
bool prepareRefresh(ImageMirror& mirror, std::uint64_t address, std::uint64_t bytes) {
if ((!mirror.writable && !mirror.heap) || bytes == 0) return false;
prepareRange(address, bytes);
@ -1024,6 +1111,105 @@ void refreshHeapMirrors(std::vector<ImageMirror*>& mirrors, std::vector<RefreshB
}
}
bool sparseMirrorsEnabled() {
static const bool disabled = std::getenv("APS5_NO_SPARSE_HEAP_MIRROR") != nullptr;
return !disabled;
}
using Pieces = std::vector<std::pair<std::uint64_t, std::uint64_t>>;
std::uint64_t sparseExtent(std::uint64_t base, std::uint64_t reserved, const Pieces& pieces) {
constexpr std::uint64_t granule = std::uint64_t{1} << 20u;
const auto used = pieces.empty() ? 0 : pieces.back().second - base;
const auto wanted = used + used / 8;
return std::min(reserved, std::max<std::uint64_t>(granule, (wanted + granule - 1) / granule * granule));
}
std::uint64_t collectPieces(const Pieces& pieces) {
std::uint64_t generation = 0;
for (const auto& [first, last] : pieces) {
generation = GuestMemory::CollectWrites(first, static_cast<std::size_t>(last - first));
if (generation == 0) return 0;
}
return generation != 0 ? generation : GuestMemory::TrackerGeneration();
}
bool covers(const Pieces& pieces, std::uint64_t address) {
return std::any_of(pieces.begin(), pieces.end(), [&](const auto& piece) { return piece.first <= address && address < piece.second; });
}
void forgetCommitChanges(ImageMirror& mirror, const Pieces& pieces) {
if (pieces == mirror.committed) return;
std::vector<std::uint64_t> edges;
for (const auto* list : std::array<const Pieces*, 2>{&pieces, &mirror.committed}) {
for (const auto& [first, last] : *list) {
edges.push_back(first);
edges.push_back(last);
}
}
std::sort(edges.begin(), edges.end());
edges.erase(std::unique(edges.begin(), edges.end()), edges.end());
constexpr std::uint64_t block = 65536;
const auto end = mirror.base + mirror.bytes;
for (std::size_t at = 0; at + 1 < edges.size(); ++at) {
const auto from = std::max(edges[at], mirror.base);
const auto to = std::min(edges[at + 1], end);
if (from >= to || covers(pieces, from) == covers(mirror.committed, from)) continue;
for (auto index = from / block - mirror.base / block; index <= (to - 1) / block - mirror.base / block; ++index) mirror.generations[static_cast<std::size_t>(index)] = 0;
}
}
void appendSparseBlocks(ImageMirror& mirror, Pieces pieces, std::uint64_t generation, std::vector<RefreshBlock>& blocks) {
constexpr std::uint64_t block = 65536;
thread_local std::vector<std::uint8_t>* changedSlot = nullptr;
auto& changed = ShaderRecompiler::ThreadOwned(changedSlot);
forgetCommitChanges(mirror, pieces);
++Mirrors().heapChecks;
changed.assign(mirror.generations.size(), 0);
GuestMemory::ChangedBlocks(mirror.base, static_cast<std::size_t>(mirror.bytes), mirror.generations, changed);
const auto aligned = mirror.base / block * block;
const auto end = mirror.base + mirror.bytes;
bool refilled = false;
std::size_t piece = 0;
for (std::size_t at = 0; at < changed.size(); ++at) {
if (changed[at] == 0) continue;
refilled = true;
mirror.generations[at] = generation;
const auto from = std::max(mirror.base, aligned + at * block);
const auto to = std::min(end, aligned + (at + 1) * block);
auto cursor = from;
while (piece < pieces.size() && pieces[piece].second <= from) ++piece;
for (auto next = piece; next < pieces.size() && pieces[next].first < to; ++next) {
const auto first = std::max(pieces[next].first, from);
const auto last = std::min(pieces[next].second, to);
if (cursor < first) blocks.push_back({&mirror, cursor, static_cast<std::size_t>(first - cursor), 0, SparseZero});
blocks.push_back({&mirror, first, static_cast<std::size_t>(last - first), 0, SparseCopy});
cursor = last;
}
if (cursor < to) blocks.push_back({&mirror, cursor, static_cast<std::size_t>(to - cursor), 0, SparseZero});
}
if (refilled) ++Mirrors().heapRefills;
mirror.committed = std::move(pieces);
}
Pieces clipPieces(const Pieces& pieces, std::uint64_t begin, std::uint64_t end) {
Pieces result;
for (const auto& [first, last] : pieces) {
if (std::max(first, begin) < std::min(last, end)) result.emplace_back(std::max(first, begin), std::min(last, end));
}
return result;
}
bool refreshSparseMirror(ImageMirror& mirror, std::vector<RefreshBlock>& blocks) {
auto pieces = GuestMemory::DescribeCommitted(mirror.base, static_cast<std::size_t>(mirror.reserved)).ranges;
if (!pieces.empty() && pieces.back().second > mirror.base + mirror.bytes) return false;
syncRange(mirror.base, mirror.bytes);
const auto generation = collectPieces(pieces);
Require(generation != 0, "a sparse heap mirror's range is no longer write-watched");
appendSparseBlocks(mirror, std::move(pieces), generation, blocks);
return true;
}
void planSpaceMirrors(GuestBufferMemory::AddressSpace& space) {
for (std::size_t index = 0; index < space.base.size(); ++index) {
const auto& region = space.base[index];
@ -1034,6 +1220,10 @@ void planSpaceMirrors(GuestBufferMemory::AddressSpace& space) {
continue;
}
auto* mirror = region.mirror.get();
if (mirror->sparse) {
space.sparseMirrors.push_back(mirror);
continue;
}
if (space.heapRuns.empty() || mirror->base != space.heapMirrors.back()->base + space.heapMirrors.back()->bytes) space.heapRuns.push_back({mirror->base, 0, space.heapMirrors.size(), 0});
space.heapMirrors.push_back(mirror);
auto& run = space.heapRuns.back();
@ -1075,36 +1265,53 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
}
if (state.failed.contains(range->address)) return nullptr;
const auto found = state.entries.find(range->address);
if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->bytes == range->bytes && found->second->writable == range->writable && found->second->heap == heap) mirror = found->second;
if (found != state.entries.end() && sameRange(found->second->range, range) && found->second->reserved == range->bytes && found->second->writable == range->writable && found->second->heap == heap) mirror = found->second;
}
if (mirror != nullptr) {
if (!heap && prepareRefresh(*mirror, range->address, range->bytes)) appendBlocks(blocks, *mirror, range->address, range->bytes);
return mirror;
if (!mirror->sparse) {
if (!heap && prepareRefresh(*mirror, range->address, range->bytes)) appendBlocks(blocks, *mirror, range->address, range->bytes);
return mirror;
}
if (refreshSparseMirror(*mirror, blocks)) return mirror;
}
if (!GuestMemory::Accessible(reinterpret_cast<const void*>(range->address), range->bytes, range->writable && !heap)) return nullptr;
Pieces pieces;
if (!GuestMemory::Accessible(reinterpret_cast<const void*>(range->address), range->bytes, range->writable && !heap)) {
if (!heap || !sparseMirrorsEnabled()) return nullptr;
pieces = GuestMemory::DescribeCommitted(range->address, range->bytes).ranges;
if (pieces.empty()) return nullptr;
}
const bool sparse = !pieces.empty();
mirror = std::make_shared<ImageMirror>();
mirror->base = range->address;
mirror->bytes = range->bytes;
mirror->bytes = sparse ? sparseExtent(range->address, range->bytes, pieces) : range->bytes;
mirror->reserved = range->bytes;
mirror->sparse = sparse;
mirror->writable = range->writable;
mirror->heap = heap;
mirror->range = range;
if (heap) {
prepareRefresh(*mirror, range->address, range->bytes);
const auto generation = GuestMemory::CollectWrites(range->address, range->bytes);
std::uint64_t generation = 0;
if (sparse) {
syncRange(mirror->base, mirror->bytes);
if (GuestMemory::Watched(mirror->base, static_cast<std::size_t>(mirror->bytes))) generation = collectPieces(pieces);
} else {
prepareRefresh(*mirror, range->address, range->bytes);
generation = GuestMemory::CollectWrites(range->address, range->bytes);
}
constexpr std::uint64_t block = 65536;
mirror->generations.assign(static_cast<std::size_t>(((range->address + range->bytes + block - 1) / block) - range->address / block), generation);
mirror->generations.assign(static_cast<std::size_t>(((mirror->base + mirror->bytes + block - 1) / block) - mirror->base / block), generation);
if (generation == 0) {
std::lock_guard lock(state.mutex);
++state.heapUnwatched;
return nullptr;
}
if (heapMirrorsBeside(*range) + range->bytes > heapMirrorBudget()) {
if (heapMirrorsBeside(*range) + mirror->bytes > heapMirrorBudget()) {
sweepMirrors();
if (const auto held = heapMirrorsBeside(*range); held + range->bytes > heapMirrorBudget()) throw std::runtime_error(heapMirrorError(*range, held, "past the budget"));
if (const auto held = heapMirrorsBeside(*range); held + mirror->bytes > heapMirrorBudget()) throw std::runtime_error(heapMirrorError(*range, held, "past the budget"));
}
}
try {
mirror->buffer = std::make_shared<Buffer>(context, range->bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
mirror->buffer = std::make_shared<Buffer>(context, static_cast<std::size_t>(mirror->bytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
} catch (const std::runtime_error& error) {
if (heap) throw std::runtime_error(heapMirrorError(*range, heapMirrorsBeside(*range), error.what()));
std::fprintf(stderr, "[gpu] image mirror of 0x%llx+0x%llx failed: %s; falling back to copies\n", static_cast<unsigned long long>(range->address), static_cast<unsigned long long>(range->bytes), error.what());
@ -1116,7 +1323,17 @@ std::shared_ptr<ImageMirror> acquireMirror(const Context& context, const std::sh
// named for the [hooksync] attribution like the refresh's flush.
{
const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::MirrorRefresh);
GuestMemory::Read(range->address, mirror->buffer->Bytes());
const auto target = mirror->buffer->Bytes().first(static_cast<std::size_t>(mirror->bytes));
if (!sparse) GuestMemory::Read(range->address, target);
std::size_t cursor = 0;
for (const auto& [first, last] : pieces) {
const auto at = static_cast<std::size_t>(first - mirror->base);
std::memset(target.data() + cursor, 0, at - cursor);
GuestMemory::Read(first, target.subspan(at, static_cast<std::size_t>(last - first)));
cursor = static_cast<std::size_t>(last - mirror->base);
}
if (sparse) std::memset(target.data() + cursor, 0, target.size() - cursor);
mirror->committed = std::move(pieces);
}
if (range->writable && !heap) mirror->shadow.assign(mirror->buffer->Bytes().begin(), mirror->buffer->Bytes().end());
std::lock_guard lock(state.mutex);
@ -1207,7 +1424,7 @@ void reportMirrors() {
}
heapBytes = state.heapBytes;
}
AgcDriver::ProfilePrint_nid_no_patch("[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable, %zu heap %.1f MiB), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs; heap refills %llu, unwatched heaps %llu, %llu heap mirrors released for imports (%.1f MiB)\n", count, bytes / 1048576.0, writable, heaps, heapBytes / 1048576.0, static_cast<unsigned long long>(state.builds), static_cast<unsigned long long>(state.subranges), static_cast<unsigned long long>(state.rebuilds), static_cast<unsigned long long>(state.refreshes), static_cast<unsigned long long>(state.blocksCompared), static_cast<unsigned long long>(state.blocksCopied), static_cast<unsigned long long>(state.syncs), static_cast<unsigned long long>(state.heapRefills), static_cast<unsigned long long>(state.heapUnwatched), static_cast<unsigned long long>(state.heapImported), state.heapImportedBytes / 1048576.0);
AgcDriver::ProfilePrint_nid_no_patch("[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable, %zu heap %.1f MiB), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu sparse heaps bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs; heap refills %llu, unwatched heaps %llu, %llu heap mirrors released for imports (%.1f MiB)\n", count, bytes / 1048576.0, writable, heaps, heapBytes / 1048576.0, static_cast<unsigned long long>(state.builds), static_cast<unsigned long long>(state.subranges), static_cast<unsigned long long>(state.sparseBinds), static_cast<unsigned long long>(state.rebuilds), static_cast<unsigned long long>(state.refreshes), static_cast<unsigned long long>(state.blocksCompared), static_cast<unsigned long long>(state.blocksCopied), static_cast<unsigned long long>(state.syncs), static_cast<unsigned long long>(state.heapRefills), static_cast<unsigned long long>(state.heapUnwatched), static_cast<unsigned long long>(state.heapImported), state.heapImportedBytes / 1048576.0);
}
// Deferred lease release. The lease an address-based build takes (AcquireRegistered) is dropped by its
@ -1282,6 +1499,7 @@ bool WaitForLeasesAndImports(std::uintptr_t address, std::size_t bytes) noexcept
synced = true;
drained = true;
}
if (recorder != nullptr) recorder->FlushDeferredReleases();
const auto end = static_cast<std::uint64_t>(address) + bytes;
auto& imports = Imports();
const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
@ -1305,6 +1523,7 @@ bool WaitForLeasesAndImports(std::uintptr_t address, std::size_t bytes) noexcept
synced = true;
drained = true;
}
if (retiredImport && recorder != nullptr) recorder->FlushDeferredReleases();
} catch (const std::exception& error) {
std::fprintf(stderr, "[gpu] lease wait failed: %s\n", error.what());
}
@ -1461,7 +1680,7 @@ std::string AddressCopyOverflow(std::vector<AddressCopy> copies, std::uint64_t l
MirrorStats MirrorCounters() {
auto& state = Mirrors();
std::lock_guard lock(state.mutex);
MirrorStats stats{0, state.heapBytes, state.rebuilds, state.blocksCopied, state.heapRefills, state.sweeps, state.heapChecks};
MirrorStats stats{0, state.heapBytes, state.rebuilds, state.blocksCopied, state.heapRefills, state.sweeps, state.heapChecks, state.sparseBinds};
for (const auto& [base, mirror] : state.entries) stats.heapMirrors += mirror->heap ? 1 : 0;
return stats;
}
@ -1492,6 +1711,7 @@ void ClearHostImports(VkDevice device) {
#endif
}
state.imports.clear();
state.liveBytes = 0;
state.failed.clear();
state.device = VK_NULL_HANDLE;
state.refreshedGeneration = 0;
@ -1506,7 +1726,7 @@ void runImportProbe(const Context& context, std::uint64_t base, std::uint64_t by
HostImport import{base, bytes, VK_NULL_HANDLE, VK_NULL_HANDLE, 0};
VkBuffer destination = VK_NULL_HANDLE;
VkDeviceMemory destinationMemory = VK_NULL_HANDLE;
VkCommandBuffer commands = VK_NULL_HANDLE;
VkCommandBuffer commands[2]{};
VkFence fence = VK_NULL_HANDLE;
bool submitted = false;
const auto collect = [&](std::uint32_t& pages) {
@ -1536,24 +1756,34 @@ void runImportProbe(const Context& context, std::uint64_t base, std::uint64_t by
VkCommandBufferAllocateInfo allocate{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
allocate.commandPool = context.pool;
allocate.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
allocate.commandBufferCount = 1;
if ((probe.result = context.Function<PFN_vkAllocateCommandBuffers>("vkAllocateCommandBuffers")(context.device, &allocate, &commands)) != VK_SUCCESS) return "vkAllocateCommandBuffers";
allocate.commandBufferCount = 2;
if ((probe.result = context.Function<PFN_vkAllocateCommandBuffers>("vkAllocateCommandBuffers")(context.device, &allocate, commands)) != VK_SUCCESS) return "vkAllocateCommandBuffers";
VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
if ((probe.result = context.Function<PFN_vkBeginCommandBuffer>("vkBeginCommandBuffer")(commands, &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer";
if ((probe.result = context.Function<PFN_vkBeginCommandBuffer>("vkBeginCommandBuffer")(commands[0], &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer";
const VkBufferCopy region{0, 0, bytes};
context.Function<PFN_vkCmdCopyBuffer>("vkCmdCopyBuffer")(commands, import.buffer, destination, 1, &region);
if ((probe.result = context.Function<PFN_vkEndCommandBuffer>("vkEndCommandBuffer")(commands)) != VK_SUCCESS) return "vkEndCommandBuffer";
context.Function<PFN_vkCmdCopyBuffer>("vkCmdCopyBuffer")(commands[0], import.buffer, destination, 1, &region);
if ((probe.result = context.Function<PFN_vkEndCommandBuffer>("vkEndCommandBuffer")(commands[0])) != VK_SUCCESS) return "vkEndCommandBuffer";
if ((probe.result = context.Function<PFN_vkBeginCommandBuffer>("vkBeginCommandBuffer")(commands[1], &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer";
context.Function<PFN_vkCmdFillBuffer>("vkCmdFillBuffer")(commands[1], import.buffer, 0, bytes, 0x02020202u);
if ((probe.result = context.Function<PFN_vkEndCommandBuffer>("vkEndCommandBuffer")(commands[1])) != VK_SUCCESS) return "vkEndCommandBuffer";
const VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
if ((probe.result = context.Function<PFN_vkCreateFence>("vkCreateFence")(context.device, &fenceInfo, nullptr, &fence)) != VK_SUCCESS) return "vkCreateFence";
VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO};
submit.commandBufferCount = 1;
submit.pCommandBuffers = &commands;
if ((probe.result = context.Function<PFN_vkQueueSubmit>("vkQueueSubmit")(context.queue, 1, &submit, fence)) != VK_SUCCESS) return "vkQueueSubmit";
submitted = true;
if ((probe.result = context.Function<PFN_vkWaitForFences>("vkWaitForFences")(context.device, 1, &fence, VK_TRUE, 10'000'000'000ull)) != VK_SUCCESS) return "vkWaitForFences";
submitted = false;
const auto execute = [&](VkCommandBuffer buffer) -> const char* {
if ((probe.result = context.Function<PFN_vkResetFences>("vkResetFences")(context.device, 1, &fence)) != VK_SUCCESS) return "vkResetFences";
VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO};
submit.commandBufferCount = 1;
submit.pCommandBuffers = &buffer;
if ((probe.result = context.Function<PFN_vkQueueSubmit>("vkQueueSubmit")(context.queue, 1, &submit, fence)) != VK_SUCCESS) return "vkQueueSubmit";
submitted = true;
if ((probe.result = context.Function<PFN_vkWaitForFences>("vkWaitForFences")(context.device, 1, &fence, VK_TRUE, 10'000'000'000ull)) != VK_SUCCESS) return "vkWaitForFences";
submitted = false;
return nullptr;
};
if (const char* step = execute(commands[0])) return step;
if (!collect(probe.writtenAfterSubmit)) return "the collect after the submission";
if (const char* step = execute(commands[1])) return step;
if (!collect(probe.writtenAfterFill)) return "the collect after the fill";
*reinterpret_cast<volatile std::uint8_t*>(base) = 2;
if (!collect(probe.writtenByCpu)) return "the collect after a CPU store";
return nullptr;
@ -1561,7 +1791,7 @@ void runImportProbe(const Context& context, std::uint64_t base, std::uint64_t by
probe.failure = run();
if (submitted) context.Function<PFN_vkQueueWaitIdle>("vkQueueWaitIdle")(context.queue);
if (fence != VK_NULL_HANDLE) context.Function<PFN_vkDestroyFence>("vkDestroyFence")(context.device, fence, nullptr);
if (commands != VK_NULL_HANDLE) context.Function<PFN_vkFreeCommandBuffers>("vkFreeCommandBuffers")(context.device, context.pool, 1, &commands);
if (commands[0] != VK_NULL_HANDLE) context.Function<PFN_vkFreeCommandBuffers>("vkFreeCommandBuffers")(context.device, context.pool, 2, commands);
if (destination != VK_NULL_HANDLE) context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, destination, nullptr);
if (destinationMemory != VK_NULL_HANDLE) context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, destinationMemory, nullptr);
if (import.buffer != VK_NULL_HANDLE) destroyImport(context, import);
@ -1678,8 +1908,36 @@ const HostImport* HostImportFor(const Context& context, std::uint64_t address, s
}
const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
refreshImports(context, state, lease);
if (const auto* range = containingRange(lease, address, address + bytes)) return importAllocation(context, state, range->address, range->bytes, lease);
return nullptr;
if (!runCovers(lease, address, address + bytes)) return nullptr;
const auto* entry = importAllocation(context, state, address, lease);
return entry != nullptr && findImport(state, address, address + bytes) == entry ? entry : nullptr;
}
bool ImportMappedRanges(const Context& context, const GuestAllocations::Mapped& ranges, std::uint64_t generation, bool adoptDevice) {
if (context.hostImportAlignment == 0) return true;
ensurePinWaiter();
auto& state = Imports();
std::lock_guard lock(state.mutex);
if (!adoptDevice && (state.device != context.device || state.watchDevice != context.device)) return false;
const bool adopting = state.device != context.device;
GuestAllocations::Lease current;
if (adopting) {
current = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
refreshImports(context, state, current);
} else if (state.refreshedGeneration > generation) {
return true;
}
if (!adopting) current = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
for (const auto& mapped : ranges) {
const auto range = mapped.lock();
if (range == nullptr || leasedRangeOwner(current, range->address) != range) continue;
try {
if (importAllocation(context, state, range->address, current) == nullptr && GuestAllocations::GuestAllocationsGeneration_nid_postfix() != generation) state.failed.erase(importRun(current, range->address).base);
} catch (const std::exception& error) {
std::fprintf(stderr, "[gpu] import of GPU memory mapped at 0x%llx+0x%zx failed: %s\n", static_cast<unsigned long long>(range->address), range->bytes, error.what());
}
}
return true;
}
bool RegisteredReadableCovers(std::uint64_t address, std::size_t bytes) {
@ -1688,6 +1946,13 @@ bool RegisteredReadableCovers(std::uint64_t address, std::size_t bytes) {
return containingRange(lease, address, address + bytes) != nullptr;
}
std::uint64_t RegisteredReadableEnd(std::uint64_t address) {
if (address == std::numeric_limits<std::uint64_t>::max()) return 0;
const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
const auto* range = containingRange(lease, address, address + 1);
return range != nullptr ? range->address + range->bytes : 0;
}
bool HostImportCovers(const Context& context, std::uint64_t address, std::size_t bytes) {
if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return false;
auto& state = Imports();
@ -1768,6 +2033,15 @@ void GuestBufferMemory::AcquireRegistered() {
else if (current->importsEpoch != state.epoch) spaces.rebuiltEpoch.fetch_add(1, std::memory_order_relaxed);
else hit = true;
}
if (hit) {
for (auto* mirror : current->sparseMirrors) {
if (refreshSparseMirror(*mirror, blocks)) continue;
hit = false;
blocks.clear();
spaces.current.compare_exchange_strong(current, std::shared_ptr<const AddressSpace>{});
break;
}
}
if (hit) {
spaces.hits.fetch_add(1, std::memory_order_relaxed);
space = std::move(current);
@ -1817,7 +2091,7 @@ void GuestBufferMemory::AcquireRegistered() {
if (importable) {
auto& state = Imports();
std::lock_guard lock(state.mutex);
entry = importAllocation(context, state, range->address, range->bytes, lease);
entry = importAllocation(context, state, range->address, lease);
}
lap(timing.importsUs, at);
if (entry != nullptr) {
@ -1833,7 +2107,12 @@ void GuestBufferMemory::AcquireRegistered() {
lap(timing.mirrorsUs, at);
if (region.mirror != nullptr) {
mirrored = true;
if (region.mirror->heap) heaps.push_back(region.mirror.get());
if (region.mirror->sparse) {
region.sparse = true;
region.backed = region.mirror->committed;
} else if (region.mirror->heap) {
heaps.push_back(region.mirror.get());
}
regions.push_back(std::move(region));
continue;
}
@ -1997,12 +2276,17 @@ const GuestBufferMemory::Region* GuestBufferMemory::owner(std::uint64_t address)
return own;
}
void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic) {
void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic, bool swept) {
validate(address, bytes);
const auto begin = address & ~std::uint64_t{3};
const auto end = begin == address ? address + bytes : (address + bytes + 3) & ~std::uint64_t{3};
switch (baseOverlap(begin, end, nullptr)) {
const Region* inside = nullptr;
switch (baseOverlap(begin, end, &inside)) {
case BaseOverlap::Inside:
if (!servesView(*inside, begin)) {
dissolveSpace(false);
break;
}
return;
case BaseOverlap::Partial:
Spaces().dissolvedOverlap.fetch_add(1, std::memory_order_relaxed);
@ -2015,18 +2299,62 @@ void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t b
// the shader stores to them; what is written back is decided by Writes() alone.
Region region{begin, end, true, {}, nullptr};
region.atomic = atomic;
region.swept = swept;
region.lastView = begin;
auto committed = GuestMemory::DescribeCommitted(begin, static_cast<std::size_t>(end - begin));
if (!committed.whole) {
// A GPU heap bound whole while the guest commits its pages on demand, or a descriptor left
region.sparse = true;
region.sparseHeap = true;
region.backed = std::move(committed.ranges);
}
regions.push_back(std::move(region));
regionsSorted = false;
}
void GuestBufferMemory::AddWritable(std::uint64_t address, std::size_t bytes, bool atomic) {
addDescriptorRegion(address, bytes, atomic);
bool GuestBufferMemory::servesView(const Region& region, std::uint64_t view) {
return region.mirror == nullptr || view + 4 <= region.mirror->base + region.mirror->bytes;
}
bool GuestBufferMemory::sparseHeapCandidate(const Region& region) {
return region.sparseHeap && region.sparse && region.mirror == nullptr && region.direct == nullptr && (region.hostBacked || region.writable) && mirrorsEnabled() && sparseMirrorsEnabled();
}
bool GuestBufferMemory::bindSparseMirror(Region& region, const GuestAllocations::Lease& lease) {
const auto backedEnd = region.backed.empty() ? region.begin : region.backed.back().second;
const auto neededEnd = std::min(region.end, std::max(backedEnd, std::max(region.lastView, region.begin) + 4u));
const auto* range = containingRange(lease, region.begin, neededEnd);
if (range == nullptr || !range->releasable) return false;
std::vector<RefreshBlock> blocks;
std::shared_ptr<ImageMirror> mirror;
try {
mirror = acquireMirror(context, leasedRangeOwner(lease, range->address), blocks, true);
} catch (const std::runtime_error& error) {
static std::atomic<int> reported{0};
if (reported.fetch_add(1, std::memory_order_relaxed) < 4) std::fprintf(stderr, "[gpu] %s; copying the sparse heap instead\n", error.what());
return false;
}
if (mirror != nullptr && mirror->heap && !mirror->sparse) {
std::vector<ImageMirror*> whole{mirror.get()};
refreshHeapMirrors(whole, blocks);
}
compareBlocks(blocks);
if (mirror == nullptr || !mirror->heap || region.begin < mirror->base || (region.begin - mirror->base) % context.limits.minStorageBufferOffsetAlignment != 0) return false;
region.mirror = std::move(mirror);
if (!servesView(region, std::max(region.lastView, region.begin))) {
region.mirror = nullptr;
return false;
}
if (region.mirror->sparse) region.backed = clipPieces(region.mirror->committed, region.begin, region.end);
else region.backed = clipPieces({{region.mirror->base, region.mirror->base + region.mirror->bytes}}, region.begin, region.end);
region.snapshot.clear();
std::lock_guard lock(Mirrors().mutex);
++Mirrors().sparseBinds;
return true;
}
void GuestBufferMemory::AddWritable(std::uint64_t address, std::size_t bytes, bool atomic, bool swept) {
addDescriptorRegion(address, bytes, atomic, swept);
writes.emplace_back(address, address + bytes);
}
@ -2034,7 +2362,7 @@ void GuestBufferMemory::AddReadable(std::uint64_t address, std::size_t bytes) {
// Not in `writes`: no reference copy (copyRegion), no write-back, no pending-write note, no
// direct-write mark, and UploadPrepare copies it without the device lock. A written descriptor
// overlapping the range still covers it through its own Writes() entry.
addDescriptorRegion(address, bytes, false);
addDescriptorRegion(address, bytes, false, false);
}
void GuestBufferMemory::AddSnapshot(const GuestMemorySnapshot& snapshot) {
@ -2157,6 +2485,8 @@ VkBufferUsageFlags gpuCopyUsage(bool addressable) {
// APS5_WRITTEN_SHADOW_MIN_KIB / APS5_WRITTEN_SHADOW_MAX_KIB (16 / 2048) bound the written window
// (a kernel streaming once through a large buffer, the engine's memcpy kernel over 4-8 MiB video
// frames, would only gain the two copies), APS5_ATOMIC_STAGE_MAX_KIB (1024) the atomic one.
constexpr std::uint64_t SweptShadowMax = std::uint64_t{16} << 20u;
bool writtenShadowEnabled() {
static const bool disabled = std::getenv("APS5_NO_WRITTEN_SHADOW") != nullptr;
return !disabled;
@ -2297,7 +2627,7 @@ bool GuestBufferMemory::stagingEligible(const Region& region, bool addressable)
const auto bytes = region.end - region.begin;
if (!WritesOverlap(region.begin, static_cast<std::size_t>(bytes))) return false;
if (region.atomic && atomicStagingEnabled() && bytes <= atomicStageMax()) return true;
return writtenShadowEnabled() && bytes >= writtenShadowMin() && bytes <= writtenShadowMax();
return writtenShadowEnabled() && bytes >= writtenShadowMin() && bytes <= (region.swept ? std::max(writtenShadowMax(), SweptShadowMax) : writtenShadowMax());
}
void GuestBufferMemory::UploadPrepare(bool addressable) {
@ -2331,13 +2661,17 @@ void GuestBufferMemory::UploadPrepare(bool addressable) {
// crossing the mirror's end cannot be served by it: this build copies the range.
if (previous.mirror == nullptr) std::swap(previous, region);
Require(region.mirror == nullptr, "image mirrors overlap");
if (region.begin >= previous.begin && region.end <= previous.end) continue;
if (region.begin >= previous.begin && region.end <= previous.end && servesView(previous, std::max(region.lastView, region.begin))) continue;
if (previous.mirror->sparse) previous.backed = clipPieces(previous.mirror->committed, previous.begin, previous.end);
previous.mirror = nullptr;
previous.hostBacked = true;
}
mergeBacked(previous, region);
previous.lastView = std::max({previous.lastView, region.lastView, region.begin});
if (region.end > previous.end) previous.direct = nullptr;
previous.atomic = previous.atomic || region.atomic;
previous.swept = previous.swept || region.swept;
previous.sparseHeap = previous.sparseHeap || region.sparseHeap;
// A range starting before the mirror (only possible after the swap) keeps its prefix; the
// earlier merged region ends at or before it, so the merged list stays sorted.
previous.begin = std::min(previous.begin, region.begin);
@ -2391,6 +2725,11 @@ void GuestBufferMemory::UploadPrepare(bool addressable) {
const bool stale = importsStale(context, state);
for (auto& region : regions) {
if (region.mirror != nullptr) continue;
if (sparseHeapCandidate(region)) {
region.direct = nullptr;
region.pending = true;
continue;
}
// An import found when the lease was acquired is reused while none was dropped since.
const HostImport* entry = region.direct != nullptr && state.epoch == importsEpoch ? region.direct : nullptr;
region.direct = nullptr;
@ -2447,7 +2786,7 @@ void GuestBufferMemory::UploadPrepare(bool addressable) {
// GuestMemory::Read stores pending GPU results first (the flush hook locks for itself).
for (auto& region : regions) {
if (region.pending || region.mirror != nullptr || region.direct != nullptr) continue;
if (region.writable && WritesOverlap(region.begin, static_cast<std::size_t>(region.end - region.begin))) {
if (sparseHeapCandidate(region) || (region.writable && WritesOverlap(region.begin, static_cast<std::size_t>(region.end - region.begin)))) {
region.pending = true;
continue;
}
@ -2525,6 +2864,12 @@ void GuestBufferMemory::UploadFinish(bool addressable) {
if (profile) refreshUs += microsecondsSince(refreshStart);
continue;
}
if (sparseHeapCandidate(region)) {
const auto refreshStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
const bool bound = bindSparseMirror(region, importRanges());
if (profile) refreshUs += microsecondsSince(refreshStart);
if (bound) continue;
}
bool copyOnGpu = false;
if (context.hostImportAlignment != 0) {
auto& state = Imports();
@ -2540,7 +2885,7 @@ void GuestBufferMemory::UploadFinish(bool addressable) {
region.direct = nullptr;
if (entry == nullptr) entry = findImport(state, region.begin, region.end);
if (entry == nullptr) {
if (const auto* range = containingRange(importRanges(), region.begin, region.end)) entry = importAllocation(context, state, range->address, range->bytes, importRanges());
if (runCovers(importRanges(), region.begin, region.end)) entry = importAllocation(context, state, region.begin, importRanges());
}
// A staged region (see stagingEligible) is copied out of the import even when aligned;
// without a recorder to record the copies it binds in place like any other.
@ -2661,8 +3006,17 @@ void GuestBufferMemory::copyRegion(Region& region, bool addressable) {
}
}
const auto allocateStart = std::chrono::steady_clock::now();
if (region.sparse && region.buffer == nullptr && (region.hostBacked || region.writable)) {
const auto backedEnd = region.backed.empty() ? region.begin : region.backed.back().second;
const auto viewEnd = std::max(region.lastView, region.begin) + 4u;
constexpr std::uint64_t granule = 256;
const auto extent = std::max(backedEnd, viewEnd) - region.begin;
const auto end = region.begin + std::min<std::uint64_t>((extent + granule - 1) / granule * granule, bytes);
region.bufferEnd = end < region.end ? end : 0;
}
const auto allocated = region.bufferEnd != 0 ? region.bufferEnd - region.begin : bytes;
// A buffer made by UploadPrepare for a GPU copy that fell back here (its import went) is kept.
if (region.buffer == nullptr) region.buffer = std::make_shared<Buffer>(context, static_cast<std::size_t>(bytes), usage);
if (region.buffer == nullptr) region.buffer = std::make_shared<Buffer>(context, static_cast<std::size_t>(allocated), usage);
const auto readStart = std::chrono::steady_clock::now();
if (profile) allocateUs.fetch_add(microsecondsSince(allocateStart), std::memory_order_relaxed);
// Named for the [hooksync] attribution: the reads below go through the flush hook.
@ -2676,11 +3030,18 @@ void GuestBufferMemory::copyRegion(Region& region, bool addressable) {
if (written) region.uploaded.assign(region.buffer->Bytes().begin(), region.buffer->Bytes().end());
} else {
// `uploaded` holds the backed pieces back to back.
const auto target = region.buffer->Bytes().first(static_cast<std::size_t>(allocated));
const bool zeroGaps = region.bufferEnd != 0;
std::size_t cursor = 0;
for (const auto& [first, last] : region.backed) {
const auto piece = region.buffer->Bytes().subspan(static_cast<std::size_t>(first - region.begin), static_cast<std::size_t>(last - first));
const auto at = static_cast<std::size_t>(first - region.begin);
if (zeroGaps && at > cursor) std::memset(target.data() + cursor, 0, at - cursor);
const auto piece = target.subspan(at, static_cast<std::size_t>(last - first));
GuestMemory::Read(first, piece);
if (written) region.uploaded.insert(region.uploaded.end(), piece.begin(), piece.end());
cursor = at + piece.size();
}
if (zeroGaps && cursor < target.size()) std::memset(target.data() + cursor, 0, target.size() - cursor);
}
}
else std::memcpy(region.buffer->Bytes().data(), region.snapshot.data(), region.snapshot.size());
@ -2859,15 +3220,25 @@ VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std:
const auto* found = owner(address);
Require(found != nullptr, "guest buffer has no GPU owner");
const auto& region = *found;
Require(address >= region.begin && address + bytes <= region.end && (region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr), "guest buffer view exceeds its GPU owner");
const auto describe = [&](const char* what) {
char text[256];
std::snprintf(text, sizeof(text), "guest buffer view 0x%llx+0x%llx %s its GPU owner 0x%llx-0x%llx (buffer %d, direct %d, mirror %d)", static_cast<unsigned long long>(address), static_cast<unsigned long long>(bytes), what, static_cast<unsigned long long>(region.begin), static_cast<unsigned long long>(region.end), region.buffer != nullptr, region.direct != nullptr, region.mirror != nullptr);
return std::string(text);
};
if (!(address >= region.begin && address + bytes <= region.end && (region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr))) Require(false, describe("exceeds"));
const auto base = region.direct != nullptr ? region.direct->base : region.mirror != nullptr ? region.mirror->base : region.begin;
const auto offset = address - base;
Require(context.limits.minStorageBufferOffsetAlignment != 0, "no storage buffer offset alignment");
Require(base % 4 == 0, "a guest buffer view in a GPU owner that does not start at a DWORD boundary is not implemented");
adjustment = static_cast<std::uint32_t>(offset % std::max<VkDeviceSize>(context.limits.minStorageBufferOffsetAlignment, 4));
const auto range = ViewBytes(bytes, adjustment);
const auto end = region.direct != nullptr ? region.direct->base + region.direct->bytes : region.end;
Require(address - adjustment + range <= end, "guest buffer view exceeds its GPU owner");
auto range = ViewBytes(bytes, adjustment);
const auto end = region.direct != nullptr ? region.direct->base + region.direct->bytes : servedEnd(region);
const bool cut = region.direct == nullptr && end < region.end;
if (cut) {
Require(address - adjustment < end, "guest buffer view starts past its copy");
range = std::min<std::uint64_t>(range, end - (address - adjustment));
}
if (address - adjustment + range > end) Require(false, describe("rounded up to its view range exceeds"));
Require(range <= context.limits.maxStorageBufferRange, "guest buffer view exceeds descriptor range limit");
const auto handle = region.direct != nullptr ? region.direct->buffer : region.mirror != nullptr ? region.mirror->buffer->Handle() : region.buffer->Handle();
return {handle, offset - adjustment, range};
@ -2882,7 +3253,12 @@ ShaderRecompiler::BdaAbi::Range GuestBufferMemory::addressRange(const Region& re
const auto address = region.direct != nullptr ? region.direct->address + (region.begin - region.direct->base) : region.mirror != nullptr ? region.mirror->buffer->DeviceAddress() + (region.begin - region.mirror->base) : region.buffer->DeviceAddress();
Require(region.end - region.begin <= std::numeric_limits<std::uint64_t>::max() - address, "GPU address range overflow");
const auto permissions = ShaderRecompiler::BdaAbi::Read | (region.direct != nullptr && region.writable ? ShaderRecompiler::BdaAbi::Write : 0u);
return {region.begin, region.end, address, permissions, 0};
return {region.begin, region.direct != nullptr ? region.end : servedEnd(region), address, permissions, 0};
}
std::uint64_t GuestBufferMemory::servedEnd(const Region& region) {
if (region.mirror != nullptr) return std::min(region.end, region.mirror->base + region.mirror->bytes);
return region.bufferEnd != 0 ? region.bufferEnd : region.end;
}
std::vector<ShaderRecompiler::BdaAbi::Range> GuestBufferMemory::AddressRanges() const {
@ -3009,6 +3385,25 @@ void storeChanged(std::uint64_t address, std::span<const std::byte> current, std
if (cursor < address + current.size()) unchanged(cursor, address + current.size());
}
void storeSparse(ImageMirror& mirror, std::uint64_t address, std::span<const std::byte> current, std::span<const std::byte> original) {
constexpr std::uint64_t block = 65536;
const auto end = address + current.size();
const auto forget = [&](std::uint64_t from, std::uint64_t to) {
const auto at = static_cast<std::size_t>(from - address);
if (from >= to || std::memcmp(current.data() + at, original.data() + at, static_cast<std::size_t>(to - from)) == 0) return;
for (auto index = from / block - mirror.base / block; index <= (to - 1) / block - mirror.base / block; ++index) mirror.generations[static_cast<std::size_t>(index)] = 0;
};
auto cursor = address;
for (const auto& [first, last] : GuestMemory::DescribeCommitted(address, current.size(), true).ranges) {
forget(cursor, first);
const auto at = static_cast<std::size_t>(first - address);
const auto length = static_cast<std::size_t>(last - first);
GuestMemory::WriteChanged(first, current.subspan(at, length), original.subspan(at, length));
cursor = last;
}
forget(cursor, end);
}
std::vector<std::pair<std::uint64_t, std::uint64_t>> mergeWrites(std::vector<std::pair<std::uint64_t, std::uint64_t>> writes) {
std::sort(writes.begin(), writes.end());
std::vector<std::pair<std::uint64_t, std::uint64_t>> merged;
@ -3028,7 +3423,9 @@ void GuestBufferMemory::takeHeapReferences() {
if (found == nullptr || found->mirror == nullptr || !found->mirror->heap || !found->mirror->writable) continue;
Require(end <= found->end, "written range exceeds its heap mirror");
const auto& mirror = *found->mirror;
const auto bytes = mirror.buffer->Bytes().subspan(static_cast<std::size_t>(begin - mirror.base), static_cast<std::size_t>(end - begin));
const auto served = std::min(end, servedEnd(*found));
if (begin >= served) continue;
const auto bytes = mirror.buffer->Bytes().subspan(static_cast<std::size_t>(begin - mirror.base), static_cast<std::size_t>(served - begin));
heapReferences.emplace_back(begin, std::vector<std::byte>(bytes.begin(), bytes.end()));
}
}
@ -3056,12 +3453,15 @@ void GuestBufferMemory::WriteBack() {
// refresh does not mistake the GPU's results for a guest change.
auto& mirror = *region.mirror;
const auto offset = static_cast<std::size_t>(begin - mirror.base);
const auto length = static_cast<std::size_t>(end - begin);
const auto served = std::min(end, servedEnd(region));
if (mirror.heap && begin >= served) continue;
const auto length = static_cast<std::size_t>((mirror.heap ? served : end) - begin);
const auto current = mirror.buffer->Bytes().subspan(offset, length);
if (mirror.heap) {
const auto reference = std::find_if(heapReferences.begin(), heapReferences.end(), [&](const auto& entry) { return entry.first == begin; });
Require(reference != heapReferences.end() && reference->second.size() == length, "heap mirror write-back has no reference");
storeChanged(begin, current, reference->second);
if (mirror.sparse) storeSparse(mirror, begin, current, reference->second);
else storeChanged(begin, current, reference->second);
continue;
}
GuestMemory::WriteChanged(begin, current, std::span<const std::byte>(mirror.shadow).subspan(offset, length));

10
core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp
View File

@ -54,7 +54,7 @@ float toSignedLodBias(std::uint32_t raw) {
}
GuestSamplerResource DecodeSamplerResource(std::span<const std::uint32_t> words, bool unnormalizedProven, bool forceDegammaPaired) {
GuestSamplerResource DecodeSamplerResource(std::span<const std::uint32_t> words, bool unnormalizedProven, bool forceDegammaPaired, bool integerBorder) {
Require(words.size() == 4, "guest sampler descriptor must contain 4 dwords");
const auto clampX = (words[0] >> 0u) & 0x7u;
@ -135,12 +135,12 @@ GuestSamplerResource DecodeSamplerResource(std::span<const std::uint32_t> words,
VkBorderColor border;
switch (borderColorType) {
case 0: border = VK_BORDER_COLOR_INT_TRANSPARENT_BLACK; break;
case 1: border = VK_BORDER_COLOR_INT_OPAQUE_BLACK; break;
case 2: border = VK_BORDER_COLOR_INT_OPAQUE_WHITE; break;
case 0: border = integerBorder ? VK_BORDER_COLOR_INT_TRANSPARENT_BLACK : VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; break;
case 1: border = integerBorder ? VK_BORDER_COLOR_INT_OPAQUE_BLACK : VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK; break;
case 2: border = integerBorder ? VK_BORDER_COLOR_INT_OPAQUE_WHITE : VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE; break;
default:
if (readsBorderColor(clampX) || readsBorderColor(clampY) || readsBorderColor(clampZ)) throw std::runtime_error("AGC graphics: guest sampler descriptor uses a border color table which is not implemented");
border = VK_BORDER_COLOR_INT_TRANSPARENT_BLACK;
border = integerBorder ? VK_BORDER_COLOR_INT_TRANSPARENT_BLACK : VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
break;
}

68
core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp
View File

@ -128,9 +128,35 @@ void appendKey(std::vector<std::byte>& key, const TValue& value) {
key.insert(key.end(), bytes.begin(), bytes.end());
}
bool fragmentWritesMemory(std::span<const CompiledShader> shaders) {
const auto any = [](const std::vector<bool>& written, std::uint32_t count) { return written.size() < count || std::ranges::any_of(written, [](bool element) { return element; }); };
for (const auto& shader : shaders) {
if (shader.stage != ShaderRecompiler::ShaderStage::Fragment || shader.program == nullptr) continue;
for (const auto& binding : shader.program->bindings) {
switch (binding.role) {
case ShaderRecompiler::DescriptorRole::GuestBuffers:
if (binding.kind != ShaderRecompiler::DescriptorKind::UniformBuffer && binding.kind != ShaderRecompiler::DescriptorKind::UniformTexelBuffer && any(binding.bufferWritten, binding.count)) return true;
break;
case ShaderRecompiler::DescriptorRole::GuestImages:
if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage && any(binding.imageWritten, binding.count)) return true;
break;
case ShaderRecompiler::DescriptorRole::Gds:
case ShaderRecompiler::DescriptorRole::BdaPagetable:
return true;
case ShaderRecompiler::DescriptorRole::GuestSamplers:
case ShaderRecompiler::DescriptorRole::FaultBuffer:
case ShaderRecompiler::DescriptorRole::FlattenedSrt:
case ShaderRecompiler::DescriptorRole::ShaderData:
break;
}
}
}
return false;
}
bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& state, const VertexInputLayout& input, const ShaderResources& resources, std::span<const CompiledShader> shaders) {
using Stage = ShaderRecompiler::ShaderStage;
if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList) return false;
if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList || SplitsAlternateAlpha(state)) return false;
keys = {};
for (const auto& shader : shaders) {
if (shader.stage == Stage::Geometry || shader.program->PipelineVariantId() == 0) return false;
@ -212,6 +238,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
std::vector<VkPipelineShaderStageCreateInfo> stages(shaders.size());
std::vector<PipelineSpecialization> specializations;
specializations.reserve(shaders.size());
const VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT meshSubgroup{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT, nullptr, 32u};
for (std::uint32_t i = 0; i < shaders.size(); ++i) {
const auto& shader = *shaders[i].program;
VkShaderModuleCreateInfo module{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
@ -223,6 +250,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
stages[i].stage = stage;
stages[i].module = _modules[i];
stages[i].pName = "main";
if (shaders[i].stage == ShaderRecompiler::ShaderStage::Mesh && context.meshWave32 && shader.hostSubgroupSize == 32u) stages[i].pNext = &meshSubgroup;
specializations.emplace_back(shader);
stages[i].pSpecializationInfo = specializations.back().Info();
}
@ -345,8 +373,28 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
depthStencil.front = state.stencilFront;
depthStencil.back = state.stencilBack;
VkPipelineColorBlendStateCreateInfo blend{VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO};
blend.attachmentCount = static_cast<std::uint32_t>(state.blends.size());
blend.pAttachments = state.blends.empty() ? nullptr : state.blends.data();
const bool alternateAlpha = SplitsAlternateAlpha(state);
std::vector<VkPipelineColorBlendAttachmentState> firstChannels(state.blends.begin(), state.blends.end());
std::vector<VkPipelineColorBlendAttachmentState> secondChannels(state.blends.begin(), state.blends.end());
if (alternateAlpha) {
const auto keepsStencil = [](const VkStencilOpState& face) { return face.writeMask == 0 || (face.failOp == VK_STENCIL_OP_KEEP && face.passOp == VK_STENCIL_OP_KEEP && face.depthFailOp == VK_STENCIL_OP_KEEP); };
Require(!(state.depth && (state.depthWrite || (state.stencilTest && (!keepsStencil(state.stencilFront) || !keepsStencil(state.stencilBack))))) && !state.depthBiasPerFace, "blending into an 8_8 color target with the alternate component order and a separate alpha blend from a draw that writes depth or stencil is not implemented");
Require(!fragmentWritesMemory(shaders), "blending into an 8_8 color target with the alternate component order and a separate alpha blend from a pixel shader that writes memory is not implemented");
for (std::size_t index = 0; index < secondChannels.size(); ++index) {
const bool split = index < state.colors.size() && state.colors[index].componentMapping == 0xecu && state.blends[index].blendEnable;
if (!split) {
secondChannels[index].colorWriteMask = 0;
continue;
}
firstChannels[index].colorWriteMask &= ~VK_COLOR_COMPONENT_G_BIT;
secondChannels[index].colorWriteMask &= VK_COLOR_COMPONENT_G_BIT;
secondChannels[index].srcColorBlendFactor = state.blends[index].srcAlphaBlendFactor;
secondChannels[index].dstColorBlendFactor = state.blends[index].dstAlphaBlendFactor;
secondChannels[index].colorBlendOp = state.blends[index].alphaBlendOp;
}
}
blend.attachmentCount = static_cast<std::uint32_t>(firstChannels.size());
blend.pAttachments = firstChannels.empty() ? nullptr : firstChannels.data();
std::copy(state.blendConstants.begin(), state.blendConstants.end(), blend.blendConstants);
VkGraphicsPipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO};
pipelineInfo.flags = context.pipelineExecutableInfo ? VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR : 0;
@ -369,8 +417,12 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput
pipelineInfo.renderPass = renderPass;
if (dynamicRendering) pipelineInfo.pNext = &rendering;
timing.Mark("modules_and_state");
if (libraries) pipeline = LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized);
if (libraries) pipeline = LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized, _modules);
else Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines");
if (alternateAlpha) {
blend.pAttachments = secondChannels.data();
Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &secondChannel), "vkCreateGraphicsPipelines");
}
if (state.depthBiasPerFace) {
raster.cullMode = VK_CULL_MODE_FRONT_BIT;
Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &backFaces), "vkCreateGraphicsPipelines");
@ -395,6 +447,7 @@ void Pipeline::release() noexcept {
optimized.reset();
if (pipeline) context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline")(context.device, pipeline, nullptr);
if (backFaces) context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline")(context.device, backFaces, nullptr);
if (secondChannel) context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline")(context.device, secondChannel, nullptr);
if (renderPass) context.Function<PFN_vkDestroyRenderPass>("vkDestroyRenderPass")(context.device, renderPass, nullptr);
if (layout) context.Function<PFN_vkDestroyPipelineLayout>("vkDestroyPipelineLayout")(context.device, layout, nullptr);
for (auto module : _modules) {
@ -402,6 +455,7 @@ void Pipeline::release() noexcept {
}
pipeline = VK_NULL_HANDLE;
backFaces = VK_NULL_HANDLE;
secondChannel = VK_NULL_HANDLE;
renderPass = VK_NULL_HANDLE;
layout = VK_NULL_HANDLE;
_modules.clear();
@ -414,6 +468,7 @@ void Pipeline::Abandon() noexcept {
framebuffers.clear();
pipeline = VK_NULL_HANDLE;
backFaces = VK_NULL_HANDLE;
secondChannel = VK_NULL_HANDLE;
renderPass = VK_NULL_HANDLE;
layout = VK_NULL_HANDLE;
_modules.clear();
@ -538,6 +593,11 @@ void Pipeline::setLibraryState(VkCommandBuffer commands, const State& state) con
stencil(VK_STENCIL_FACE_BACK_BIT, depth ? state.stencilBack : VkStencilOpState{});
}
void Pipeline::ContinueSecondChannel(VkCommandBuffer commands) const {
Require(secondChannel != VK_NULL_HANDLE, "the pipeline blends every channel in one pass");
context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, secondChannel);
}
void Pipeline::ContinueBackFaces(VkCommandBuffer commands, const State& state) const {
Require(backFaces != VK_NULL_HANDLE, "the pipeline draws both faces in one pass");
context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, backFaces);

23
core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp
View File

@ -37,8 +37,10 @@ using Key = std::vector<std::byte>;
struct DeviceLibraries {
PFN_vkDestroyPipeline destroyPipeline = nullptr;
PFN_vkDestroyPipelineLayout destroyLayout = nullptr;
PFN_vkDestroyShaderModule destroyModule = nullptr;
std::map<Key, VkPipelineLayout> layouts;
std::array<std::map<Key, VkPipeline>, 4> parts;
std::vector<VkShaderModule> modules;
PipelineLibraryCounters counters;
};
@ -99,7 +101,7 @@ std::span<const VkDynamicState> PipelineLibraryDynamicStates() {
return dynamicStates;
}
VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr<OptimizedPipeline>* optimized) {
VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr<OptimizedPipeline>* optimized, std::span<VkShaderModule> modules) {
Require(info.renderPass == VK_NULL_HANDLE && info.pDynamicState != nullptr, "pipeline library parts need dynamic rendering and the library dynamic states");
auto& store = Libraries();
std::lock_guard lock(store.mutex);
@ -107,6 +109,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
if (device.destroyPipeline == nullptr) {
device.destroyPipeline = context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline");
device.destroyLayout = context.Function<PFN_vkDestroyPipelineLayout>("vkDestroyPipelineLayout");
device.destroyModule = context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule");
}
auto pipelineLayout = device.layouts.find(keys.layout);
if (pipelineLayout == device.layouts.end()) {
@ -135,6 +138,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
const std::array<const Key*, 4> contextKeys{&none, &keys.layout, &keys.layout, &keys.renderPass};
const VkPipelineRenderingCreateInfoKHR shaderRendering{VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO_KHR};
VkPipelineDepthStencilStateCreateInfo dynamicDepth{VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO};
static const bool optimize = std::getenv("APS5_NO_PIPELINE_LTO") == nullptr;
std::vector<VkPipeline> libraries;
for (std::size_t part = 0; part < flags.size(); ++part) {
if (part == 0 && mesh) continue;
@ -179,6 +183,15 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
VkPipeline handle = VK_NULL_HANDLE;
Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &create, nullptr, &handle), "vkCreateGraphicsPipelines library");
cache.emplace(key, handle);
if (optimize) {
for (std::uint32_t stage = 0; stage < create.stageCount; ++stage) {
for (auto& module : modules) {
if (module == VK_NULL_HANDLE || module != create.pStages[stage].module) continue;
device.modules.push_back(module);
module = VK_NULL_HANDLE;
}
}
}
++device.counters.built[part];
libraries.push_back(handle);
}
@ -191,7 +204,6 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
VkPipeline pipeline = VK_NULL_HANDLE;
Check(context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &create, nullptr, &pipeline), "vkCreateGraphicsPipelines link");
++device.counters.linked;
static const bool optimize = std::getenv("APS5_NO_PIPELINE_LTO") == nullptr;
if (optimize && optimized != nullptr) {
auto slot = std::make_shared<OptimizedPipeline>();
slot->device = context.device;
@ -199,15 +211,15 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
*optimized = slot;
const auto createPipelines = context.Function<PFN_vkCreateGraphicsPipelines>("vkCreateGraphicsPipelines");
const auto cache = context.pipelineCache;
const auto pipelineLayout = info.layout;
enqueue(store, context.device, [slot, createPipelines, cache, pipelineLayout, libraries, &store, devicePointer = &device] {
const auto libraryLayout = pipelineLayout->second;
enqueue(store, context.device, [slot, createPipelines, cache, libraryLayout, libraries, &store, devicePointer = &device] {
if (slot->released.load()) return;
VkPipelineLibraryCreateInfoKHR linked{VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR};
linked.libraryCount = static_cast<std::uint32_t>(libraries.size());
linked.pLibraries = libraries.data();
VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &linked};
create.flags = VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT;
create.layout = pipelineLayout;
create.layout = libraryLayout;
VkPipeline result = VK_NULL_HANDLE;
if (createPipelines(slot->device, cache, 1, &create, nullptr, &result) != VK_SUCCESS) return;
slot->handle.store(result);
@ -242,6 +254,7 @@ void ClearPipelineLibraries(VkDevice device) {
for (auto& part : libraries.parts) {
for (const auto& [key, handle] : part) libraries.destroyPipeline(device, handle, nullptr);
}
for (const auto module : libraries.modules) libraries.destroyModule(device, module, nullptr);
for (const auto& [key, handle] : libraries.layouts) libraries.destroyLayout(device, handle, nullptr);
store.devices.erase(found);
}

111
core/libs/prx/libSceAgcDriver/Graphics/src/PlaceholderImages.cpp Normal file
View File

@ -0,0 +1,111 @@
#include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp"
#include <string>
#include <vector>
namespace AgcDriver::Graphics {
namespace {
bool Supported(const Context& context, const PlaceholderImageShape& shape) {
VkFormatProperties properties{};
context.formatProperties(context.physical, shape.format, &properties);
if ((properties.optimalTilingFeatures & shape.features) != shape.features) return false;
VkImageFormatProperties limits{};
if (context.imageFormatProperties(context.physical, shape.format, shape.type, VK_IMAGE_TILING_OPTIMAL, shape.usage, 0, &limits) != VK_SUCCESS) return false;
return (limits.sampleCounts & shape.samples) != 0;
}
}
PlaceholderImages::PlaceholderImages(const Context& context) : context(context) {
try {
std::vector<VkImageMemoryBarrier> toClear;
for (std::uint32_t index = 0; index < images.size(); ++index) {
const auto shape = PlaceholderImageShapeOf(ShaderRecompiler::RuntimeAbi::FirstImageBinding + index);
if (!Supported(context, shape)) continue;
auto& image = images[index];
VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
info.imageType = shape.type;
info.format = shape.format;
info.extent = {1, 1, 1};
info.mipLevels = 1;
info.arrayLayers = 1;
info.samples = shape.samples;
info.tiling = VK_IMAGE_TILING_OPTIMAL;
info.usage = shape.usage;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
Check(context.Function<PFN_vkCreateImage>("vkCreateImage")(context.device, &info, nullptr, &image.image), "vkCreateImage placeholder");
VkMemoryRequirements requirements{};
context.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(context.device, image.image, &requirements);
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &image.memory), "vkAllocateMemory placeholder");
Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image.image, image.memory, 0), "vkBindImageMemory placeholder");
VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
viewInfo.image = image.image;
viewInfo.viewType = shape.viewType;
viewInfo.format = shape.format;
viewInfo.subresourceRange = {shape.aspect, 0, 1, 0, 1};
Check(context.Function<PFN_vkCreateImageView>("vkCreateImageView")(context.device, &viewInfo, nullptr, &image.view), "vkCreateImageView placeholder");
VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
barrier.image = image.image;
barrier.subresourceRange = viewInfo.subresourceRange;
toClear.push_back(barrier);
}
if (toClear.empty()) return;
CommandBatch batch(context);
const auto commands = batch.Handle();
const auto pipelineBarrier = context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier");
pipelineBarrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, static_cast<std::uint32_t>(toClear.size()), toClear.data());
std::vector<VkImageMemoryBarrier> toGeneral;
for (const auto& barrier : toClear) {
if (barrier.subresourceRange.aspectMask == VK_IMAGE_ASPECT_DEPTH_BIT) {
const VkClearDepthStencilValue zero{0.0f, 0};
context.Function<PFN_vkCmdClearDepthStencilImage>("vkCmdClearDepthStencilImage")(commands, barrier.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &zero, 1, &barrier.subresourceRange);
} else {
const VkClearColorValue zero{};
context.Function<PFN_vkCmdClearColorImage>("vkCmdClearColorImage")(commands, barrier.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &zero, 1, &barrier.subresourceRange);
}
auto general = barrier;
general.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
general.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
general.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
general.newLayout = VK_IMAGE_LAYOUT_GENERAL;
toGeneral.push_back(general);
}
pipelineBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, static_cast<std::uint32_t>(toGeneral.size()), toGeneral.data());
batch.SubmitAndWait();
} catch (...) {
release();
throw;
}
}
PlaceholderImages::~PlaceholderImages() {
release();
}
void PlaceholderImages::release() noexcept {
for (auto& image : images) {
if (image.view) context.Function<PFN_vkDestroyImageView>("vkDestroyImageView")(context.device, image.view, nullptr);
if (image.image) context.Function<PFN_vkDestroyImage>("vkDestroyImage")(context.device, image.image, nullptr);
if (image.memory) context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, image.memory, nullptr);
image = {};
}
}
VkImageView PlaceholderImages::View(std::uint32_t heap) const {
const auto index = heap - ShaderRecompiler::RuntimeAbi::FirstImageBinding;
Require(heap >= ShaderRecompiler::RuntimeAbi::FirstImageBinding && index < images.size() && images[index].view != VK_NULL_HANDLE,
"the device has no placeholder image for typed image heap " + std::to_string(heap));
return images[index].view;
}
}

10
core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp
View File

@ -2452,6 +2452,12 @@ void Recorder::Keep(std::shared_ptr<void> object, std::size_t bytes) {
open->keptBytes += bytes;
}
void Recorder::KeepBytes(const void* owner, std::size_t bytes) {
if (owner == nullptr || bytes == 0) return;
ensureOpen();
if (open->keptOwners.insert(owner).second) open->keptBytes += bytes;
}
void Recorder::BoundKeptBytes() {
if (!GuestMemory::GpuMutex().HeldByThisThread()) return;
if (open != nullptr && open->keptBytes >= KeptBytesBudget) Submit();
@ -3221,6 +3227,10 @@ void Recorder::Sync() {
activeSyncSite = previousSite;
}
void Recorder::FlushDeferredReleases() {
ReleaseDeferredKeeps();
}
void Recorder::SyncThrough(std::uint64_t address, std::size_t bytes, bool waitUnlocked) {
if (bytes == 0) return;
const auto end = address + bytes;

67
core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp
View File

@ -3,10 +3,40 @@
#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include <algorithm>
#include <exception>
namespace AgcDriver::Graphics {
namespace {
VkDeviceSize ClassAllocationBytes(const VkMemoryRequirements& requirements, std::size_t capacity) {
const auto alignment = std::max<VkDeviceSize>(requirements.alignment, 1);
return std::max<VkDeviceSize>(requirements.size, (static_cast<VkDeviceSize>(capacity) + alignment - 1) / alignment * alignment);
}
VkBuffer BindExactBuffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize availableBytes) {
VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
info.size = size;
info.usage = usage;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
VkBuffer buffer = VK_NULL_HANDLE;
Check(context.Function<PFN_vkCreateBuffer>("vkCreateBuffer")(context.device, &info, nullptr, &buffer), "vkCreateBuffer pooled");
try {
VkMemoryRequirements requirements{};
context.Function<PFN_vkGetBufferMemoryRequirements>("vkGetBufferMemoryRequirements")(context.device, buffer, &requirements);
Require(requirements.size <= availableBytes, "pooled buffer memory is smaller than the requested buffer");
Require(requirements.alignment == 0 || offset % requirements.alignment == 0, "pooled buffer offset breaks the requested buffer's alignment");
Check(context.Function<PFN_vkBindBufferMemory>("vkBindBufferMemory")(context.device, buffer, memory, offset), "vkBindBufferMemory pooled");
} catch (...) {
context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, buffer, nullptr);
throw;
}
return buffer;
}
}
Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) : context(context), size(size), capacity(BufferPool::Capacity(size, properties)), usage(BufferPool::Usage(usage, properties)), properties(properties) {
Require(size != 0, "zero-sized GPU buffer");
const bool addressable = (usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0;
@ -21,12 +51,24 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag
capacity = allocation->bytes;
offset = allocation->offset;
slab = allocation->slab;
if (allocation->bufferBytes != size) {
context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, buffer, nullptr);
buffer = VK_NULL_HANDLE;
deviceAddress = 0;
try {
buffer = BindExactBuffer(context, size, this->usage, memory, offset, allocationBytes);
initializeAddress(usage);
} catch (...) {
release();
throw;
}
}
ready = true;
return;
}
try {
VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
info.size = capacity;
info.size = size;
info.usage = this->usage;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
Check(context.Function<PFN_vkCreateBuffer>("vkCreateBuffer")(context.device, &info, nullptr, &buffer), "vkCreateBuffer");
@ -35,8 +77,8 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
const VkMemoryAllocateFlagsInfo flags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, nullptr, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0};
if (addressable) allocation.pNext = &flags;
allocation.allocationSize = requirements.size;
allocationBytes = requirements.size;
allocation.allocationSize = ClassAllocationBytes(requirements, capacity);
allocationBytes = allocation.allocationSize;
// The CPU reads most of these buffers back (write-back, diffs), which is very slow from
// write-combined memory, so the default host properties prefer cached host memory.
constexpr VkMemoryPropertyFlags hostDefault = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
@ -78,7 +120,7 @@ Buffer::~Buffer() {
void Buffer::release() noexcept {
if (ready && cache) {
cache->Put({buffer, memory, mapping, deviceAddress, allocationBytes, capacity, usage, properties, offset, slab});
cache->Put({buffer, memory, mapping, deviceAddress, allocationBytes, capacity, usage, properties, offset, slab, size});
return;
}
if (slab) {
@ -117,19 +159,28 @@ DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsa
memory = allocation->memory;
allocationBytes = allocation->allocationBytes;
capacity = allocation->bytes;
if (allocation->bufferBytes == size) return;
context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, buffer, nullptr);
buffer = VK_NULL_HANDLE;
try {
buffer = BindExactBuffer(context, size, this->usage, memory, 0, allocationBytes);
} catch (...) {
release();
throw;
}
return;
}
try {
VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
info.size = capacity;
info.size = size;
info.usage = this->usage;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
Check(context.Function<PFN_vkCreateBuffer>("vkCreateBuffer")(context.device, &info, nullptr, &buffer), "vkCreateBuffer device");
VkMemoryRequirements requirements{};
context.Function<PFN_vkGetBufferMemoryRequirements>("vkGetBufferMemoryRequirements")(context.device, buffer, &requirements);
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
allocationBytes = requirements.size;
allocation.allocationSize = ClassAllocationBytes(requirements, capacity);
allocationBytes = allocation.allocationSize;
allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory device buffer");
Check(context.Function<PFN_vkBindBufferMemory>("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory device");
@ -145,7 +196,7 @@ DeviceBuffer::~DeviceBuffer() {
void DeviceBuffer::release() noexcept {
if (buffer && memory && cache) {
cache->Put({buffer, memory, nullptr, 0, allocationBytes, capacity, usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT});
cache->Put({buffer, memory, nullptr, 0, allocationBytes, capacity, usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 0, false, size});
return;
}
if (buffer) context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")(context.device, buffer, nullptr);

6
core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp
View File

@ -67,9 +67,9 @@ namespace AgcDriver::Graphics {
SamplerCache::SamplerCache(std::size_t capacity) : capacity(std::max<std::size_t>(capacity, 1)) {}
std::shared_ptr<Sampler> SamplerCache::Get(const Context& context, std::span<const std::uint32_t> words, bool compareEnable, bool unnormalizedProven, bool forceDegammaPaired) {
std::shared_ptr<Sampler> SamplerCache::Get(const Context& context, std::span<const std::uint32_t> words, bool compareEnable, bool unnormalizedProven, bool forceDegammaPaired, bool integerBorder) {
Require(words.size() == 4, "guest sampler descriptor must contain 4 dwords");
const std::array<std::uint32_t, 5> key{words[0], words[1], words[2], words[3], (compareEnable ? 1u : 0u) | (unnormalizedProven ? 2u : 0u)};
const std::array<std::uint32_t, 5> key{words[0], words[1], words[2], words[3], (compareEnable ? 1u : 0u) | (unnormalizedProven ? 2u : 0u) | (integerBorder ? 4u : 0u)};
std::lock_guard lock(mutex);
++clock;
if (const auto found = entries.find(key); found != entries.end()) {
@ -79,7 +79,7 @@ namespace AgcDriver::Graphics {
return found->second.sampler;
}
++misses;
auto resource = DecodeSamplerResource(words, unnormalizedProven, forceDegammaPaired);
auto resource = DecodeSamplerResource(words, unnormalizedProven, forceDegammaPaired, integerBorder);
resource.compareEnable = compareEnable;
auto sampler = std::make_shared<Sampler>(context, resource);
// The cap keeps live samplers well below the device's limit (NVIDIA: ~4000); a set in flight

7
core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp
View File

@ -64,9 +64,9 @@ template <typename T> void _readHeaderArray(std::span<const std::byte> header, s
}
ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& shader, std::span<const std::byte> header) {
const auto numThreadX = read(shader, computeNumThreadX, RegisterBank::Shader);
const auto numThreadY = read(shader, computeNumThreadY, RegisterBank::Shader);
const auto numThreadZ = read(shader, computeNumThreadZ, RegisterBank::Shader);
const auto numThreadX = read(shader, computeNumThreadX, RegisterBank::Shader) & 0xffffu;
const auto numThreadY = read(shader, computeNumThreadY, RegisterBank::Shader) & 0xffffu;
const auto numThreadZ = read(shader, computeNumThreadZ, RegisterBank::Shader) & 0xffffu;
if (numThreadX == 0 || numThreadY == 0 || numThreadZ == 0) {
throw std::runtime_error("AGC graphics: COMPUTE_NUM_THREAD_X/Y/Z must be nonzero");
}
@ -163,6 +163,7 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con
}
ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span<const std::byte> header, std::uint64_t headerAddress, std::span<const std::uint32_t> userData, std::vector<DecodeRead>* reads, bool staticAbi) {
if (header.empty()) return {};
if (header.size() < sizeof(Shader)) throw std::runtime_error("AGC graphics: shader header is smaller than the fixed AGC header");
Shader shader;
std::memcpy(&shader, header.data(), sizeof(Shader));

182
core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp
View File

@ -3,6 +3,7 @@
#include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp"
#include "ThreadOwned.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
#include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp"
#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp"
#include <algorithm>
@ -54,7 +55,8 @@ VkComponentSwizzle ComponentSwizzleFor(std::uint8_t dstSel) {
VkComponentMapping ViewComponents(const GuestTextureResource& resource) {
if (IsConvertedTextureFormat(resource.format)) return {VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_A};
return {ComponentSwizzleFor(resource.dstSelX), ComponentSwizzleFor(resource.dstSelY), ComponentSwizzleFor(resource.dstSelZ), ComponentSwizzleFor(resource.dstSelW)};
const auto channel = [&](std::uint8_t dstSel) { return TextureComponentChannel(resource.format, ComponentSwizzleFor(dstSel)); };
return {channel(resource.dstSelX), channel(resource.dstSelY), channel(resource.dstSelZ), channel(resource.dstSelW)};
}
// Sampled textures are reused across draws and dispatches while their guest bytes are unchanged; a
@ -358,6 +360,7 @@ struct LookupRecord {
DccKeys keys;
std::uint64_t generation;
const StorageTexture* source;
bool depth = false;
};
thread_local std::vector<LookupRecord>* lookupLogSlot = nullptr;
@ -408,6 +411,14 @@ bool ClearedViewEnabled() {
return !disabled;
}
std::uint64_t DepthCopyMinBytes() {
static const std::uint64_t bytes = [] {
const char* text = std::getenv("APS5_DEPTH_COPY_MIN_KIB");
return (text != nullptr ? std::strtoull(text, nullptr, 10) : 1024ull) << 10u;
}();
return bytes;
}
// Surfaces whose storage image could not be made (no writable committed pages, say): remembered
// so the sampled lookup does not throw and fall back on every use. A failure is keyed by the surface
// (address and size: the heap reuses addresses) and holds only while the guest mappings are what
@ -503,7 +514,10 @@ std::uint64_t sampledBudget(const Context& context, TextureCache& cache) {
std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource, VkComponentMapping components, std::uint64_t guestBytes = 0, bool depthCompare = false) {
CaptureTrace::Log("sampled-lookup address=%llx width=%u height=%u dcc=%llx", static_cast<unsigned long long>(resource.baseAddress), resource.width, resource.height, static_cast<unsigned long long>(resource.dccAddress));
if (auto* recorder = Recorder::Active(); recorder != nullptr) recorder->BoundKeptBytes();
if (auto depth = DepthSurfaceTexture(context, words, resource, components)) return depth;
if (auto depth = DepthSurfaceTexture(context, words, resource, components)) {
logLookup({depth.get(), resource, 0, DccKeys::Uncompressed, 0, nullptr, true});
return depth;
}
const auto depthBitsWidth = words.size() >= 4 ? ShaderRecompiler::DepthBitsTextureWidth(words[1], words[3]) : 0u;
if (depthBitsWidth == 32u) {
char text[160];
@ -532,8 +546,10 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
// A storage image whose results for this surface (or for a mip chain containing it) are still on
// the GPU supplies the texture by a view of it; anything else needs those results in guest
// memory first.
const bool depthCopy = depthCompare && guestBytes >= DepthCopyMinBytes() && Texture::DepthCopyFormat(ResolveTextureFormat(resource.format)) != VK_FORMAT_UNDEFINED && resource.dimension != TextureDimension::k3D;
auto source = StorageTexture::FindPending(address, guestBytes);
if (source != nullptr && (depthCompare || !Texture::CanCopyFrom(*source, resource) || MetadataMoved(*source, resource))) source.reset();
if (source != nullptr && (depthCompare ? !depthCopy || !Texture::CanDepthCopyFrom(*source, resource) : !Texture::CanCopyFrom(*source, resource))) source.reset();
if (source != nullptr && MetadataMoved(*source, resource)) source.reset();
std::optional<DccKeys> keys;
bool clearThroughKeys = false;
if (source != nullptr && resource.dccAddress != 0 && IsDccClear(source->FilledKeys())) {
@ -557,12 +573,13 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
// import, recorded behind the producer, so no bytes are read or compared on the CPU and nothing
// waits for the producer. A fast-cleared surface (keys) is viewed only through an image whose own
// descriptor carries the DCC address (below); it stays a snapshot otherwise, its texels not read.
if (!depthCompare && source == nullptr && !clearThroughKeys && SampledFromStorageEligible(context, resource, guestBytes)) {
if ((!depthCompare || depthCopy) && source == nullptr && !clearThroughKeys && SampledFromStorageEligible(context, resource, guestBytes)) {
keys = scanKeys();
if (*keys == DccKeys::Uncompressed) {
source = sampledStorageSource(context, resource, guestBytes);
} else if (ClearedViewEnabled() && StorageClearAvailable(context, resource.format, *keys)) {
if (source != nullptr && depthCompare && !Texture::CanDepthCopyFrom(*source, resource)) source.reset();
} else if (!depthCompare && ClearedViewEnabled() && StorageClearAvailable(context, resource.format, *keys)) {
// A fast-cleared surface is viewed as well, through its image cleared on the GPU (the
// lookup's Refresh, see StorageTexture::upload), when the image's own descriptor names
// the same DCC metadata so the refresh sees the keys. A snapshot of the clear texels
@ -597,6 +614,11 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
}
if (!keys.has_value()) keys = scanKeys();
if (disabled) {
if (source != nullptr && depthCompare) {
auto texture = std::make_shared<Texture>(context, source, resource, components, Texture::DepthCopyTag{});
texture->RefreshDepthCopy();
return texture;
}
if (source != nullptr) return std::make_shared<Texture>(context, source, resource, components);
std::vector<std::byte> snapshot(bytes);
ReadTextureSurface(resource, *keys, snapshot);
@ -613,6 +635,7 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
// see (keys, and no pending results to prefer) ends the view: a snapshot holds the clear.
if ((source == nullptr || source == it->source) && (source != nullptr || *keys == DccKeys::Uncompressed) && StorageImageServesKeys(*it->source, resource.dccAddress)) {
if (!GuestMemory::UnchangedSince(address, bytes, it->source->Generation())) it->source->Refresh();
if (it->texture->IsDepthCopy()) it->texture->RefreshDepthCopy();
it->keys = *keys;
touchTexture(cache, it);
logLookup({it->texture.get(), resource, guestBytes, *keys, 0, it->source.get()});
@ -645,7 +668,12 @@ std::shared_ptr<Texture> cachedTexture(const Context& context, std::span<const s
if (source != nullptr) {
entry.source = source;
entry.sourceVersion = source->Version();
entry.texture = std::make_shared<Texture>(context, source, resource, components);
if (depthCompare) {
entry.texture = std::make_shared<Texture>(context, source, resource, components, Texture::DepthCopyTag{});
entry.texture->RefreshDepthCopy();
} else {
entry.texture = std::make_shared<Texture>(context, source, resource, components);
}
entry.accounted = source->Cached() ? 0u : heldBytes(*source);
counters.fromStorage.fetch_add(1, std::memory_order_relaxed);
} else {
@ -1051,7 +1079,7 @@ bool UsesAddressTables(const CompiledShader& compute) {
}
ShaderResources::ShaderResources(const Context& context, const CompiledShader& compute, std::span<const GuestMemorySnapshot> snapshots, bool deferred) : context(context), guestMemory(context), deferredCompute(compute), deferredSnapshots(snapshots) {
ShaderResources::ShaderResources(const Context& context, const CompiledShader& compute, std::span<const GuestMemorySnapshot> snapshots, bool deferred, std::uint64_t dispatchThreads) : context(context), guestMemory(context), dispatchThreads(dispatchThreads), deferredCompute(compute), deferredSnapshots(snapshots) {
Require(compute.stage == ShaderRecompiler::ShaderStage::Compute, "compute resources require a compute shader");
// Every use of a compute build is a recorded dispatch that calls MarkGpuWrites, which staged
// buffers need (a synchronous draw's use would not).
@ -1316,7 +1344,8 @@ void ShaderResources::buildPrepare(std::span<const CompiledShader> shaders, cons
// vector) counts as written, like imageWritten below.
const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element];
const bool atomic = element < binding.bufferAtomic.size() && binding.bufferAtomic[element];
const auto index = addGuestBuffer(std::span<const std::uint32_t>(binding.guestDescriptor).subspan(static_cast<std::size_t>(element) * 4, 4), target, indexAddress, indexBytes, written, atomic);
const bool read = element < binding.bufferRead.size() && binding.bufferRead[element];
const auto index = addGuestBuffer(std::span<const std::uint32_t>(binding.guestDescriptor).subspan(static_cast<std::size_t>(element) * 4, 4), target, indexAddress, indexBytes, written, atomic, read);
const auto& push = shader.program->pushConstants;
if (!push.empty()) {
const auto position = shader.program->memoryOffsetDword * 4u + element;
@ -1449,13 +1478,17 @@ void ShaderResources::buildComplete() {
bufferCount += binding.allocations.size();
if (binding.layout.descriptorType != VK_DESCRIPTOR_TYPE_STORAGE_BUFFER) imageCount += binding.layout.descriptorCount;
}
allocationDescriptors.resize(allocations.size());
std::vector<VkDescriptorBufferInfo> buffers;
std::vector<VkDescriptorImageInfo> images;
auto& buffers = descriptorBuffers;
auto& images = descriptorImages;
buffers.reserve(bufferCount);
images.reserve(imageCount);
std::vector<VkWriteDescriptorSet> writes;
auto& writes = descriptorWrites;
writes.reserve(bindings.size());
const auto absentImage = [this](std::uint32_t binding) {
constexpr auto bindingCount = static_cast<std::uint32_t>(ShaderRecompiler::RuntimeAbi::Binding::Count);
const auto view = context.nullDescriptors ? VK_NULL_HANDLE : context.placeholderImages->View(binding % bindingCount);
return VkDescriptorImageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL};
};
for (const auto& binding : bindings) {
VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
write.dstSet = _set;
@ -1466,14 +1499,13 @@ void ShaderResources::buildComplete() {
case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
write.pBufferInfo = buffers.data() + buffers.size();
for (const auto index : binding.allocations) {
allocationDescriptors[index] = descriptor(allocations[index]);
buffers.push_back(allocationDescriptors[index]);
buffers.push_back(descriptor(allocations[index]));
}
break;
case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
write.pImageInfo = images.data() + images.size();
for (const auto index : binding.imageAllocations) {
if (index == std::numeric_limits<std::size_t>::max()) images.push_back({VK_NULL_HANDLE, VK_NULL_HANDLE, VK_IMAGE_LAYOUT_GENERAL});
if (index == std::numeric_limits<std::size_t>::max()) images.push_back(absentImage(binding.layout.binding));
else images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()});
}
Require(binding.imageAllocations.size() == binding.layout.descriptorCount, "descriptor allocations disagree with compact binding");
@ -1481,7 +1513,7 @@ void ShaderResources::buildComplete() {
case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
write.pImageInfo = images.data() + images.size();
for (const auto index : binding.imageAllocations) {
if (index == std::numeric_limits<std::size_t>::max()) images.push_back({VK_NULL_HANDLE, VK_NULL_HANDLE, VK_IMAGE_LAYOUT_GENERAL});
if (index == std::numeric_limits<std::size_t>::max()) images.push_back(absentImage(binding.layout.binding));
else images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL});
}
Require(binding.imageAllocations.size() == binding.layout.descriptorCount, "descriptor allocations disagree with compact binding");
@ -1557,12 +1589,21 @@ std::atomic<std::uint64_t> resourceCacheTouches{0};
}
namespace {
bool LoopGuardEnabled() {
static const bool enabled = std::getenv("APS5_LOOP_GUARD") != nullptr;
return enabled;
}
}
void ShaderResources::noteReusable() {
// Taken whether or not the object turns out reusable: the lookups' records are this build's.
captureValidation();
reusable = false;
directRegions.clear();
if (NeedsCompletion() || HoldsLease()) return;
if (guestMemory.HasCopiedWrites() || HoldsLease() || usesBda || (bda != nullptr && LoopGuardEnabled())) return;
if (TemplateDataRefresh() && std::any_of(allocations.begin(), allocations.end(), [](const Allocation& allocation) { return allocation.buffer != nullptr && !allocation.guest && allocation.size > MaxRefreshBytes; })) return;
const auto regions = guestMemory.DirectRegions();
if (!regions.has_value()) return;
@ -1580,8 +1621,9 @@ void ShaderResources::noteReusable() {
}
bool ShaderResources::NeverReusable(std::span<const CompiledShader> shaders) {
return std::any_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) {
return shader.program != nullptr && std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [](const ShaderRecompiler::DescriptorBinding& binding) { return binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable || binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer; });
const bool loopGuard = LoopGuardEnabled();
return std::any_of(shaders.begin(), shaders.end(), [loopGuard](const CompiledShader& shader) {
return shader.program != nullptr && std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [loopGuard](const ShaderRecompiler::DescriptorBinding& binding) { return binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable || (loopGuard && binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer); });
});
}
@ -1637,6 +1679,7 @@ void VisitContentKey(const CompiledShader& shader, bool dataWords, bool movableB
packBits(binding.imageDepthCompare);
packBits(binding.imageAtomic);
packBits(binding.samplerUnnormalized);
packBits(binding.samplerIntegerBorder);
packBits(binding.imageUnnormalized);
// Read-only elements are bound without a write set: an object built for one written set
// must not serve a build with another (the variant implies it, this makes it explicit).
@ -1671,7 +1714,7 @@ namespace {
// of the reused ones, fast = every image proved current from stamps, full = the lookups were repeated.
// Why the fast path left an object to the full walk (the "fast-fail by reason" counts).
using FastFail = ShaderResources::FastFail;
constexpr std::array<const char*, static_cast<std::size_t>(FastFail::Count)> FastFailNames{"no record", "collect", "pending image", "evicted image", "memory changed", "keys", "cleared view", "storage keys"};
constexpr std::array<const char*, static_cast<std::size_t>(FastFail::Count)> FastFailNames{"no record", "collect", "pending image", "evicted image", "memory changed", "keys", "cleared view", "storage keys", "depth surface"};
using OwnRefreshFallback = ShaderResources::OwnRefreshFallback;
constexpr std::array<const char*, static_cast<std::size_t>(OwnRefreshFallback::Count)> OwnRefreshFallbackNames{"disabled", "snapshot texture", "cleared view", "foreign view", "surface key", "not imported", "uncached", "re-run failed"};
@ -1823,7 +1866,7 @@ void ShaderResources::captureValidation() {
};
validatedTextures.assign(textures.size(), {});
for (std::size_t i = 0; i < textures.size(); ++i) {
if (const auto* found = record(textures[i].get())) validatedTextures[i] = {found->resource, found->bytes, found->keys, found->generation, 0, found->source, true};
if (const auto* found = record(textures[i].get())) validatedTextures[i] = {found->resource, found->bytes, found->keys, found->generation, 0, found->source, found->depth, true};
}
lookupLog().clear();
}
@ -1879,6 +1922,10 @@ bool ShaderResources::fastRevalidate(std::uint64_t serialBefore, std::span<const
for (std::size_t i = 0; i < textures.size(); ++i) {
auto& surface = validatedTextures[i];
if (!surface.valid) return fail(FastFail::NoRecord);
if (surface.depth) {
if (!DepthSurfaceHolds(context, surface.resource, textures[i].get())) return fail(FastFail::Depth);
continue;
}
const auto address = surface.resource.baseAddress;
const auto bytes = static_cast<std::size_t>(surface.bytes);
surface.collected = GuestMemory::CollectWrites(address, bytes);
@ -1997,6 +2044,10 @@ bool ShaderResources::fastRevalidateEach() {
for (std::size_t i = 0; i < textures.size(); ++i) {
auto& surface = validatedTextures[i];
if (!surface.valid) return false;
if (surface.depth) {
if (!DepthSurfaceHolds(context, surface.resource, textures[i].get())) return false;
continue;
}
const auto address = surface.resource.baseAddress;
const auto bytes = static_cast<std::size_t>(surface.bytes);
surface.collected = GuestMemory::CollectWrites(address, bytes);
@ -2299,6 +2350,9 @@ bool ShaderResources::Revalidate(std::span<const CompiledShader> shaders, ProofR
}
}
pendingSerialSeen = EpochRevalidate() && StorageTexture::PendingSerial() == serialBefore ? serialBefore : 0;
for (const auto& texture : textures) {
if (texture != nullptr && texture->IsDepthCopy()) texture->RefreshDepthCopy();
}
return finish(fast, true);
}
@ -2646,7 +2700,7 @@ DescriptorCache::Stats DescriptorCache::Counters() const {
return stats;
}
std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic) {
std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read) {
Require(words.size() == 4, "buffer descriptor must contain four DWORDs");
Require((words[1] & 0x40000000u) == 0, "buffer descriptor has reserved bits set");
const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}};
@ -2659,13 +2713,23 @@ std::size_t ShaderResources::addGuestBuffer(std::span<const std::uint32_t> words
}
Require(byteSize <= context.limits.maxStorageBufferRange, "shader buffer exceeds descriptor range limit");
Require(byteSize <= std::numeric_limits<std::size_t>::max(), "shader buffer size exceeds host address space");
const auto size = static_cast<std::size_t>(byteSize);
auto size = static_cast<std::size_t>(byteSize);
if (const std::uint64_t stride = descriptor.Stride(); stride != 0) {
constexpr std::uint64_t GuestPageBytes = 0x4000;
const auto end = address + size;
const auto overhang = end % GuestPageBytes;
if (overhang != 0 && overhang < stride) {
const auto allocationEnd = RegisteredReadableEnd(address);
if (allocationEnd > address && allocationEnd < end && end - allocationEnd < stride) size = static_cast<std::size_t>(allocationEnd - address);
}
}
Require(target == nullptr || !overlap(address, size, target->address, target->bytes), "shader buffer aliases the render target");
// APS5_ALL_BUFFERS_WRITTEN=1: every element is noted as written, as before bufferWritten existed.
static const bool allWritten = std::getenv("APS5_ALL_BUFFERS_WRITTEN") != nullptr;
Require(!written || !overlap(address, size, indexAddress, indexBytes), "writable shader buffer aliases the index buffer");
written = written || allWritten;
if (written) guestMemory.AddWritable(address, size, atomic);
const bool swept = read && dispatchThreads != 0 && descriptor.Stride() != 0 && dispatchThreads * descriptor.Stride() * 2u >= byteSize;
if (written) guestMemory.AddWritable(address, size, atomic, swept);
else {
guestMemory.AddReadable(address, size);
++readOnlyBuffers;
@ -2846,7 +2910,7 @@ void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding&
const bool sampledHeap = localBinding >= ShaderRecompiler::RuntimeAbi::FirstImageBinding && localBinding < ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding;
const bool storageHeap = localBinding >= ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding && localBinding < static_cast<std::uint32_t>(ShaderRecompiler::RuntimeAbi::Binding::Samplers);
Require((binding.kind == ShaderRecompiler::DescriptorKind::SampledImage && sampledHeap) || (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage && storageHeap) || (binding.kind == ShaderRecompiler::DescriptorKind::Sampler && heap == ShaderRecompiler::RuntimeAbi::Binding::Samplers), "resource class disagrees with typed descriptor heap");
Require(binding.kind == ShaderRecompiler::DescriptorKind::Sampler || context.nullDescriptors, "typed image heaps require enabled nullDescriptor");
Require(binding.kind == ShaderRecompiler::DescriptorKind::Sampler || context.nullDescriptors || context.placeholderImages != nullptr, "typed image heaps require nullDescriptor or placeholder images");
if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) {
// Storage images are guest textures the shader writes (looked up in stage B).
Require(binding.role == ShaderRecompiler::DescriptorRole::GuestImages, "storage image binding has a non-image role");
@ -2884,14 +2948,15 @@ void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding&
const auto words = std::span<const std::uint32_t>(binding.guestDescriptor).subspan(static_cast<std::size_t>(element) * elementWords, elementWords);
const bool compareEnable = binding.samplerDepthCompare.at(element);
const bool unnormalized = element < binding.samplerUnnormalized.size() && binding.samplerUnnormalized[element];
const bool integerBorder = element < binding.samplerIntegerBorder.size() && binding.samplerIntegerBorder[element];
if (context.samplerCache != nullptr && !SamplerCacheDisabled()) {
samplers.push_back(context.samplerCache->Get(context, words, compareEnable, unnormalized, true));
samplers.push_back(context.samplerCache->Get(context, words, compareEnable, unnormalized, true, integerBorder));
} else {
auto resource = DecodeSamplerResource(words, unnormalized, true);
auto resource = DecodeSamplerResource(words, unnormalized, true, integerBorder);
resource.compareEnable = compareEnable;
samplers.push_back(std::make_shared<Sampler>(context, resource));
}
samplerSources.push_back({{words[0], words[1], words[2], words[3]}, compareEnable, unnormalized});
samplerSources.push_back({{words[0], words[1], words[2], words[3]}, compareEnable, unnormalized, integerBorder});
item.imageAllocations.push_back(samplers.size() - 1);
}
Require(samplers.size() <= context.limits.maxDescriptorSetSamplers, "pipeline sampler descriptors exceed device limits");
@ -3049,9 +3114,9 @@ void ShaderResources::applyAnisoOverride() {
if (!variant.has_value()) continue;
const auto& words = *variant;
if (context.samplerCache != nullptr && !SamplerCacheDisabled()) {
samplers[index] = context.samplerCache->Get(context, words, source.compareEnable, source.unnormalized);
samplers[index] = context.samplerCache->Get(context, words, source.compareEnable, source.unnormalized, false, source.integerBorder);
} else {
auto resource = DecodeSamplerResource(words, source.unnormalized);
auto resource = DecodeSamplerResource(words, source.unnormalized, false, source.integerBorder);
resource.compareEnable = source.compareEnable;
samplers[index] = std::make_shared<Sampler>(context, resource);
}
@ -3325,48 +3390,27 @@ std::shared_ptr<ShaderResources::DrawBindings> ShaderResources::PrepareDrawBindi
result->cache = context.descriptorCache;
result->allocation = result->cache->Allocate(_layout, sizes);
Require(result->allocation.set != VK_NULL_HANDLE, "draw snapshot descriptor allocation failed");
std::vector<VkCopyDescriptorSet> copies;
for (const auto& binding : bindings) {
VkCopyDescriptorSet copy{VK_STRUCTURE_TYPE_COPY_DESCRIPTOR_SET};
copy.srcSet = _set;
copy.srcBinding = binding.layout.binding;
copy.dstSet = result->allocation.set;
copy.dstBinding = binding.layout.binding;
copy.descriptorCount = binding.layout.descriptorCount;
copies.push_back(copy);
}
const auto update = context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets");
update(context.device, 0, nullptr, static_cast<std::uint32_t>(copies.size()), copies.data());
auto buffers = descriptorBuffers;
auto writes = descriptorWrites;
std::vector<std::size_t> snapshotIndices(allocations.size(), selected.size());
for (std::size_t index = 0; index < selected.size(); ++index) snapshotIndices[selected[index]] = index;
std::size_t infoCount = 0;
for (const auto& binding : bindings) infoCount += binding.allocations.size();
std::vector<VkDescriptorBufferInfo> infos;
infos.reserve(infoCount);
std::vector<VkWriteDescriptorSet> writes;
for (const auto& binding : bindings) {
if (std::none_of(binding.allocations.begin(), binding.allocations.end(),
[&](std::size_t index) { return snapshotIndices[index] != selected.size(); })) continue;
const auto first = infos.size();
for (const auto index : binding.allocations) {
const auto snapshotIndex = snapshotIndices[index];
if (snapshotIndex == selected.size()) infos.push_back(allocationDescriptors[index]);
else {
const auto& snapshot = result->snapshots[snapshotIndex];
infos.push_back({snapshot.buffer->Handle(), 0, snapshot.buffer->Bytes().size()});
}
}
VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
for (std::size_t index = 0; index < bindings.size(); ++index) {
const auto& binding = bindings[index];
auto& write = writes[index];
write.dstSet = result->allocation.set;
write.dstBinding = binding.layout.binding;
write.dstArrayElement = 0;
write.descriptorCount = static_cast<std::uint32_t>(binding.allocations.size());
write.descriptorType = binding.layout.descriptorType;
write.pBufferInfo = infos.data() + first;
writes.push_back(write);
if (binding.allocations.empty()) continue;
const auto offset = static_cast<std::size_t>(write.pBufferInfo - descriptorBuffers.data());
write.pBufferInfo = buffers.data() + offset;
for (std::size_t element = 0; element < binding.allocations.size(); ++element) {
const auto snapshotIndex = snapshotIndices[binding.allocations[element]];
if (snapshotIndex == selected.size()) continue;
const auto& snapshot = result->snapshots[snapshotIndex];
buffers[offset + element] = {snapshot.buffer->Handle(), 0, snapshot.buffer->Bytes().size()};
}
}
update(context.device, static_cast<std::uint32_t>(writes.size()), writes.data(), 0, nullptr);
context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast<std::uint32_t>(writes.size()), writes.data(), 0, nullptr);
recorder.Keep(result);
for (const auto& snapshot : result->snapshots) recorder.KeepBytes(snapshot.buffer.get(), snapshot.buffer->Bytes().size());
return result;
}
@ -3421,6 +3465,10 @@ void ShaderResources::MarkGpuWrites(Recorder& recorder) {
AgcDriver::ProfilePrint_nid_no_patch("[buffers] descriptor elements bound: %llu total, %llu written, %llu read-only; %llu pending-write notes skipped\n", static_cast<unsigned long long>(elements), static_cast<unsigned long long>(elements - readOnly), static_cast<unsigned long long>(readOnly), static_cast<unsigned long long>(counters.notesSkipped.load()));
}
void ShaderResources::CheckFaults() const {
if (bda) bda->CheckFault();
}
void ShaderResources::WriteBackBuffers() {
if (bda) bda->CheckFault();
if (SkipWriteBack()) return;

8
core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp
View File

@ -573,10 +573,16 @@ std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders,
for (const auto& [location, signature] : current.inputs) {
const auto output = previous.outputs.find(location);
if (output == previous.outputs.end() || output->second != signature) {
std::string exports;
for (const auto value : shaders[i - 1].program->parameterExports) exports += " " + std::to_string(value);
std::string parameters;
for (const auto& parameter : shaders[i].program->fragmentParameters) parameters += " " + std::to_string(parameter.location) + "<-" + std::to_string(parameter.sourceLocation);
std::string outputs;
for (const auto& [index, kind] : previous.outputs) outputs += " " + std::to_string(index) + ":" + kind;
Require(false, "graphics interfaces disagree at location " + std::to_string(location) +
" (stage " + std::to_string(i - 1) + " output " +
(output == previous.outputs.end() ? "missing" : output->second) +
", stage " + std::to_string(i) + " input " + signature + ")");
", stage " + std::to_string(i) + " input " + signature + "; previous outputs:" + outputs + "; parameter exports:" + exports + "; fragment parameters:" + parameters + ")");
}
}
}

63
core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp
View File

@ -104,6 +104,7 @@ constexpr std::uint32_t DepthControlMask = ~0x007007f0u;
constexpr std::uint32_t ShaderControlMask = ~(0x0000f870u | 0x00020600u | 0x00010000u);
constexpr std::uint32_t PixelStageRunsMask = 0x00020747u;
constexpr std::uint32_t AlphaToCoverageMask = ~0x0001ff00u;
constexpr std::uint32_t AlphaToCoverageUndithered = 0x0000aa01u;
constexpr std::uint32_t ScanModeMask = ~0x22u;
constexpr std::uint32_t ScanControlMask = ~0x76023fffu;
constexpr std::uint32_t ScreenOffsetMask = ~0x01ff01ffu;
@ -226,8 +227,11 @@ void decodeDepth(const Registers& cx, std::uint32_t depthControl, State& result)
}
depth.format = zFormat == 1 ? (stencil ? VK_FORMAT_D16_UNORM_S8_UINT : VK_FORMAT_D16_UNORM) : (stencil ? VK_FORMAT_D32_SFLOAT_S8_UINT : VK_FORMAT_D32_SFLOAT);
depth.clearDepth = readFloat(cx, 0x00b);
if (zFormat != 0 && (read(cx, 0x010) & 0x20000000u) != 0 && find(cx, 0x005) != cx.end()) depth.htileAddress = base(0x005, 0x01e);
depth.clearStencil = static_cast<std::uint8_t>(read(cx, 0x00a) & 0xffu);
if (const auto htile = find(cx, 0x005); zFormat != 0 && htile != cx.end() && htile->second != 0 && (read(cx, 0x010) & (1u << 29u)) != 0) {
depth.htileAddress = base(0x005, 0x01e);
depth.htileStencil = stencil && (read(cx, 0x011) & (1u << 29u)) == 0;
}
result.depth = depth;
result.depthTest = (depthControl & 2u) != 0;
result.depthWrite = result.depthTest && (depthControl & 4u) != 0 && !depthReadOnly;
@ -398,11 +402,13 @@ DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number,
if (number == floating) return single(VK_FORMAT_R16_SFLOAT, 2);
if (number == uint) return single(VK_FORMAT_R16_UINT, 2);
return fail();
case 3:
if (swap != 0) return fail();
if (number == unorm) return {VK_FORMAT_R8G8_UNORM, 2};
if (number == snorm) return {VK_FORMAT_R8G8_SNORM, 2};
case 3: {
if (swap > 1) return fail();
const auto mapping = static_cast<std::uint8_t>(alternate ? 0xecu : 0xe4u);
if (number == unorm) return {VK_FORMAT_R8G8_UNORM, 2, mapping};
if (number == snorm) return {VK_FORMAT_R8G8_SNORM, 2, mapping};
return fail();
}
case 4:
if (number == floating) return single(VK_FORMAT_R32_SFLOAT, 4);
if (number == uint) return single(VK_FORMAT_R32_UINT, 4);
@ -475,7 +481,7 @@ void intersect(VkRect2D& result, const Registers& registers, std::uint32_t offse
}
ShaderStages DecodeShaderStages(const QueueState& queue) {
ShaderStages DecodeShaderStages(const QueueState& queue, bool vertexWorkgroup) {
const auto value = read(queue.context, 0x2d5);
const auto validate = [&](bool condition, const char* reason) {
if (condition) return;
@ -487,7 +493,8 @@ ShaderStages DecodeShaderStages(const QueueState& queue) {
validate((value & 3u) != 3u && ((value >> 3u) & 3u) != 3u && ((value >> 6u) & 3u) != 3u, "reserved LS_EN, ES_EN or VS_EN encoding");
const auto primitive = read(queue.userConfig, 0x242, RegisterBank::UserConfig);
const bool tessellation = primitive == 9;
const bool geometry = (value & 0x20u) != 0;
validate(!vertexWorkgroup || ((value & 0x02002020u) == 0x2000u && !tessellation), "a vertex program that needs a workgroup requires PRIMGEN_EN without GS_EN, PRIMGEN_PASSTHRU_EN or tessellation");
const bool geometry = (value & 0x20u) != 0 || vertexWorkgroup;
validate(tessellation == ((value & 4u) != 0), "Patch topology and HS_EN disagree");
validate(!tessellation || !geometry, "combined tessellation and geometry is unsupported by the reference path");
const auto path = tessellation ? ShaderPath::Tessellation : geometry ? ShaderPath::Geometry : ShaderPath::Vertex;
@ -511,15 +518,16 @@ ShaderStages DecodeShaderStages(const QueueState& queue) {
const auto vertices = (group >> 9u) & 0x1ffu;
const auto primitives = group & 0x1ffu;
const auto maxVertices = read(queue.context, 0x1ff);
const auto verticesPerPrimitive = read(queue.context, 0x2ce);
const auto verticesPerPrimitive = vertexWorkgroup ? 3u : read(queue.context, 0x2ce);
validate((primitive == 1 || primitive == 2 || primitive == 4 || primitive == 5 || primitive == 6) && read(queue.context, 0x29b) == 2 && verticesPerPrimitive >= 3, "unsupported geometry input or output assembly");
const auto inputSize = primitive == 1 ? 1u : primitive == 2 ? 2u : 3u;
validate(vertices >= inputSize && maxVertices != 0 && maxVertices <= 256 && verticesPerPrimitive <= 256, "invalid geometry subgroup output");
validate(vertices >= inputSize && maxVertices != 0 && (!vertexWorkgroup || maxVertices >= inputSize) && maxVertices <= 256 && verticesPerPrimitive <= 256, "invalid geometry subgroup output");
const auto inputStep = primitive == 5 || primitive == 6 ? 1u : inputSize;
const auto groupPrimitives = std::min({primitives, (vertices - inputSize) / inputStep + 1u, maxVertices / verticesPerPrimitive});
const auto outputLimit = vertexWorkgroup ? (maxVertices - inputSize) / inputStep + 1u : maxVertices / verticesPerPrimitive;
const auto groupPrimitives = std::min({primitives, (vertices - inputSize) / inputStep + 1u, outputLimit});
validate(groupPrimitives != 0, "geometry subgroup contains no primitives");
const auto resources = read(queue.shader, 0x8b, RegisterBank::Shader);
validate(((read(queue.shader, 0x8a, RegisterBank::Shader) >> 29u) & 3u) == 3 && ((resources >> 16u) & 3u) == 3, "unsupported geometry VGPR allocation");
validate(vertexWorkgroup || (((read(queue.shader, 0x8a, RegisterBank::Shader) >> 29u) & 3u) == 3 && ((resources >> 16u) & 3u) == 3), "unsupported geometry VGPR allocation");
const auto esgsItemSize = read(queue.context, 0x2ab);
validate(esgsItemSize != 0 && esgsItemSize * vertices <= 0xffffu, "invalid VGT_ESGS_RING_ITEMSIZE");
const auto threads = std::max({(groupPrimitives - 1u) * inputStep + inputSize, primitives, maxVertices, primitives * (verticesPerPrimitive - 2u)});
@ -537,12 +545,12 @@ std::string DepthMaintenanceRejection(const QueueState& queue) {
return zeroMessage(0x000, control->second, "DB_RENDER_CONTROL depth copy, resummarize or decompress");
}
State DecodeState(const QueueState& queue) {
State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
if (auto reason = DepthMaintenanceRejection(queue); !reason.empty()) throw std::runtime_error(reason);
const auto& cx = queue.context;
State result{};
result.conservativeRasterization = decodeConservativeRasterization(queue);
result.stages = DecodeShaderStages(queue);
result.stages = DecodeShaderStages(queue, vertexWorkgroup);
const auto primitive = read(queue.userConfig, 0x242, RegisterBank::UserConfig);
APS5_LOG_OUT_DEBUG("DecodeState primitive=%u path=%u vertexWave=%u", primitive, static_cast<unsigned>(result.stages.path), result.stages.vertexWaveSize);
switch (primitive) {
@ -595,7 +603,8 @@ State DecodeState(const QueueState& queue) {
Require((depthControl & 0xc0000000u) == 0, "depth-conditional color writes are unsupported");
}
zero(cx, 0x203, shaderControlMask(read(cx, 0x1c4)), "depth export, shader coverage or ordered fragment execution");
zero(cx, 0x2dc, AlphaToCoverageMask, "alpha-to-coverage");
result.alphaToCoverage = read(cx, 0x2dc) == AlphaToCoverageUndithered;
if (!result.alphaToCoverage) zero(cx, 0x2dc, AlphaToCoverageMask, "dithered or rounded alpha-to-coverage");
zero(cx, 0x2f8, ~0u, "multisampling or coverage conversion");
zero(cx, 0x292, ScanModeMask, "scan conversion mode");
zero(cx, 0x293, ScanControlMask, "sample iteration, primitive discard or out-of-order rasterization");
@ -615,7 +624,7 @@ State DecodeState(const QueueState& queue) {
// explicitly; KEEP_TOGETHER_ENABLE (bit 24) only affects primitive distribution across the chip.
const bool pointsOrLines = result.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || result.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST;
const auto lineOffset = (raster & 0x1800u) != 0 && !pointsOrLines ? 0x2000u : 0u;
const auto rasterMode = raster & ~0x7u & ~(1u << 24u) & ~0x1800u & ~lineOffset & ~(1u << 19u);
const auto rasterMode = raster & ~0x7u & ~(1u << 24u) & ~0x1800u & ~lineOffset & ~(1u << 19u) & ~(1u << 16u);
if (rasterMode != 0 && rasterMode != 0x240u && rasterMode != 0x248u) throw std::runtime_error("AGC graphics: " + zeroMessage(0x205, raster, "polygon mode, depth bias or nonstandard rasterization"));
result.cullMode = ((raster & 1u) != 0 ? VK_CULL_MODE_FRONT_BIT : 0u) | ((raster & 2u) != 0 ? VK_CULL_MODE_BACK_BIT : 0u);
if (result.rectList) result.cullMode = VK_CULL_MODE_NONE;
@ -746,6 +755,10 @@ State DecodeState(const QueueState& queue) {
state.dstAlphaBlendFactor = alphaFactorWithOpaqueDestination(state.dstAlphaBlendFactor);
}
for (std::uint32_t i = 0; i < 4; ++i) result.blendConstants[i] = readFloat(cx, 0x105 + i);
if (mapping == 0xecu) {
const auto destinationAlpha = [](VkBlendFactor factor) { return factor == VK_BLEND_FACTOR_DST_ALPHA || factor == VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA || factor == VK_BLEND_FACTOR_SRC_ALPHA_SATURATE; };
if (destinationAlpha(state.srcColorBlendFactor) || destinationAlpha(state.dstColorBlendFactor) || destinationAlpha(state.srcAlphaBlendFactor) || destinationAlpha(state.dstAlphaBlendFactor)) throw std::runtime_error("AGC graphics: blending into an 8_8 color target with the alternate component order with a factor of the destination alpha, which the target keeps in its second channel, is not implemented");
}
}
if (color.packing != ShaderRecompiler::ColorExportPacking::None) {
const auto slotExport = (exportFormat >> (4u * color.exportIndex)) & 0xfu;
@ -767,6 +780,13 @@ State DecodeState(const QueueState& queue) {
Require(((exportFormat >> 4u) & 0xfu) == (exportFormat & 0xfu), "dual-source blending needs the MRT1 export in the MRT0 export format");
Require(((shaderMask >> 4u) & 0xfu) == (shaderMask & 0xfu), "dual-source blending needs the MRT1 export with the MRT0 shader mask");
}
if (result.alphaToCoverage) {
const auto mrt0 = std::find_if(result.colors.begin(), result.colors.end(), [](const ColorTarget& color) { return color.exportIndex == 0; });
Require(mrt0 != result.colors.end(), "alpha-to-coverage without a written MRT0 color target is unsupported");
const auto format = exportFormat & 0xfu;
Require(format == 4u || format == 5u || format == 6u || format == 9u, "alpha-to-coverage needs an MRT0 export format with a floating-point alpha");
Require(mrt0->componentMapping == 0xe4u && mrt0->packing == ShaderRecompiler::ColorExportPacking::None, "alpha-to-coverage into a component-swapped or packed MRT0 target is unsupported");
}
APS5_LOG_OUT_DEBUG("DecodeState done colorTarget=%u render=%ux%u topology=%u", result.hasColorTarget ? 1u : 0u, result.renderExtent.width, result.renderExtent.height, static_cast<unsigned>(result.topology));
return result;
}
@ -980,7 +1000,9 @@ std::string DrawRejection(const QueueState& queue, bool indexed) {
std::uint32_t zFormat = 0;
static_cast<void>(value(cx, 0x1c4, zFormat));
if (auto reason = nonzero(cx, 0x203, shaderControlMask(zFormat), "depth export, shader coverage or ordered fragment execution"); !reason.empty()) return reason;
if (auto reason = nonzero(cx, 0x2dc, AlphaToCoverageMask, "alpha-to-coverage"); !reason.empty()) return reason;
if (value(cx, 0x2dc, word) && word != AlphaToCoverageUndithered) {
if (auto reason = nonzero(cx, 0x2dc, AlphaToCoverageMask, "dithered or rounded alpha-to-coverage"); !reason.empty()) return reason;
}
if (auto reason = nonzero(cx, 0x2f8, ~0u, "multisampling or coverage conversion"); !reason.empty()) return reason;
if (auto reason = nonzero(cx, 0x292, ScanModeMask, "scan conversion mode"); !reason.empty()) return reason;
if (auto reason = nonzero(cx, 0x293, ScanControlMask, "sample iteration, primitive discard or out-of-order rasterization"); !reason.empty()) return reason;
@ -1033,6 +1055,15 @@ std::vector<RegisterRead>*& RegisterReadLog() {
return log;
}
bool SplitsAlternateAlpha(const State& state) {
for (std::size_t index = 0; index < state.colors.size() && index < state.blends.size(); ++index) {
const auto& blend = state.blends[index];
if (state.colors[index].componentMapping != 0xecu || !blend.blendEnable) continue;
if (blend.srcColorBlendFactor != blend.srcAlphaBlendFactor || blend.dstColorBlendFactor != blend.dstAlphaBlendFactor || blend.colorBlendOp != blend.alphaBlendOp) return true;
}
return false;
}
bool DrawKeyCovers(RegisterRead read) {
// One bit per register offset of each bank, set from the table once.
static const auto covered = [] {

182
core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp
View File

@ -87,6 +87,7 @@ std::atomic<std::uint64_t> pretestSkipped{0}, unregisteredDropped{0}, keyFlipKep
// Images stored by a FlushPending after a hook skip of theirs (AccessKeptByCpu), of which by the
// hook for the read site of the last skip; images evictStale dropped (the [hooksync] and [storage] lines).
std::atomic<std::uint64_t> flushedAfterSkip{0}, flushedAfterSkipSameSite{0}, staleEvicted{0};
std::atomic<std::uint64_t> depthCopyTextures{0}, depthCopyRefreshes{0};
std::atomic<std::uint64_t> refreshesProved{0};
struct StorageTraffic {
@ -457,6 +458,159 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex
}
}
VkFormat Texture::DepthCopyFormat(VkFormat colorFormat) {
switch (colorFormat) {
case VK_FORMAT_R32_SFLOAT: return VK_FORMAT_D32_SFLOAT;
case VK_FORMAT_R16_UNORM: return VK_FORMAT_D16_UNORM;
default: return VK_FORMAT_UNDEFINED;
}
}
bool Texture::CanDepthCopyFrom(const StorageTexture& source, const GuestTextureResource& descriptor) {
if (!CanCopyFrom(source, descriptor) || descriptor.dimension == TextureDimension::k3D) return false;
if (DepthCopyFormat(ResolveTextureFormat(descriptor.format)) == VK_FORMAT_UNDEFINED) return false;
if (ResolveTextureFormat(source.Descriptor().format) != ResolveTextureFormat(descriptor.format)) return false;
const auto geometry = DescribeSurface(descriptor);
return geometry.imageDepth == 1 && source.ImageDepth() == 1 && source.ImageLayers() >= geometry.imageLayers && source.Descriptor().mipCount >= descriptor.mipCount;
}
Texture::Texture(const Context& context, const std::shared_ptr<StorageTexture>& source, const GuestTextureResource& descriptor, VkComponentMapping components, DepthCopyTag) : context(context), storageSource(source), depthCopy(true) {
try {
Require(source != nullptr && CanDepthCopyFrom(*source, descriptor), "storage image cannot serve a depth copy of the sampled texture");
const auto colorFormat = ResolveTextureFormat(descriptor.format);
depthFormat = DepthCopyFormat(colorFormat);
const auto geometry = DescribeSurface(descriptor);
copyWidth = descriptor.width;
copyHeight = descriptor.height;
copyLevels = descriptor.mipCount;
copyLayers = geometry.imageLayers;
copyElementBytes = BytesPerElement(descriptor.format);
APS5_LOG_OUT("Texture address=0x%llx %ux%u mips=%u layers=%u depth copy of storage image (vk=%d -> %d)", static_cast<unsigned long long>(descriptor.baseAddress), descriptor.width, descriptor.height, descriptor.mipCount, copyLayers, static_cast<int>(colorFormat), static_cast<int>(depthFormat));
VkImageCreateInfo imageInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
imageInfo.flags = descriptor.dimension == TextureDimension::kCube ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0u;
imageInfo.imageType = ImageTypeFor(descriptor.dimension);
imageInfo.format = depthFormat;
imageInfo.extent = {descriptor.width, descriptor.height, 1u};
imageInfo.mipLevels = copyLevels;
imageInfo.arrayLayers = copyLayers;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
Check(context.Function<PFN_vkCreateImage>("vkCreateImage")(context.device, &imageInfo, nullptr, &image), "vkCreateImage depth copy");
owned = std::make_shared<OwnedImage>(context, image, VK_NULL_HANDLE);
VkMemoryRequirements requirements{};
context.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(context.device, image, &requirements);
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
allocationBytes = requirements.size;
allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &owned->memory), "vkAllocateMemory depth copy");
Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory depth copy");
const auto viewLevelCount = std::min(descriptor.lastLevel, descriptor.mipCount - 1u) - descriptor.baseLevel + 1u;
const auto viewLayerCount = geometry.imageLayers - descriptor.baseArray;
if (descriptor.dimension == TextureDimension::kCube) {
Require(viewLayerCount % 6u == 0, "guest cube texture view does not contain a multiple of 6 array slices");
}
VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
viewInfo.image = image;
viewInfo.viewType = ViewTypeFor(descriptor.dimension, viewLayerCount);
viewInfo.format = depthFormat;
viewInfo.components = VkComponentMapping{};
viewInfo.subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, descriptor.baseLevel, viewLevelCount, descriptor.baseArray, viewLayerCount};
VkImageViewMinLodCreateInfoEXT minLod{VK_STRUCTURE_TYPE_IMAGE_VIEW_MIN_LOD_CREATE_INFO_EXT};
ChainMinLod(context, descriptor, viewInfo, minLod);
Check(context.Function<PFN_vkCreateImageView>("vkCreateImageView")(context.device, &viewInfo, nullptr, &view), "vkCreateImageView depth copy");
createFirstLayerView(descriptor, viewInfo);
static_cast<void>(components);
depthCopyTextures.fetch_add(1, std::memory_order_relaxed);
} catch (...) {
release();
throw;
}
}
bool Texture::RefreshDepthCopy() {
if (!depthCopy || storageSource == nullptr) return false;
if (copiedOnce && storageSource->Version() == copiedVersion) return false;
std::uint64_t linearBytes = 0;
for (std::uint32_t level = 0; level < copyLevels; ++level) linearBytes += static_cast<std::uint64_t>(std::max(copyWidth >> level, 1u)) * std::max(copyHeight >> level, 1u) * copyElementBytes * copyLayers;
auto linear = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearBytes), VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
std::vector<VkBufferImageCopy> fromSource, toDepth;
std::uint64_t offset = 0;
for (std::uint32_t level = 0; level < copyLevels; ++level) {
const VkExtent3D extent{std::max(copyWidth >> level, 1u), std::max(copyHeight >> level, 1u), 1u};
VkBufferImageCopy region{};
region.bufferOffset = offset;
region.bufferRowLength = 0;
region.bufferImageHeight = 0;
region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, copyLayers};
region.imageOffset = {0, 0, 0};
region.imageExtent = extent;
fromSource.push_back(region);
region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT;
toDepth.push_back(region);
offset += static_cast<std::uint64_t>(extent.width) * extent.height * copyElementBytes * copyLayers;
}
auto* recorder = Recorder::Active();
std::unique_ptr<CommandBatch> batch;
VkCommandBuffer commands = VK_NULL_HANDLE;
if (recorder != nullptr) {
commands = recorder->Commands();
recorder->Keep(linear);
recorder->Keep(owned);
recorder->Keep(storageSource);
} else {
batch = std::make_unique<CommandBatch>(context);
commands = batch->Handle();
}
const auto barrier = context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier");
const VkMemoryBarrier produced{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT};
VkImageMemoryBarrier toTransferDst{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
toTransferDst.srcAccessMask = VK_ACCESS_SHADER_READ_BIT;
toTransferDst.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
toTransferDst.oldLayout = copiedOnce ? VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL : VK_IMAGE_LAYOUT_UNDEFINED;
toTransferDst.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
toTransferDst.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toTransferDst.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toTransferDst.image = image;
toTransferDst.subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, 0, copyLevels, 0, copyLayers};
barrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &produced, 0, nullptr, 1, &toTransferDst);
context.Function<PFN_vkCmdCopyImageToBuffer>("vkCmdCopyImageToBuffer")(commands, storageSource->Image(), VK_IMAGE_LAYOUT_GENERAL, linear->Handle(), static_cast<std::uint32_t>(fromSource.size()), fromSource.data());
VkBufferMemoryBarrier linearReady{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER};
linearReady.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
linearReady.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
linearReady.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
linearReady.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
linearReady.buffer = linear->Handle();
linearReady.offset = 0;
linearReady.size = VK_WHOLE_SIZE;
barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 1, &linearReady, 0, nullptr);
context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, linear->Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, static_cast<std::uint32_t>(toDepth.size()), toDepth.data());
VkImageMemoryBarrier toShaderRead{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
toShaderRead.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
toShaderRead.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
toShaderRead.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
toShaderRead.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
toShaderRead.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toShaderRead.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toShaderRead.image = image;
toShaderRead.subresourceRange = toTransferDst.subresourceRange;
barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &toShaderRead);
if (batch) batch->SubmitAndWait();
copiedOnce = true;
copiedVersion = storageSource->Version();
depthCopyRefreshes.fetch_add(1, std::memory_order_relaxed);
return true;
}
Texture::DepthCopyCounts Texture::TakeDepthCopyCounts() {
return {depthCopyTextures.exchange(0, std::memory_order_relaxed), depthCopyRefreshes.exchange(0, std::memory_order_relaxed)};
}
bool Texture::CanCopyFrom(const StorageTexture& source, const GuestTextureResource& descriptor) {
const auto& from = source.Descriptor();
if (IsBlockCompressed(descriptor.format) || IsBlockCompressed(from.format)) return false;
@ -1408,6 +1562,13 @@ bool StorageTexture::Refresh() {
stampedBlocks.assign(generations.size(), 0);
cpuBlocks.assign(generations.size(), 0);
tracked = GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast<std::size_t>(guestBytes), generations, stampedBlocks, cpuBlocks);
for (std::uint32_t unit = 0; tracked && blockUnits && unit < trackedLayers; ++unit) {
if (stampedBlocks[unit] != GuestMemory::BlockMaybeWritten) continue;
std::array<std::uint8_t, 1> block{};
if (!compareUntracked(layerBegin(unit), static_cast<std::size_t>(layerBytes(unit)), block) || block[0] != GuestMemory::BlockUnchanged) continue;
stampedBlocks[unit] = GuestMemory::BlockUnchanged;
cpuBlocks[unit] = 0;
}
compared = !tracked && compareUntracked(descriptor.baseAddress, static_cast<std::size_t>(guestBytes), stampedBlocks, true);
if (compared) cpuBlocks = stampedBlocks;
};
@ -1754,7 +1915,7 @@ void StorageTexture::upload(const std::vector<bool>* layers) {
const auto now = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast<std::size_t>(guestBytes));
const auto stampLayers = [&](bool selectedOnly) {
const bool whole = !selectedOnly || layers == nullptr;
if (guestBytes > GenerationBaselineBytes) generationBaseline.clear();
if (guestBytes > GenerationBaselineBytes || !guestBytesSettled()) generationBaseline.clear();
else if (whole) {
generationBaseline.resize(static_cast<std::size_t>(guestBytes));
GuestMemory::ReadCommitted(descriptor.baseAddress, generationBaseline);
@ -1776,7 +1937,9 @@ void StorageTexture::upload(const std::vector<bool>* layers) {
// The clear is recorded into the open batch like a direct upload (the image kept by it): a
// batch of its own submitted the recorder's work first and waited for all of it, 25-40 ms
// under the GPU mutex at the movie stage. APS5_NO_RECORDED_CLEAR=1 waits as before.
captureGuestBytes(nullptr);
static_cast<void>(HostImportFor(context, descriptor.baseAddress, static_cast<std::size_t>(guestBytes)));
if (GuestMemory::Watched(descriptor.baseAddress, static_cast<std::size_t>(guestBytes))) originalValid = false;
else captureGuestBytes(nullptr);
forgetBorrowed(0, trackedLayers);
stampLayers(false);
static const bool recordClear = std::getenv("APS5_NO_RECORDED_CLEAR") == nullptr;
@ -3305,10 +3468,6 @@ bool StorageTexture::FillClear(std::span<const std::uint32_t, 4> pattern, std::u
return false;
}
}
if (HostImportFor(context, descriptor.baseAddress, static_cast<std::size_t>(guestBytes)) == nullptr) {
refusal = "not imported";
return false;
}
auto* recorder = Recorder::Active();
if (recorder == nullptr) {
refusal = "no recorder";
@ -3623,7 +3782,9 @@ std::shared_ptr<StorageTexture> StorageTexture::pendingAlias() const {
const auto& theirs = candidate->descriptor;
// The same texels under another format of the same size: the device copy moves them bit
// for bit, as the guest bytes would.
const bool sameShape = mine.width == theirs.width && mine.height == theirs.height && mine.depthOrLastArray == theirs.depthOrLastArray && mine.baseArray == theirs.baseArray && mine.mipCount == theirs.mipCount && mine.tileMode == theirs.tileMode && mine.dimension == theirs.dimension && BytesPerElement(mine.format) == BytesPerElement(theirs.format) && geometry.imageLayers == candidate->geometry.imageLayers && geometry.imageDepth == candidate->geometry.imageDepth && trackedLayers == candidate->trackedLayers;
const auto oneSlice = [](const GuestTextureResource& resource) { return resource.dimension == TextureDimension::k2D || (resource.dimension == TextureDimension::k2DArray && resource.depthOrLastArray == 0); };
const bool sameDimension = mine.dimension == theirs.dimension || (oneSlice(mine) && oneSlice(theirs));
const bool sameShape = mine.width == theirs.width && mine.height == theirs.height && mine.depthOrLastArray == theirs.depthOrLastArray && mine.baseArray == theirs.baseArray && mine.mipCount == theirs.mipCount && mine.tileMode == theirs.tileMode && sameDimension && BytesPerElement(mine.format) == BytesPerElement(theirs.format) && geometry.imageLayers == candidate->geometry.imageLayers && geometry.imageDepth == candidate->geometry.imageDepth && trackedLayers == candidate->trackedLayers;
if (sameShape && candidate->Cached() && candidate->uploadedKeys == DccKeys::Uncompressed) return candidate;
}
return nullptr;
@ -3745,6 +3906,13 @@ void StorageTexture::writeBack(std::uint64_t address, std::size_t bytes) {
writeBackLayers(layers);
}
bool StorageTexture::guestBytesSettled() const {
const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> range{{{descriptor.baseAddress, descriptor.baseAddress + guestBytes}}};
if (AnyShadowedOverlaps(range) || AnyPendingOverlaps(range)) return false;
const auto* recorder = Recorder::Active();
return recorder == nullptr || !recorder->PendingWriteOverlaps(descriptor.baseAddress, static_cast<std::size_t>(guestBytes));
}
bool StorageTexture::unchangedSinceBaseline(std::uint64_t from, std::uint64_t to) const {
const auto& baseline = originalValid ? original : generationBaseline;
if (baseline.size() != guestBytes) return false;

16
core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp
View File

@ -67,7 +67,7 @@ constexpr FormatEntry kFormatLookup[] = {
{132, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, 4, false},
{133, VK_FORMAT_R5G6B5_UNORM_PACK16, 2, false},
{134, VK_FORMAT_A1R5G5B5_UNORM_PACK16, 2, false},
{136, VK_FORMAT_R4G4B4A4_UNORM_PACK16, 2, false},
{136, VK_FORMAT_B4G4R4A4_UNORM_PACK16, 2, false},
{169, VK_FORMAT_BC1_RGBA_UNORM_BLOCK, 8, true},
{170, VK_FORMAT_BC1_RGBA_SRGB_BLOCK, 8, true},
{171, VK_FORMAT_BC2_UNORM_BLOCK, 16, true},
@ -154,6 +154,20 @@ std::optional<std::uint32_t> FindGuestTextureFormat(VkFormat format, std::uint32
return std::nullopt;
}
VkComponentSwizzle TextureComponentChannel(std::uint32_t guestFormat, VkComponentSwizzle component) {
if (component < VK_COMPONENT_SWIZZLE_R || component > VK_COMPONENT_SWIZZLE_A) return component;
const auto index = static_cast<std::size_t>(component - VK_COMPONENT_SWIZZLE_R);
constexpr std::array<VkComponentSwizzle, 4> bgrb{VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_B};
constexpr std::array<VkComponentSwizzle, 4> bgra{VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_A};
constexpr std::array<VkComponentSwizzle, 4> argb{VK_COMPONENT_SWIZZLE_A, VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_B};
switch (guestFormat) {
case 133: return bgrb[index];
case 134: return bgra[index];
case 136: return argb[index];
default: return component;
}
}
std::uint32_t BytesPerElement(std::uint32_t guestFormat) {
return findFormatEntry(guestFormat).bytesPerElement;
}

69
core/libs/prx/libSceAgcDriver/tests/BdaExecution.cpp
View File

@ -90,7 +90,46 @@ private:
}
static void RunWrittenRangeDecodeTests() {
namespace Abi = ShaderRecompiler::BdaAbi;
std::vector<std::uint32_t> words(Abi::FaultBufferBytes / sizeof(std::uint32_t));
std::vector<std::pair<std::uint64_t, std::size_t>> marked;
const auto decode = [&] {
marked.clear();
return Abi::ForEachWrittenRange(words.data(), [&](std::uint64_t address, std::size_t bytes) { marked.emplace_back(address, bytes); });
};
const auto claim = [&](std::uint32_t index, std::uint32_t slot, std::uint64_t address, std::uint32_t mask) {
words[Abi::WrittenListWord + index] = slot;
words[Abi::WrittenSlotsWord + 2 * slot] = Abi::WrittenChunkKey(address);
words[Abi::WrittenSlotsWord + 2 * slot + 1] = mask;
words[Abi::WrittenCountWord] = std::max(words[Abi::WrittenCountWord], index + 1);
};
constexpr std::uint64_t page = std::uint64_t{1} << Abi::WrittenPageShift;
Require(decode() && marked.empty(), "an empty written-page record marked pages");
constexpr std::uint64_t top = 0x7ffffffe0000ULL;
claim(0, 5, top, 0b1011u);
Require(decode() && marked == std::vector<std::pair<std::uint64_t, std::size_t>>{{top, 2 * page}, {top + 3 * page, page}}, "written pages near the top of the address space were not decoded as written");
std::fill(words.begin(), words.end(), 0u);
constexpr std::uint64_t high = 0x200000001000ULL;
claim(0, Abi::WrittenPageSlots - 1, high, 1u << ((high >> Abi::WrittenPageShift) & (Abi::WrittenChunkPages - 1u)));
Require(decode() && marked == std::vector<std::pair<std::uint64_t, std::size_t>>{{high, page}}, "a written page above 16 TiB was decoded at another address");
std::fill(words.begin(), words.end(), 0u);
constexpr std::uint64_t chunk = 0x3e0000000ULL;
claim(0, 7, chunk, 0xffffffffu);
claim(1, 8, chunk + (std::uint64_t{1} << Abi::WrittenChunkShift), 1u);
claim(2, 9, chunk - (std::uint64_t{1} << Abi::WrittenChunkShift), 0x80000000u);
Require(decode() && marked == std::vector<std::pair<std::uint64_t, std::size_t>>{{chunk, 32 * page}, {chunk + 32 * page, page}, {chunk - page, page}}, "whole, first-page and last-page chunks were not decoded exactly");
words[Abi::WrittenListWord + 1] = Abi::WrittenPageSlots;
Require(!decode(), "a claim outside the slots was accepted");
std::fill(words.begin(), words.end(), 0u);
words[Abi::WrittenCountWord] = 1;
Require(!decode(), "a claim of an empty slot was accepted");
words[Abi::WrittenCountWord] = Abi::WrittenPageSlots + 1;
Require(!decode(), "more claims than slots were accepted");
}
static void RunBdaDwordWriteTests(const Context& context) {
RunWrittenRangeDecodeTests();
namespace Abi = ShaderRecompiler::BdaAbi;
constexpr std::uint64_t guest = 0x7fff12340f80ULL;
constexpr std::size_t firstBytes = 160;
@ -114,20 +153,23 @@ static void RunBdaDwordWriteTests(const Context& context) {
}
return nullptr;
};
const auto noted = [&](std::uint64_t address) {
const auto page = static_cast<std::uint32_t>(address >> Abi::WrittenPageShift) + 1u;
std::vector<std::uint32_t> slots(Abi::WrittenPageSlots);
std::memcpy(slots.data(), fault.Bytes().data() + Abi::WrittenSlotsWord * sizeof(std::uint32_t), slots.size() * sizeof(std::uint32_t));
return std::find(slots.begin(), slots.end(), page) != slots.end();
const auto notedBytes = [&](std::uint64_t first, std::uint64_t last) {
std::uint64_t covered = 0, total = 0;
const auto* faultWords = reinterpret_cast<const std::uint32_t*>(fault.Bytes().data());
Require(Abi::ForEachWrittenRange(faultWords, [&](std::uint64_t begin, std::size_t bytes) {
total += bytes;
for (const auto address : {first, last}) covered += address >= begin && address < begin + bytes ? 1u : 0u;
}), "a BDA dword write left an invalid written-page record");
return std::pair{covered, total};
};
const auto write = [&](std::uint64_t address, std::uint32_t dwords, std::uint32_t storedBytes, Abi::FaultReason reason) {
const auto write = [&](std::uint64_t address, std::uint32_t dwords, std::uint32_t storedBytes, Abi::FaultReason reason, std::uint32_t groups = 1) {
const Abi::Header header{Abi::Version, static_cast<std::uint32_t>(ranges.size()), sizeof(Abi::Range), 0};
std::memcpy(table.Bytes().data(), &header, sizeof(header));
std::memcpy(table.Bytes().data() + sizeof(header), ranges.data(), sizeof(ranges));
std::memset(fault.Bytes().data(), 0, fault.Bytes().size());
for (auto* buffer : {&first, &second, &readOnly}) std::memset(buffer->Bytes().data(), 0xa5, buffer->Bytes().size());
Pipeline pipeline(context, MakeBdaDwordWriteTestShader(address, dwords, values.data()), {&table, &fault, &output});
pipeline.Run(1);
pipeline.Run(groups);
Abi::Fault report{};
std::memcpy(&report, fault.Bytes().data(), sizeof(report));
for (std::uint32_t byte = 0; byte < dwords * 4u; ++byte) {
@ -142,7 +184,17 @@ static void RunBdaDwordWriteTests(const Context& context) {
} else {
Require(report.state == Abi::FaultState::Ready && report.reason == reason && report.instruction == 0x1234, "a BDA dword write did not publish its fault");
}
if (storedBytes != 0) Require(noted(address) && noted(address + storedBytes - 1u), "a BDA dword write did not note its written pages");
const auto [covered, total] = notedBytes(address, address + (storedBytes != 0 ? storedBytes - 1u : 0u));
if (storedBytes == 0) {
Require(total == 0, "a BDA dword write that stored nothing noted pages");
} else {
const auto pages = ((address + storedBytes - 1u) >> Abi::WrittenPageShift) - (address >> Abi::WrittenPageShift) + 1u;
Require(covered == 2 && total == pages << Abi::WrittenPageShift, "a BDA dword write did not note exactly its written pages");
const auto chunks = ((address + storedBytes - 1u) >> Abi::WrittenChunkShift) - (address >> Abi::WrittenChunkShift) + 1u;
std::uint32_t claims = 0;
std::memcpy(&claims, fault.Bytes().data() + Abi::WrittenCountWord * sizeof(std::uint32_t), sizeof(claims));
Require(claims == chunks, "a BDA dword write claimed a written-page slot more than once per chunk");
}
};
write(guest + 16, 4, 16, static_cast<Abi::FaultReason>(0));
write(guest + 20, 4, 16, static_cast<Abi::FaultReason>(0));
@ -154,6 +206,7 @@ static void RunBdaDwordWriteTests(const Context& context) {
write(guest + 41, 4, 16, static_cast<Abi::FaultReason>(0));
write(guest + firstBytes - 8, 4, 16, static_cast<Abi::FaultReason>(0));
write(guest + firstBytes + 12, 4, 4, Abi::FaultReason::Permission);
write(guest + 0x7c, 2, 8, static_cast<Abi::FaultReason>(0), 64);
}
void RunBdaExecutionTests(const Context& context) {

326
core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp
View File

@ -71,15 +71,16 @@ void importCrossingTests(const Context& context, const BdaTestAccess& access) {
const auto crossing = leased.Descriptor(second - 16, 32, adjustment);
const auto crossingBytes = access.bytes(crossing.buffer);
Require(crossing.offset + crossing.range <= crossingBytes.size(), "a view crossing into the next registered range is bound past its host import");
Require(crossingBytes[crossing.offset + adjustment + 15] == std::byte{0x11} && crossingBytes[crossing.offset + adjustment + 16] == std::byte{0x22}, "a view crossing into the next registered range misses its bytes");
Require(secondImport.base == first && secondImport.bytes == 2 * half, "adjacent registered ranges with the same access were not imported as one span");
Require(crossing.buffer == secondImport.buffer && crossing.offset + adjustment == second - 16 - secondImport.base, "a view crossing into the next registered range is not bound in place in the spanning import");
const auto inside = leased.Descriptor(second + 64, 16, adjustment);
const auto insideBytes = access.bytes(inside.buffer);
Require(inside.offset + inside.range <= insideBytes.size() && (inside.buffer == secondImport.buffer ? inside.offset + adjustment == 64 : insideBytes[inside.offset + adjustment] == std::byte{0x22}), "a view in the second registered range is bound past its buffer or misses its bytes");
Require(inside.offset + inside.range <= insideBytes.size() && (inside.buffer == secondImport.buffer ? inside.offset + adjustment == second + 64 - secondImport.base : insideBytes[inside.offset + adjustment] == std::byte{0x22}), "a view in the second registered range is bound past its buffer or misses its bytes");
for (const auto& range : leased.AddressRanges()) {
if (range.end <= first || range.begin >= second + half) continue;
const auto mapped = access.addressBytes(range.deviceAddress);
Require(mapped.size() >= range.end - range.begin, "a BDA range runs past the end of its host import");
if (range.begin <= second && second < range.end && range.deviceAddress != secondImport.address) Require(mapped[second - range.begin] == std::byte{0x22}, "the BDA range over the second registered range misses its bytes");
if (range.begin <= second && second < range.end && (range.deviceAddress < secondImport.address || range.deviceAddress >= secondImport.address + secondImport.bytes)) Require(mapped[second - range.begin] == std::byte{0x22}, "the BDA range over the second registered range misses its bytes");
}
leased.WriteBack();
}
@ -88,6 +89,36 @@ void importCrossingTests(const Context& context, const BdaTestAccess& access) {
::operator delete(block, std::align_val_t{half});
}
void importSpanTests(const Context& context) {
constexpr std::size_t piece = 65536;
void* block = ::operator new(3 * piece, std::align_val_t{piece});
auto* guest = static_cast<std::uint8_t*>(block);
const auto first = reinterpret_cast<std::uintptr_t>(block);
auto importing = context;
importing.hostImportAlignment = piece;
const auto change = [&](std::uint8_t* range, bool add, bool writable) {
auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, range, piece, true, writable, true);
else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, range);
GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
};
change(guest, true, false);
change(guest + piece, true, false);
change(guest + 2 * piece, true, true);
const auto* span = HostImportFor(importing, first + piece - 16, 32);
Require(span != nullptr && span->base == first && span->bytes == 2 * piece, "adjacent read-only ranges were not imported as one span");
const auto* writable = HostImportFor(importing, first + 2 * piece, piece);
Require(writable != nullptr && writable->base == first + 2 * piece && writable->bytes == piece, "a writable range joined the read-only span");
Require(HostImportFor(importing, first + 2 * piece - 16, 32) == nullptr, "a view crossing ranges of different access was imported");
change(guest + piece, false, false);
const auto* alone = HostImportFor(importing, first, piece);
Require(alone != nullptr && alone->base == first && alone->bytes == piece, "the span outlived the removal of one of its ranges");
Require(HostImportFor(importing, first + piece, 16) == nullptr, "the removed range is still imported");
change(guest, false, false);
change(guest + 2 * piece, false, false);
::operator delete(block, std::align_val_t{piece});
}
ShaderRecompiler::DescriptorBinding binding(Role role, std::uint32_t slot) {
return {ShaderRecompiler::DescriptorKind::StorageBuffer, role, 0, slot, 1, {}, false};
}
@ -569,6 +600,242 @@ void gpuMappingTests(const Context& context) {
::operator delete(gpu, std::align_val_t{bytes});
}
void stridedOverhangTests(const Context& context, const BdaTestAccess& access) {
constexpr std::size_t allocation = 0x8000;
constexpr std::uint32_t stride = 20;
constexpr std::uint32_t records = (allocation + stride - 1) / stride;
auto* block = static_cast<std::byte*>(::operator new(allocation * 2, std::align_val_t{allocation}));
std::memset(block, 0x5a, allocation * 2);
{
GuestAllocations::Mutation mutation;
mutation.Add(block, allocation, true, false, true);
}
const auto address = reinterpret_cast<std::uintptr_t>(block);
ShaderRecompiler::RecompileResult shader;
auto buffer = binding(Role::GuestBuffers, 6);
buffer.guestDescriptor = {static_cast<std::uint32_t>(address), (static_cast<std::uint32_t>(address >> 32) & 0xffffu) | (stride << 16u), records, 0x31000000u};
buffer.bufferWritten = {false};
shader.bindings = {buffer};
CompiledShader compiled{ShaderRecompiler::ShaderStage::Compute, &shader, 0};
{
ShaderResources resources(context, compiled);
Require(access.descriptor(6).range == allocation, "a strided buffer's rounded-up last record was bound past its allocation");
resources.WriteBack();
}
{
GuestAllocations::Mutation mutation;
mutation.Add(block + allocation, allocation, true, false, true);
}
{
ShaderResources resources(context, compiled);
Require(access.descriptor(6).range == allocation, "a strided buffer's rounded-up last record was bound into the next allocation");
resources.WriteBack();
}
{
GuestAllocations::Mutation mutation;
mutation.Remove(block + allocation);
}
buffer.guestDescriptor[2] = records - 1;
shader.bindings = {buffer};
{
ShaderResources resources(context, compiled);
Require(access.descriptor(6).range == static_cast<VkDeviceSize>(records - 1) * stride, "a strided buffer inside its allocation was shortened");
resources.WriteBack();
}
{
GuestAllocations::Mutation mutation;
mutation.Remove(block);
}
::operator delete(block, std::align_val_t{allocation});
}
std::byte* reserveSparse(std::size_t reserved, std::size_t committed) {
#ifdef _WIN32
auto* block = static_cast<std::byte*>(VirtualAlloc(nullptr, reserved, MEM_RESERVE, PAGE_NOACCESS));
Require(block != nullptr && VirtualAlloc(block, committed, MEM_COMMIT, PAGE_READWRITE) != nullptr, "cannot reserve the sparse heap");
#else
void* mapped = mmap(nullptr, reserved, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
Require(mapped != MAP_FAILED && mprotect(mapped, committed, PROT_READ | PROT_WRITE) == 0, "cannot reserve the sparse heap");
auto* block = static_cast<std::byte*>(mapped);
#endif
return block;
}
void releaseSparse(std::byte* block, std::size_t reserved) {
#ifdef _WIN32
static_cast<void>(reserved);
VirtualFree(block, 0, MEM_RELEASE);
#else
munmap(block, reserved);
#endif
}
void sparseHeapTests(const Context& context, const BdaTestAccess& access) {
constexpr std::size_t reserved = 64u << 20u;
constexpr std::size_t committed = 65536;
auto* block = reserveSparse(reserved, committed);
std::memset(block, 0x5a, committed);
const auto base = reinterpret_cast<std::uintptr_t>(block);
access.limitMemory(4u << 20u);
{
GuestBufferMemory memory(context);
memory.AddWritable(base, reserved);
memory.Upload(false);
std::uint32_t adjustment = 0;
const auto view = memory.Descriptor(base, reserved, adjustment);
Require(adjustment == 0 && view.range == committed, "a sparse heap view is not cut at its last committed byte");
Require(memory.CopiedBytes() == committed, "a cut sparse heap is charged at its reserved size");
const auto bytes = access.bytes(view.buffer);
Require(bytes.size() < reserved && bytes[view.offset + committed - 1] == std::byte{0x5a}, "a sparse heap was not copied up to its last committed byte");
bytes[view.offset + 8] = std::byte{0x77};
memory.WriteBack();
Require(block[8] == std::byte{0x77}, "a store to a cut sparse heap was not written back");
}
{
constexpr std::size_t late = 1u << 20u;
GuestBufferMemory memory(context);
memory.AddWritable(base, reserved);
memory.AddReadable(base + late, 16);
memory.Upload(true);
std::uint32_t adjustment = 0;
const auto view = memory.Descriptor(base + late, 16, adjustment);
Require(view.range == 16 + adjustment, "a view in the uncommitted part of a sparse heap is not bound");
const auto bytes = access.bytes(view.buffer);
const auto whole = memory.Descriptor(base, reserved, adjustment);
Require(whole.buffer == view.buffer && bytes.size() >= whole.offset + late + 16, "a view in the uncommitted part of a sparse heap has another owner");
Require(bytes[whole.offset + committed - 1] == std::byte{0x5a} && std::all_of(bytes.begin() + static_cast<std::ptrdiff_t>(whole.offset + committed), bytes.begin() + static_cast<std::ptrdiff_t>(whole.offset + late + 16), [](std::byte byte) { return byte == std::byte{}; }), "the uncommitted part of a cut sparse heap does not read as zeros");
Require(whole.range == late + 256, "a sparse heap copy does not end past its last view's start");
Require(memory.CopiedBytes() == late + 256, "a cut sparse heap charge does not include its last view");
const auto ranges = memory.AddressRanges();
Require(ranges.size() == 1 && ranges[0].begin == base && ranges[0].end == base + late + 256, "the BDA range of a cut sparse heap is not clamped to its copy");
memory.WriteBack();
}
access.limitMemory(std::nullopt);
releaseSparse(block, reserved);
}
void sparseHeapMirrorTests(const Context& context, const BdaTestAccess& access) {
#ifdef _WIN32
if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) {
std::cout << "guest arena unavailable or not write-watched: sparse heap mirrors not tested\n";
return;
}
constexpr std::size_t reserved = 64u << 20u;
constexpr std::size_t committed = 2 * 65536;
constexpr std::size_t extent = 1u << 20u;
constexpr std::size_t late = 512u << 10u;
constexpr std::size_t grown = 2u << 20u;
constexpr std::size_t past = 3584u << 10u;
void* block = GuestArena::GuestArenaAllocate_nid_postfix(reserved, 65536);
auto* guest = static_cast<std::uint8_t*>(block);
GuestArena::GuestArenaCommit_nid_postfix(block, committed, PAGE_READWRITE, 65536);
std::memset(block, 0x5a, committed);
const auto base = reinterpret_cast<std::uintptr_t>(block);
const auto registry = [&](bool add) {
auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, reserved, true, true, true);
else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block);
GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
};
const auto bind = [&](bool written, const std::function<void(std::span<std::byte>, const VkDescriptorBufferInfo&)>& gpu) {
GuestBufferMemory memory(context);
if (written) memory.AddWritable(base, reserved);
else memory.AddReadable(base, reserved);
memory.Upload(false);
std::uint32_t adjustment = 0;
const auto view = memory.Descriptor(base, reserved, adjustment);
Require(adjustment == 0, "a sparse heap view at its base is adjusted");
gpu(access.bytes(view.buffer), view);
memory.WriteBack();
return view;
};
const auto address = [&] {
GuestBufferMemory memory(context);
memory.AcquireRegistered();
memory.Upload(true);
const auto ranges = memory.AddressRanges();
const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == base; });
const auto result = found != ranges.end() ? std::pair{found->end, found->deviceAddress} : std::pair<std::uint64_t, std::uint64_t>{0, 0};
memory.WriteBack();
return result;
};
const auto sweep = [&] {
GuestBufferMemory memory(context);
memory.AcquireRegistered();
memory.Upload(true);
memory.WriteBack();
};
sweep();
registry(true);
const auto before = MirrorCounters();
const auto first = bind(true, [&](std::span<std::byte> bytes, const VkDescriptorBufferInfo& view) {
Require(view.range == extent && bytes[view.offset + committed - 1] == std::byte{0x5a} && bytes[view.offset + committed] == std::byte{}, "a sparse heap is not bound to a mirror of its committed extent");
bytes[view.offset + 8] = std::byte{0x77};
guest[24] = 0x55;
});
const auto made = MirrorCounters();
Require(made.heapMirrors == before.heapMirrors + 1 && made.heapBytes == before.heapBytes + extent && made.rebuilds == before.rebuilds + 1 && made.sparseBinds == before.sparseBinds + 1, "a sparse heap was not mirrored once");
Require(guest[8] == 0x77 && guest[24] == 0x55, "a sparse heap mirror's write-back lost the GPU's store or rolled back the CPU's");
const auto second = bind(false, [&](std::span<std::byte> bytes, const VkDescriptorBufferInfo& view) {
Require(bytes[view.offset + 8] == std::byte{0x77} && bytes[view.offset + 24] == std::byte{0x55}, "the sparse heap mirror missed the stores");
});
const auto again = MirrorCounters();
Require(second.buffer == first.buffer && again.rebuilds == made.rebuilds && again.blocksCopied == made.blocksCopied + 1, "a sparse heap bound again was read again beyond its written block");
bind(false, [](std::span<std::byte>, const VkDescriptorBufferInfo&) {});
Require(MirrorCounters().blocksCopied == again.blocksCopied && MirrorCounters().rebuilds == again.rebuilds, "an unchanged sparse heap was read again");
GuestArena::GuestArenaCommit_nid_postfix(guest + late, 65536, PAGE_READWRITE, 65536);
guest[late + 4] = 0x66;
bind(false, [&](std::span<std::byte> bytes, const VkDescriptorBufferInfo& view) {
Require(view.buffer == first.buffer && bytes[view.offset + late + 4] == std::byte{0x66} && bytes[view.offset + late + 65536] == std::byte{}, "a page committed after the mirror was made was not read into it");
});
const auto [end, device] = address();
Require(end == base + extent && device != 0 && access.addressBytes(device)[late + 4] == std::byte{0x66} && MirrorCounters().rebuilds == again.rebuilds, "an address-based build does not map the sparse heap mirror up to its extent");
GuestArena::GuestArenaCommit_nid_postfix(guest + grown, 65536, PAGE_READWRITE, 65536);
guest[grown + 2] = 0x44;
const auto [grownEnd, grownDevice] = address();
Require(grownEnd == base + 3 * extent && grownDevice != device && access.addressBytes(grownDevice)[grown + 2] == std::byte{0x44} && access.addressBytes(grownDevice)[8] == std::byte{0x77}, "a sparse heap committed past its mirror was not mirrored again");
{
GuestBufferMemory memory(context);
memory.AddReadable(base, reserved);
memory.AddReadable(base + past, 16);
memory.Upload(false);
std::uint32_t adjustment = 0;
const auto view = memory.Descriptor(base + past, 16, adjustment);
const auto whole = memory.Descriptor(base, reserved, adjustment);
Require(view.buffer == whole.buffer && access.bytes(whole.buffer)[whole.offset + grown + 2] == std::byte{0x44}, "a view past the sparse heap mirror is not copied with the heap");
memory.WriteBack();
}
registry(false);
sweep();
Require(MirrorCounters().heapMirrors == before.heapMirrors && MirrorCounters().heapBytes == before.heapBytes, "the sparse heap mirror outlived its range");
{
constexpr std::size_t whole = 1u << 20u;
GuestArena::GuestArenaReset_nid_postfix(block, reserved);
GuestArena::GuestArenaCommit_nid_postfix(guest, whole, PAGE_READWRITE, 65536);
std::memset(guest, 0x31, whole);
auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, whole, true, true, true);
GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
const auto start = MirrorCounters();
const auto view = bind(true, [&](std::span<std::byte> bytes, const VkDescriptorBufferInfo& info) {
Require(info.range == whole, "a heap bound whole over a smaller registered range is not clamped to that range");
Require(bytes[info.offset + whole - 1] == std::byte{0x31}, "a heap bound whole over a smaller registered range misses its last committed byte");
bytes[info.offset + 16] = std::byte{0x12};
});
Require(view.range == whole && MirrorCounters().sparseBinds == start.sparseBinds + 1 && guest[16] == 0x12, "a heap bound whole over a smaller registered range was copied");
mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix();
GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block);
GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation);
sweep();
}
GuestArena::GuestArenaReset_nid_postfix(block, reserved);
GuestArena::GuestArenaRelease_nid_postfix(block, reserved);
#else
static_cast<void>(context);
static_cast<void>(access);
#endif
}
void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
alignas(64) std::array<std::uint32_t, 16> guest{};
guest[0] = 123;
@ -576,7 +843,9 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
GuestBufferMemory memory(context);
memory.AddWritable(address, sizeof(guest));
memory.AddWritable(address + 16, 16);
Require(memory.CopiedBytes() == 0, "a region counted as copied before its upload");
memory.Upload(true);
Require(memory.CopiedBytes() == sizeof(guest), "copied bytes " + std::to_string(memory.CopiedBytes()) + " do not match the one region copied");
std::uint32_t adjustment = 0;
const auto first = memory.Descriptor(address, sizeof(guest), adjustment);
Require(adjustment == 0, "a view at its owner's start is bound off it");
@ -615,7 +884,9 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
const std::array<CompiledShader, 1> stages{{{ShaderRecompiler::ShaderStage::TessellationControl, &control, 0}}};
const std::array<GuestMemorySnapshot, 1> unusedSnapshots{{{0, source}}};
ShaderResources resources(disabled, stages, ColorTarget{}, 0, 0, 0, unusedSnapshots);
Require(ShaderResources::NeverReusable(stages) && !resources.Reusable(), "a fault-buffer build is reusable");
Require(!ShaderResources::NeverReusable(stages), "a fault-buffer build is refused the resource cache");
Require(!resources.Reusable(), "a fault-buffer build with copied writes is reusable");
Require(resources.NeedsCompletion() && !resources.UsesAddressTable(), "a fault-buffer build reaches guest memory through the BDA table");
const auto fault = access.bytes(access.descriptor(5).buffer);
for (const auto byte : fault) Require(byte == std::byte{}, "rect-list fault buffer was not initialized");
const ShaderRecompiler::BdaAbi::Fault report{ShaderRecompiler::BdaAbi::FaultState::Ready, ShaderRecompiler::BdaAbi::FaultReason::InvalidRectangle, 0, 0, 0, 0, 0};
@ -627,9 +898,23 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
resources.WriteBack();
Require(guest[1] == changed, "rect-list fault-only path lost guest buffer writes");
}
{
ShaderRecompiler::RecompileResult control;
control.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version;
control.bindings = {binding(Role::FaultBuffer, 5)};
const std::array<CompiledShader, 1> stages{{{ShaderRecompiler::ShaderStage::TessellationControl, &control, 0}}};
ShaderResources resources(disabled, stages, ColorTarget{}, 0, 0, 0, {});
Require(!ShaderResources::NeverReusable(stages) && resources.Reusable() && resources.ChecksFaults(), "a fault-only build is not reusable");
resources.CheckFaults();
const auto fault = access.bytes(access.descriptor(5).buffer);
const ShaderRecompiler::BdaAbi::Fault report{ShaderRecompiler::BdaAbi::FaultState::Ready, ShaderRecompiler::BdaAbi::FaultReason::InvalidRectangle, 0, 0, 0, 0, 0};
std::memcpy(fault.data(), &report, sizeof(report));
reject([&] { resources.CheckFaults(); }, "rect-list requires");
}
{
ShaderResources resources(context, compiled, snapshots);
Require(ShaderResources::NeverReusable(std::span<const CompiledShader>(&compiled, 1)) && !resources.Reusable(), "an address-based build is reusable");
Require(resources.UsesAddressTable(), "an address-based build does not use the BDA table");
ShaderRecompiler::RecompileResult plain;
plain.bindings = {binding(Role::GuestBuffers, 6)};
const CompiledShader plainStage{ShaderRecompiler::ShaderStage::Compute, &plain, 0};
@ -690,11 +975,38 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
tail.Upload(true);
reject([&] { tail.Descriptor(address + 58, 4, adjustment); }, "exceeds its GPU owner");
}
{
void* plain = GuestHeap::GuestHeapAllocate_nid_postfix(64);
void* gpu = GuestHeap::GuestHeapAlignForGpu_nid_postfix(16, 64);
const auto lookup = [](const void* pointer) {
GuestAllocations::Mutation mutation;
return mutation.Find(pointer);
};
Require(!lookup(plain).gpu && lookup(gpu).gpu, "a guest heap block's GPU access does not follow how it was allocated");
gpu = GuestHeap::GuestHeapReallocate_nid_postfix(gpu, 128);
plain = GuestHeap::GuestHeapRealign_nid_postfix(plain, 128, 64);
Require(!lookup(plain).gpu && lookup(gpu).gpu, "a reallocated guest heap block changed its GPU access");
{
GuestBufferMemory leased(context);
leased.AcquireRegistered();
leased.Upload(true);
const auto ranges = leased.AddressRanges();
const auto listed = [&](const void* pointer) {
const auto address = reinterpret_cast<std::uintptr_t>(pointer);
return std::any_of(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin <= address && address < range.end; });
};
Require(!listed(plain), "a guest heap block without GPU access is in an address-based build");
Require(listed(gpu), "a guest heap block with GPU access is missing from an address-based build");
leased.WriteBack();
}
GuestHeap::GuestHeapFree_nid_postfix(plain);
GuestHeap::GuestHeapFree_nid_postfix(gpu);
}
{
// The cached address space: a second build in an unchanged registry takes the first one's
// space, and a guest free of a range pinned only by the cache goes through the pin waiter's
// drop (no GPU work to wait for) and empties the registry of it.
void* block = GuestHeap::GuestHeapAllocate_nid_postfix(64);
void* block = GuestHeap::GuestHeapAlignForGpu_nid_postfix(16, 64);
const auto blockAddress = reinterpret_cast<std::uintptr_t>(block);
std::memset(block, 0x5a, 64);
const auto registered = [&] {
@ -724,11 +1036,15 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) {
if (dropped.enabled) Require(dropped.waiterDrops == after.waiterDrops + 1 && LeaseCounters().cacheDrops == dropped.waiterDrops, "the free did not drop the cached address space");
}
importCrossingTests(context, access);
importSpanTests(context);
heapMirrorTests(context, access);
importedHeapMirrorTests(context, access);
importWatchTests();
importedFreshTests(context);
gpuMappingTests(context);
stridedOverhangTests(context, access);
sparseHeapTests(context, access);
sparseHeapMirrorTests(context, access);
Require(AddressCopyOverflow({{0x1000, 0x3000, 0x2000, "uncommitted pages"}}, 0x2000).empty(), "copies within the limit were refused");
const auto copies = AddressCopyOverflow({{0x1000, 0x2000, 0x1000, "not mirrored"}, {0x10000, 0x30000, 0x18000, "uncommitted pages"}}, 0x2000);
Require(!copies.empty() && copies.find("0x10000+0x20000 (0.1 MiB committed, uncommitted pages)") < copies.find("0x1000+0x1000"), "the copy limit does not name the largest copy first");

58
core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp
View File

@ -105,6 +105,14 @@ PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) {
return it == table.end() ? nullptr : it->second;
}
VkDeviceMemory MemoryOf(VkBuffer buffer) {
return mock.bound.at(buffer).first;
}
VkDeviceSize ClassOf(VkBuffer buffer) {
return mock.memoryBytes.at(MemoryOf(buffer));
}
Context mockContext() {
Context context{};
context.deviceProc = mockProc;
@ -119,44 +127,46 @@ Context mockContext() {
void SizeClassesAndDirections() {
mock = MockDevice{};
auto context = mockContext();
VkBuffer first = VK_NULL_HANDLE;
VkDeviceMemory first = VK_NULL_HANDLE;
{
DeviceBuffer upload(context, 10 * MiB + 4096, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
first = upload.Handle();
Expect(mock.sizes.at(first) == 11 * MiB, "a 10 MiB + 4 KiB device buffer was created with " + std::to_string(mock.sizes.at(first)) + " bytes, not its 11 MiB class");
first = MemoryOf(upload.Handle());
Expect(mock.sizes.at(upload.Handle()) == 10 * MiB + 4096, "a 10 MiB + 4 KiB device buffer was created with " + std::to_string(mock.sizes.at(upload.Handle())) + " bytes, not the requested size");
Expect(ClassOf(upload.Handle()) == 11 * MiB, "a 10 MiB + 4 KiB device buffer has " + std::to_string(ClassOf(upload.Handle())) + " bytes of memory, not its 11 MiB class");
constexpr VkBufferUsageFlags all = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
Expect((mock.usages.at(first) & all) == all, "a device buffer was created without storage and both transfer directions");
Expect((mock.usages.at(upload.Handle()) & all) == all, "a device buffer was created without storage and both transfer directions");
Expect(upload.Size() == 10 * MiB + 4096, "a device buffer reports its class instead of the requested size");
}
const auto made = mock.allocations;
DeviceBuffer writeBack(context, 10 * MiB + 512 * 1024, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
Expect(writeBack.Handle() == first && mock.allocations == made, "a retained 11 MiB device buffer did not serve a write-back of another size and direction in its class");
Expect(MemoryOf(writeBack.Handle()) == first && mock.allocations == made, "a retained 11 MiB device buffer did not serve a write-back of another size and direction in its class");
Expect(writeBack.Size() == 10 * MiB + 512 * 1024, "a reused device buffer reports the retained size");
Expect(mock.sizes.at(writeBack.Handle()) == 10 * MiB + 512 * 1024, "a reused device buffer kept the VkBuffer of the earlier size");
}
void LargerClass() {
mock = MockDevice{};
auto context = mockContext();
constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VkBuffer twelve = VK_NULL_HANDLE;
VkDeviceMemory twelve = VK_NULL_HANDLE;
{
DeviceBuffer large(context, 12 * MiB, usage);
twelve = large.Handle();
twelve = MemoryOf(large.Handle());
}
{
DeviceBuffer smaller(context, 7 * MiB, usage);
Expect(smaller.Handle() == twelve, "a 7 MiB request did not take the retained 12 MiB buffer");
Expect(MemoryOf(smaller.Handle()) == twelve, "a 7 MiB request did not take the retained 12 MiB buffer");
}
const auto made = mock.allocations;
{
DeviceBuffer tooSmall(context, 5 * MiB, usage);
Expect(tooSmall.Handle() != twelve && mock.allocations == made + 1, "a 5 MiB request took a retained buffer of more than twice its size");
Expect(MemoryOf(tooSmall.Handle()) != twelve && mock.allocations == made + 1, "a 5 MiB request took a retained buffer of more than twice its size");
Expect(mock.sizes.at(tooSmall.Handle()) == 5 * MiB, "a 5 MiB device buffer was not created at its own class");
}
DeviceBuffer again(context, 12 * MiB, usage);
Expect(again.Handle() == twelve, "a buffer served to a smaller request did not return to its own 12 MiB class");
Expect(MemoryOf(again.Handle()) == twelve, "a buffer served to a smaller request did not return to its own 12 MiB class");
DeviceBuffer best(context, 4 * MiB + 1, usage);
Expect(best.Handle() != twelve && mock.sizes.at(best.Handle()) == 5 * MiB, "a 4 MiB + 1 request did not take the smallest retained class that fits (5 MiB)");
Expect(MemoryOf(best.Handle()) != twelve && ClassOf(best.Handle()) == 5 * MiB, "a 4 MiB + 1 request did not take the smallest retained class that fits (5 MiB)");
}
void KeptUntilFence() {
@ -187,28 +197,28 @@ void Budget() {
constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
std::vector<std::unique_ptr<DeviceBuffer>> live;
for (int i = 0; i < 12; ++i) live.push_back(std::make_unique<DeviceBuffer>(context, (48 + static_cast<std::size_t>(i) * 3) * MiB, usage));
const auto first = live.front()->Handle();
const auto first = MemoryOf(live.front()->Handle());
VkDeviceSize madeBytes = 0;
for (const auto& buffer : live) madeBytes += mock.sizes.at(buffer->Handle());
for (const auto& buffer : live) madeBytes += ClassOf(buffer->Handle());
Expect(madeBytes > 512 * MiB, "the budget case does not exceed the budget");
for (auto& buffer : live) buffer.reset();
Expect(mock.frees != 0 && mock.liveBytes <= 512 * MiB, "the device tier retains " + std::to_string(mock.liveBytes / MiB) + " MiB after " + std::to_string(mock.frees) + " evictions, over its 512 MiB budget");
DeviceBuffer probe(context, 48 * MiB, usage);
Expect(probe.Handle() != first, "the least recently used device buffer survived the eviction");
Expect(MemoryOf(probe.Handle()) != first, "the least recently used device buffer survived the eviction");
}
void AddressAndHostUnchanged() {
mock = MockDevice{};
auto context = mockContext();
VkBuffer scratch = VK_NULL_HANDLE;
VkDeviceMemory scratch = VK_NULL_HANDLE;
{
DeviceBuffer buffer(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
scratch = buffer.Handle();
scratch = MemoryOf(buffer.Handle());
}
Buffer addressed(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
Expect(addressed.Handle() != scratch && addressed.DeviceAddress() != 0, "an addressable device-local buffer took a scratch buffer without a device address");
Expect(MemoryOf(addressed.Handle()) != scratch && addressed.DeviceAddress() != 0, "an addressable device-local buffer took a scratch buffer without a device address");
Buffer shadow(context, 2 * MiB - 256, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
Expect(shadow.Handle() == scratch, "a device-local staging buffer did not share the scratch buffers' class");
Expect(MemoryOf(shadow.Handle()) == scratch, "a device-local staging buffer did not share the scratch buffers' class");
VkBuffer host = VK_NULL_HANDLE;
{
Buffer copy(context, 10 * MiB + 4096, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, HostVisible);
@ -226,19 +236,19 @@ void SmallDeviceClasses() {
mock = MockDevice{};
auto context = mockContext();
constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VkBuffer mebibyte = VK_NULL_HANDLE;
VkBuffer half = VK_NULL_HANDLE;
VkDeviceMemory mebibyte = VK_NULL_HANDLE;
VkDeviceMemory half = VK_NULL_HANDLE;
{
DeviceBuffer one(context, MiB, usage);
mebibyte = one.Handle();
mebibyte = MemoryOf(one.Handle());
}
{
DeviceBuffer smaller(context, 300 * 1024, usage);
Expect(smaller.Handle() != mebibyte && mock.sizes.at(smaller.Handle()) == 512 * 1024, "a 300 KiB request took a 1 MiB buffer or skipped its power-of-two class");
half = smaller.Handle();
Expect(MemoryOf(smaller.Handle()) != mebibyte && ClassOf(smaller.Handle()) == 512 * 1024, "a 300 KiB request took a 1 MiB buffer or skipped its power-of-two class");
half = MemoryOf(smaller.Handle());
}
DeviceBuffer again(context, 260 * 1024, usage);
Expect(again.Handle() == half, "a 260 KiB request did not take the retained 512 KiB buffer");
Expect(MemoryOf(again.Handle()) == half, "a 260 KiB request did not take the retained 512 KiB buffer");
constexpr auto indirect = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VkBuffer tiny = VK_NULL_HANDLE;
{

124
core/libs/prx/libSceAgcDriver/tests/ColorTransferTests.cpp
View File

@ -1,6 +1,9 @@
#include "ColorTransferTests.hpp"
#include "prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp"
#include "prx/libSceAgcDriver/Graphics/include/HalfFloatScanout.hpp"
#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp"
#include <algorithm>
#include <array>
#include <bit>
#include <cstring>
#include <memory>
@ -53,6 +56,125 @@ void checkConversion(const Context& context, std::uint32_t width, std::uint32_t
Require(std::equal(expectedTiled.begin(), expectedTiled.end(), guest.begin()), "GPU tile changed pixels or surface padding");
}
std::size_t renderTargetOffset64(std::uint32_t blocksPerRow, std::uint32_t x, std::uint32_t y) {
const auto* equation = FindTextureSwizzleEquation(27u, 8u);
Require(equation != nullptr, "no 8-byte SW_64KB_R_X equation");
std::size_t offset = 0;
for (std::uint32_t bit = 0; bit < 16u; ++bit) {
const auto mask = equation->bits[bit];
offset |= static_cast<std::size_t>(std::popcount((x & mask & 0xfffu) ^ ((y << 12u) & mask & 0xfff000u)) & 1) << bit;
}
return (static_cast<std::size_t>(y / 64u) * blocksPerRow + x / 128u) * 65536u + offset;
}
std::uint32_t deviceLocalType(const Context& context, std::uint32_t bits) {
for (std::uint32_t i = 0; i < context.memory.memoryTypeCount; ++i) {
if ((bits & (1u << i)) != 0 && (context.memory.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0) return i;
}
Require(false, "no device-local memory type for the test image");
return 0;
}
void checkHalfFloatDetile(const Context& context, std::uint32_t width, std::uint32_t height, bool redLow) {
const ColorTargetLayout layout(width, height, ColorTileMode::RenderTarget, 8);
std::vector<std::byte> storage(layout.Bytes() + layout.Alignment());
void* aligned = storage.data();
auto available = storage.size();
Require(std::align(layout.Alignment(), layout.Bytes(), aligned, available) != nullptr, "test surface alignment failed");
std::span<std::byte> guest(static_cast<std::byte*>(aligned), layout.Bytes());
std::fill(guest.begin(), guest.end(), std::byte{0xa5});
std::vector<std::byte> texels(static_cast<std::size_t>(width) * height * 8);
std::vector<std::byte> expected(static_cast<std::size_t>(width) * height * 4);
std::uint32_t seed = redLow ? 0x2545f491u : 0x9e3779b9u;
for (std::uint32_t y = 0; y < height; ++y) {
for (std::uint32_t x = 0; x < width; ++x) {
std::array<std::uint16_t, 4> halves{};
for (auto& half : halves) {
seed = seed * 1664525u + 1013904223u;
half = static_cast<std::uint16_t>(seed >> 16u);
}
const auto index = static_cast<std::size_t>(y) * width + x;
std::memcpy(texels.data() + index * 8, halves.data(), sizeof(halves));
std::memcpy(guest.data() + renderTargetOffset64(layout.BlocksPerRow(), x, y), halves.data(), sizeof(halves));
auto* pixel = expected.data() + index * 4;
pixel[0] = static_cast<std::byte>(HalfFloatScanoutCode(halves[redLow ? 2 : 0], false));
pixel[1] = static_cast<std::byte>(HalfFloatScanoutCode(halves[1], false));
pixel[2] = static_cast<std::byte>(HalfFloatScanoutCode(halves[redLow ? 0 : 2], false));
pixel[3] = static_cast<std::byte>(HalfFloatScanoutCode(halves[3], true));
}
}
VkImageCreateInfo imageInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
imageInfo.imageType = VK_IMAGE_TYPE_2D;
imageInfo.format = VK_FORMAT_R16G16B16A16_SFLOAT;
imageInfo.extent = {width, height, 1};
imageInfo.mipLevels = 1;
imageInfo.arrayLayers = 1;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
VkImage image = VK_NULL_HANDLE;
VkDeviceMemory memory = VK_NULL_HANDLE;
Check(context.Function<PFN_vkCreateImage>("vkCreateImage")(context.device, &imageInfo, nullptr, &image), "vkCreateImage half-float detile test");
struct Release {
const Context& context;
VkImage image;
VkDeviceMemory& memory;
~Release() {
context.Function<PFN_vkDestroyImage>("vkDestroyImage")(context.device, image, nullptr);
if (memory) context.Function<PFN_vkFreeMemory>("vkFreeMemory")(context.device, memory, nullptr);
}
} release{context, image, memory};
VkMemoryRequirements requirements{};
context.Function<PFN_vkGetImageMemoryRequirements>("vkGetImageMemoryRequirements")(context.device, image, &requirements);
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = deviceLocalType(context, requirements.memoryTypeBits);
Check(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory half-float detile test");
Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory half-float detile test");
Buffer upload(context, texels.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
std::memcpy(upload.Bytes().data(), texels.data(), texels.size());
Buffer fromGuest(context, expected.size(), VK_BUFFER_USAGE_TRANSFER_DST_BIT);
Buffer fromImage(context, expected.size(), VK_BUFFER_USAGE_TRANSFER_DST_BIT);
GpuColorTransfer transfer(context);
const auto barrier = context.Function<PFN_vkCmdPipelineBarrier>("vkCmdPipelineBarrier");
const auto readback = [&](VkCommandBuffer commands, Buffer& destination) {
const VkBufferCopy copy{0, 0, expected.size()};
context.Function<PFN_vkCmdCopyBuffer>("vkCmdCopyBuffer")(commands, transfer.LinearBuffer(), destination.Handle(), 1, &copy);
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT);
};
transfer.Upload(reinterpret_cast<std::uintptr_t>(aligned), width, height, ColorTileMode::RenderTarget, 8);
{
CommandBatch batch(context);
const auto commands = batch.Handle();
VkImageMemoryBarrier toTransfer{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
toTransfer.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
toTransfer.srcQueueFamilyIndex = toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toTransfer.image = image;
toTransfer.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
barrier(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toTransfer);
VkBufferImageCopy region{};
region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
region.imageExtent = {width, height, 1};
context.Function<PFN_vkCmdCopyBufferToImage>("vkCmdCopyBufferToImage")(commands, upload.Handle(), image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &region);
auto toGeneral = toTransfer;
toGeneral.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
toGeneral.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
toGeneral.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
toGeneral.newLayout = VK_IMAGE_LAYOUT_GENERAL;
barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toGeneral);
transfer.Detile(commands, redLow, false, true);
readback(commands, fromGuest);
transfer.DetileImage(commands, image, VK_IMAGE_LAYOUT_GENERAL, width, height, ColorTileMode::RenderTarget, redLow, false, true);
readback(commands, fromImage);
batch.SubmitAndWait();
}
Require(std::equal(expected.begin(), expected.end(), fromGuest.Bytes().begin()), "GPU 16-bit float detile from guest memory differs from the reference");
Require(std::equal(expected.begin(), expected.end(), fromImage.Bytes().begin()), "GPU 16-bit float detile from an image differs from the reference");
}
void checkBufferReuse(const Context& context) {
constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
VkBuffer handle = VK_NULL_HANDLE;
@ -115,4 +237,6 @@ void RunColorTransferTests(const AgcDriver::Graphics::Context& context) {
checkConversion(context, 130, 129, AgcDriver::Graphics::ColorTileMode::RenderTarget, false);
checkConversion(context, 257, 17, AgcDriver::Graphics::ColorTileMode::RenderTarget, true);
checkConversion(context, 192, 13, AgcDriver::Graphics::ColorTileMode::Linear, false);
checkHalfFloatDetile(context, 300, 130, true);
checkHalfFloatDetile(context, 129, 70, false);
}

144
core/libs/prx/libSceAgcDriver/tests/ControlFlow.cpp
View File

@ -9,6 +9,7 @@
#include <exception>
#include <fstream>
#include <initializer_list>
#include <iterator>
#include <map>
#include <span>
#include <sstream>
@ -43,24 +44,73 @@ void require(bool condition, const std::string& message) {
if (!condition) throw std::runtime_error(message);
}
std::vector<std::uint32_t> constructBlocks(const ControlFlowGraph& graph, std::uint32_t header, std::uint32_t exclude) {
class StructuralDominators {
public:
explicit StructuralDominators(const ControlFlowGraph& graph) : dominators(graph.blocks.size()) {
std::vector<std::vector<std::uint32_t>> predecessors(graph.blocks.size());
for (const auto& block : graph.blocks) {
for (const auto successor : block.successors) predecessors[successor].push_back(block.id);
const auto& terminator = block.terminator;
if (terminator.mergeBlock != InvalidControlFlowId) predecessors[terminator.mergeBlock].push_back(block.id);
if (terminator.loopHeader && terminator.continueBlock != InvalidControlFlowId) predecessors[terminator.continueBlock].push_back(block.id);
}
std::vector<std::uint32_t> all;
all.reserve(graph.blocks.size());
for (const auto& block : graph.blocks) all.push_back(block.id);
for (const auto& block : graph.blocks) dominators[block.id] = block.id == graph.entryBlock ? std::vector<std::uint32_t>{block.id} : all;
bool changed = true;
while (changed) {
changed = false;
for (const auto& block : graph.blocks) {
if (block.id == graph.entryBlock) continue;
auto next = predecessors[block.id].empty() ? std::vector<std::uint32_t>{} : dominators[predecessors[block.id].front()];
for (std::size_t i = 1; i < predecessors[block.id].size(); ++i) {
std::vector<std::uint32_t> intersection;
const auto& other = dominators[predecessors[block.id][i]];
std::set_intersection(next.begin(), next.end(), other.begin(), other.end(), std::back_inserter(intersection));
next = std::move(intersection);
}
if (!std::binary_search(next.begin(), next.end(), block.id)) {
next.insert(std::lower_bound(next.begin(), next.end(), block.id), block.id);
}
if (next != dominators[block.id]) {
dominators[block.id] = std::move(next);
changed = true;
}
}
}
}
bool Dominates(std::uint32_t dominator, std::uint32_t block) const {
const auto& values = dominators.at(block);
return std::binary_search(values.begin(), values.end(), dominator);
}
private:
std::vector<std::vector<std::uint32_t>> dominators;
};
std::vector<std::uint32_t> constructBlocks(const ControlFlowGraph& graph, const StructuralDominators& dominators, std::uint32_t header, std::uint32_t exclude) {
std::vector<std::uint32_t> blocks;
for (const auto& block : graph.blocks) {
if (graph.Dominates(header, block.id) && (exclude == InvalidControlFlowId || !graph.Dominates(exclude, block.id))) {
if (dominators.Dominates(header, block.id) && (exclude == InvalidControlFlowId || !dominators.Dominates(exclude, block.id))) {
blocks.push_back(block.id);
}
}
return blocks;
}
bool inEnclosingContinueConstruct(const ControlFlowGraph& graph, std::uint32_t header, std::uint32_t member) {
bool inEnclosingContinueConstruct(const ControlFlowGraph& graph, const StructuralDominators& dominators, std::uint32_t header, std::uint32_t member) {
return std::any_of(graph.blocks.begin(), graph.blocks.end(), [&](const BasicBlock& loop) {
const auto continueBlock = loop.terminator.continueBlock;
return loop.terminator.loopHeader && graph.Dominates(loop.id, header) && !graph.Dominates(continueBlock, header) && graph.Dominates(continueBlock, member);
return loop.terminator.loopHeader && dominators.Dominates(loop.id, header) && !dominators.Dominates(continueBlock, header) && dominators.Dominates(continueBlock, member);
});
}
void verifyStructured(const std::string& prefix, const ControlFlowGraph& graph) {
const StructuralDominators dominators(graph);
std::map<std::uint32_t, std::uint32_t> mergeOwners;
std::vector<std::pair<std::uint32_t, std::uint32_t>> loopExits;
for (const auto& block : graph.blocks) {
@ -72,10 +122,10 @@ void verifyStructured(const std::string& prefix, const ControlFlowGraph& graph)
for (const auto& block : graph.blocks) {
const auto& terminator = block.terminator;
if (terminator.loopHeader || terminator.mergeBlock == InvalidControlFlowId) continue;
for (const auto member : constructBlocks(graph, block.id, terminator.mergeBlock)) {
if (inEnclosingContinueConstruct(graph, block.id, member)) continue;
for (const auto member : constructBlocks(graph, dominators, block.id, terminator.mergeBlock)) {
if (inEnclosingContinueConstruct(graph, dominators, block.id, member)) continue;
for (const auto successor : graph.FindBlock(member).successors) {
if (successor == terminator.mergeBlock || (graph.Dominates(block.id, successor) && !graph.Dominates(terminator.mergeBlock, successor))) continue;
if (successor == terminator.mergeBlock || (dominators.Dominates(block.id, successor) && !dominators.Dominates(terminator.mergeBlock, successor))) continue;
const bool loopExit = std::any_of(loopExits.begin(), loopExits.end(), [&](const auto& exits) { return successor == exits.first || successor == exits.second; });
require(loopExit, prefix + "block " + std::to_string(member) + " of the selection at block " + std::to_string(block.id) + " branches to block " + std::to_string(successor) + " outside the construct");
}
@ -84,14 +134,51 @@ void verifyStructured(const std::string& prefix, const ControlFlowGraph& graph)
for (const auto& loop : graph.blocks) {
if (!loop.terminator.loopHeader) continue;
const auto continueBlock = loop.terminator.continueBlock;
for (const auto member : constructBlocks(graph, loop.id, loop.terminator.mergeBlock)) {
for (const auto member : constructBlocks(graph, dominators, loop.id, loop.terminator.mergeBlock)) {
const auto& terminator = graph.FindBlock(member).terminator;
if (terminator.loopHeader || graph.Dominates(continueBlock, member)) continue;
if (terminator.loopHeader || dominators.Dominates(continueBlock, member)) continue;
require(terminator.mergeBlock != continueBlock, prefix + "the selection at block " + std::to_string(member) + " in the loop at block " + std::to_string(loop.id) + " merges at the loop's continue block " + std::to_string(continueBlock));
}
}
}
ControlFlowGraph makeGraph(const std::vector<std::vector<std::uint32_t>>& successors) {
ControlFlowGraph graph;
graph.entryBlock = 0;
for (std::uint32_t id = 0; id < successors.size(); ++id) {
BasicBlock block;
block.id = id;
block.startProgramCounter = id * 8;
block.endProgramCounter = id * 8 + 8;
block.instructionBegin = id * 2;
block.instructionEnd = id * 2 + 2;
block.successors = successors[id];
auto& terminator = block.terminator;
if (successors[id].empty()) {
terminator.kind = TerminatorKind::Return;
} else if (successors[id].size() == 1) {
terminator.kind = TerminatorKind::Branch;
terminator.trueBlock = successors[id][0];
} else {
terminator.kind = TerminatorKind::ConditionalBranch;
terminator.condition = BranchCondition::SccNonZero;
terminator.trueBlock = successors[id][0];
terminator.falseBlock = successors[id][1];
}
graph.blocks.push_back(std::move(block));
}
for (const auto& block : graph.blocks) {
for (const auto successor : block.successors) graph.blocks[successor].predecessors.push_back(block.id);
}
return graph;
}
void verifyLoopMergeDoesNotEnterNestedSelection() {
auto graph = makeGraph({{1, 5}, {2, 3}, {5}, {5, 4}, {1}, {}});
Structurizer{}.Structurize(graph);
verifyStructured("loop merge structural dominance: ", graph);
}
std::uint32_t followEmptyBlocks(const std::string& prefix, const ControlFlowGraph& graph, std::uint32_t blockId, RouteState& routes) {
for (std::size_t steps = 0; steps <= graph.blocks.size(); ++steps) {
const auto& block = graph.FindBlock(blockId);
@ -303,6 +390,12 @@ void verifyNullSwappc() {
}
int main(int argc, char** argv) {
try {
verifyLoopMergeDoesNotEnterNestedSelection();
} catch (const std::exception& error) {
std::fprintf(stderr, "%s\n", error.what());
return 1;
}
if (argc > 1) {
int failures = 0;
for (int i = 1; i < argc; ++i) {
@ -371,6 +464,39 @@ latch:
buffer_store_dword v1, off, s[0:3], 0
s_endpgm)",
Store({0xbe880380u, 0x7e020280u, 0x7d880008u, 0xbf860004u, 0x7d880088u, 0xbf870003u, 0x4a020281u, 0xbf820002u, 0x4a020282u, 0x4a020283u, 0x80089008u, 0xbf0ac008u, 0xbf85fff5u}), Split::Clone},
{"loop selection arms that leave the iteration rejoin before a join the enclosing selection shares", R"(
v_mov_b32 v1, 0
s_mov_b32 s8, 0
loop:
s_cmp_eq_u32 s8, s2
s_cbranch_scc1 join
v_cmp_gt_u32 vcc, 8, v0
s_cbranch_vccz arm
v_add_nc_u32 v1, 1, v1
v_cmp_gt_u32 vcc, 16, v0
s_cbranch_vccz away
arm:
v_add_nc_u32 v1, 2, v1
v_cmp_gt_u32 vcc, 4, v0
s_cbranch_vccz aside
join:
v_add_nc_u32 v1, 3, v1
s_cmp_lt_u32 s8, 4
s_cbranch_scc0 done
s_branch latch
away:
v_add_nc_u32 v1, 4, v1
s_branch latch
aside:
v_add_nc_u32 v1, 5, v1
latch:
s_add_u32 s8, s8, 1
s_branch loop
done:
buffer_store_dword v1, off, s[0:3], 0
s_endpgm)",
Store({0x7e020280u, 0xbe880380u, 0xbf060208u, 0xbf850008u, 0x7d880088u, 0xbf860003u, 0x4a020281u, 0x7d880090u, 0xbf860007u, 0x4a020282u, 0x7d880084u,
0xbf860006u, 0x4a020283u, 0xbf0a8408u, 0xbf840006u, 0xbf820003u, 0x4a020284u, 0xbf820001u, 0x4a020285u, 0x80088108u, 0xbf82ffedu}), Split::None},
{"continue beside the inner merge of a nested selection in a loop", R"(
s_mov_b32 s8, 0
v_mov_b32 v1, 0

9
core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp
View File

@ -12,11 +12,13 @@ namespace {
using namespace AgcDriver::Graphics;
constexpr std::uint32_t FormatR8Uint = 5;
constexpr std::uint32_t FormatR16Unorm = 7;
constexpr std::uint32_t FormatR16Uint = 11;
constexpr std::uint32_t FormatR32Uint = 20;
constexpr std::uint32_t FormatR32Float = 22;
constexpr std::uint32_t FormatR11G11B10Float = 36;
constexpr std::uint32_t FormatR8G8B8A8Unorm = 56;
constexpr std::uint32_t TileDepth64KB = 0x18;
constexpr std::uint32_t TileRenderTarget64KB = 0x1b;
constexpr std::uint32_t Type2D = 9;
@ -163,8 +165,11 @@ void RunDepthSurfaceReuseTests() {
context.limits.maxFramebufferHeight = 16384;
constexpr std::uint64_t depth32 = 0x40000000;
constexpr std::uint64_t depth16 = 0x50000000;
constexpr std::uint64_t depthStencil = 0x60000000;
constexpr std::uint64_t stencil = 0x60800000;
DepthSurfaceView(context, {depth32, 0, {384, 384}, VK_FORMAT_D32_SFLOAT, 1.0f, 0});
DepthSurfaceView(context, {depth16, 0, {256, 256}, VK_FORMAT_D16_UNORM, 1.0f, 0});
DepthSurfaceView(context, {depthStencil, stencil, {320, 180}, VK_FORMAT_D32_SFLOAT_S8_UINT, 1.0f, 0});
Require(lookup(context, {depth32, FormatR32Float, 384, 384}) != nullptr, "an R32F view of a D32 surface's own extent must sample its depth plane");
Require(lookup(context, {depth16, FormatR16Unorm, 256, 256}) != nullptr, "an R16 view of a D16 surface's own extent must sample its depth plane");
@ -177,5 +182,9 @@ void RunDepthSurfaceReuseTests() {
Require(lookup(context, {depth32, FormatR32Uint, 384, 384, TileRenderTarget64KB}) == nullptr, "an R32 uint view without a depth layout must be read as reused memory");
Require(lookup(context, {depth16, FormatR32Float, 256, 256}) == nullptr, "an R32F view of a D16 surface's own extent must be read as reused memory");
Require(lookup(context, {stencil, FormatR8Uint, 320, 180}) != nullptr, "an 8-bit view of a stencil plane's own extent must sample the stencil plane");
Require(lookup(context, {stencil, FormatR8G8B8A8Unorm, 320, 180, TileRenderTarget64KB}) == nullptr, "an RGBA8 view of a stencil plane's own extent must be read as reused memory");
Require(lookup(context, {stencil, FormatR32Float, 320, 180}) == nullptr, "an R32F view of a stencil plane's own extent must be read as reused memory");
ClearDepthSurfaces(context.device);
}

31
core/libs/prx/libSceAgcDriver/tests/EmbeddedVertexFetchAliasing.cpp Normal file
View File

@ -0,0 +1,31 @@
#include "RdnaDecoder/RdnaInstructionDecoder.hpp"
#include "Translation/EmbeddedVertexFetch.hpp"
#include <array>
#include <cstdint>
#include <cstdio>
#include <vector>
using namespace ShaderRecompiler;
int main() {
int failures = 0;
for (const std::uint32_t waveSize : {32u, 64u}) {
for (const std::uint32_t indexRegister : {0u, 1u, 2u, 3u, 16u}) {
const std::array<std::uint32_t, 8> code{
0xf4000004u | (indexRegister << 6u), 0xfa00000cu,
0xf4080005u, indexRegister << 25u,
0x02140b08u,
0xe00c2000u, 0x80000b0au,
0xbf810000u,
};
const auto decoded = RdnaInstructionDecoder{}.Decode(code);
const auto plan = EmbeddedVertexFetchAnalyzer{}.Analyze(decoded, 0, 2, 0, 16, waveSize);
if (plan.loads.size() != 1 || plan.loads.front().attributeId != 3 || plan.loads.front().componentCount != 4 ||
plan.loads.front().programCounter != 20 || plan.loads.front().prologLoads != std::vector<std::uint32_t>{0, 8}) {
std::fprintf(stderr, "wave%u descriptor offset s%u lost its attribute or load provenance\n", waveSize, indexRegister);
++failures;
}
}
}
return failures == 0 ? 0 : 1;
}

151
core/libs/prx/libSceAgcDriver/tests/Graphics.cpp
View File

@ -468,6 +468,41 @@ void ShaderStageTests() {
expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "missing register");
}
void NggVertexWorkgroupTests() {
auto queue = makeState();
queue.context[0x2d5] = 0x12010;
queue.userConfig[0x242] = 4;
queue.userConfig[0x25b] = 0x10080;
queue.context[0x1ff] = 0x80;
queue.context[0x2ce] = 0;
queue.context[0x29b] = 2;
queue.context[0x2ab] = 1;
queue.shader[0x8a] = 0x222c0009;
queue.shader[0x8b] = 0x43001e;
queue.context[0x1b3] = 2;
queue.context[0x1b4] = 2;
Require(AgcDriver::Graphics::DecodeState(queue).stages.path == AgcDriver::Graphics::ShaderPath::Vertex, "an NGG vertex program without a workgroup left the vertex path");
auto state = AgcDriver::Graphics::DecodeState(queue, true);
Require(state.stages.path == AgcDriver::Graphics::ShaderPath::Geometry && state.stages.mesh, "an NGG vertex program that needs a workgroup did not take the mesh path");
auto mesh = *state.stages.mesh;
Require(mesh.inputPrimitive == 4 && mesh.primitivesPerGroup == 42 && mesh.verticesPerGroup == 126 && mesh.maxVertices == 128 && mesh.maxPrimitives == 128 && mesh.threadsPerGroup == 128 && mesh.ldsSizeDwords == 1024 && mesh.esgsItemSize == 1, "the NGG vertex subgroup of a triangle list changed");
queue.userConfig[0x242] = 6;
mesh = *AgcDriver::Graphics::DecodeState(queue, true).stages.mesh;
Require(mesh.primitivesPerGroup == 126 && mesh.verticesPerGroup == 128 && mesh.maxPrimitives == 128, "the NGG vertex subgroup of a triangle strip is not bounded by its vertex budget");
queue.shader[0x8a] = 0x222c0005;
queue.shader[0x8b] = 0x400030;
Require(AgcDriver::Graphics::DecodeState(queue, true).stages.mesh.has_value(), "an NGG vertex program that loads only its vertex ID was rejected");
queue.shader[0x8a] = 0x222c0009;
queue.shader[0x8b] = 0x43001e;
queue.context[0x1ff] = 2;
expectFailure([&] { AgcDriver::Graphics::DecodeState(queue, true); }, "invalid geometry subgroup output");
queue.context[0x1ff] = 0x80;
for (const auto stages : {0x02012010u, 0x00012030u, 0x00010010u}) {
queue.context[0x2d5] = stages;
expectFailure([&] { AgcDriver::Graphics::DecodeState(queue, true); }, "requires PRIMGEN_EN without GS_EN, PRIMGEN_PASSTHRU_EN or tessellation");
}
}
constexpr std::array<std::uint8_t, 8> IdentityExports{0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u};
std::vector<spv::BuiltIn> pixelBuiltinsRead(std::uint32_t ena, std::uint32_t addr, std::uint32_t source) {
@ -564,6 +599,9 @@ void ComputeScratchTests() {
expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeComputeStageInfo(shader, header)); }, "zero scratch size");
shader[0x213] = 0u;
Require(AgcDriver::Graphics::DecodeComputeStageInfo(shader, {}).scratchDwords == 0u, "a dispatch without SCRATCH_EN got scratch");
shader[0x207] = 0x00100040u;
Require(AgcDriver::Graphics::DecodeComputeStageInfo(shader, {}).numThreads[0] == 64u, "the partial group size leaked into the group size");
shader[0x207] = 64u;
ShaderRecompiler::RecompileRequest request{};
const std::array<std::uint32_t, 1> code{0xbf810000u};
request.shader = {ShaderRecompiler::ShaderStage::Compute, 0x30000u, code, 0, {}};
@ -613,6 +651,14 @@ void ColorPipeBankXorTests() {
}
}
void HeaderlessVertexStageTests() {
const std::array<std::uint32_t, 4> userData{1, 2, 3, 4};
const auto info = AgcDriver::Graphics::DecodeVertexStageInfo({}, 0, userData);
Require(!info.fetchEmbedded && info.resourcesNum == 0, "a vertex program without an AGC header received an embedded fetch table");
const std::array<std::byte, 8> truncated{};
expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeVertexStageInfo(truncated, 0x1000, userData)); }, "smaller than the fixed AGC header");
}
void PixelInputLayoutTests() {
using ShaderRecompiler::PixelInput;
using ShaderRecompiler::PixelInputVgpr;
@ -761,6 +807,19 @@ void TuningFieldTests() {
}
void ReversedComponentOrderTests() {
{
auto queue = makeState();
queue.context[0x31c] = (queue.context[0x31c] & ~((0x1fu << 2u) | (7u << 8u) | (3u << 11u))) | (3u << 2u) | (1u << 11u);
const auto state = AgcDriver::Graphics::DecodeState(queue);
Require(state.colors.size() == 1 && state.colors[0].format == VK_FORMAT_R8G8_UNORM && state.colors[0].componentMapping == 0xecu, "an 8_8 target with SWAP_ALT did not store the x and w exports in its two channels");
queue.context[0x1e0] = 0x40010001u;
for (const auto offset : {0x105u, 0x106u, 0x107u, 0x108u}) queue.context[offset] = 0;
const auto blended = AgcDriver::Graphics::DecodeState(queue);
Require(blended.blends.size() == 1 && blended.blends[0].blendEnable, "blending into an 8_8 SWAP_ALT target with one equation for color and alpha was rejected");
Require(!AgcDriver::Graphics::SplitsAlternateAlpha(blended), "one blend equation for color and alpha split an 8_8 SWAP_ALT draw");
queue.context[0x1e0] = 0x60000001u;
Require(AgcDriver::Graphics::SplitsAlternateAlpha(AgcDriver::Graphics::DecodeState(queue)), "a separate alpha blend into an 8_8 SWAP_ALT target did not split its second channel");
}
for (const auto& [swap, mapping] : {std::pair{2u, 0x1bu}, std::pair{3u, 0x93u}}) {
auto queue = makeState();
queue.context[0x31c] = (queue.context[0x31c] & ~(3u << 11u)) | (swap << 11u);
@ -824,6 +883,8 @@ void DepthStencilTests() {
queue.context[0x011] = 0x20000181;
for (const auto offset : {0x012u, 0x014u}) queue.context[offset] = 0x100;
for (const auto offset : {0x013u, 0x015u}) queue.context[offset] = 0x200;
queue.context[0x005] = 0x300;
queue.context[0x01e] = 0x1;
queue.context[0x10b] = 0x00050050;
queue.context[0x10c] = 0x01ffff01;
queue.context[0x10d] = 0x01000001;
@ -837,10 +898,36 @@ void DepthStencilTests() {
queue.context[0x1b4] = 2;
const auto rejection = AgcDriver::Graphics::DrawRejection(queue, false);
Require(rejection.empty(), "precheck rejected a stencil draw with a surface: " + rejection);
Require(state.depth && state.depth->address == 0x10000 && state.depth->stencilAddress == 0x20000 && state.depth->format == VK_FORMAT_D32_SFLOAT_S8_UINT && state.depth->clearStencil == 7, "depth surface decode changed");
Require(state.depth && state.depth->address == 0x10000 && state.depth->stencilAddress == 0x20000 && state.depth->format == VK_FORMAT_D32_SFLOAT_S8_UINT && state.depth->clearStencil == 7 && state.depth->htileAddress == 0x10000030000ull && !state.depth->htileStencil, "depth surface decode changed");
{
queue.context[0x011] &= ~(1u << 29u);
const auto stencilTiled = AgcDriver::Graphics::DecodeState(queue);
Require(stencilTiled.depth && stencilTiled.depth->htileStencil, "HTILE holds the stencil state without TILE_STENCIL_DISABLE");
queue.context[0x011] |= 1u << 29u;
queue.context[0x010] &= ~(1u << 29u);
const auto untiled = AgcDriver::Graphics::DecodeState(queue);
Require(untiled.depth && untiled.depth->htileAddress == 0, "HTILE must be ignored without TILE_SURFACE_ENABLE");
queue.context[0x010] |= 1u << 29u;
using AgcDriver::Graphics::HtileFillClears;
using AgcDriver::Graphics::HtileFillCovers;
constexpr VkImageAspectFlags both = VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT;
Require(HtileFillClears(0x0003fff0u, false) == VK_IMAGE_ASPECT_DEPTH_BIT, "a depth-only HTILE fill with ZMask 0 clears the depth");
Require(HtileFillClears(0x000000f0u, true) == both, "a depth and stencil fast clear (ZMask 0, SMem 0) clears both aspects");
Require(HtileFillClears(0x000003f0u, true) == VK_IMAGE_ASPECT_DEPTH_BIT, "SMem 3 leaves the stencil uncleared");
Require(HtileFillClears(0xfffff0ffu, true) == VK_IMAGE_ASPECT_STENCIL_BIT, "ZMask 0xf with SMem 0 clears only the stencil");
Require(HtileFillClears(0xffffffffu, true) == 0 && HtileFillClears(0xffffffffu, false) == 0, "an expanded fill clears nothing");
const VkExtent2D extent{16, 8};
Require(HtileFillCovers(0x10000, extent, 0x10000, 0x8000) && HtileFillCovers(0x10000, extent, 0xfff0, 0x8010), "a fill over the whole 32 KiB metadata block covers the surface");
Require(!HtileFillCovers(0x10000, extent, 0x10000, 0x7ffc) && !HtileFillCovers(0x10000, extent, 0x10004, 0x8000) && !HtileFillCovers(0, extent, 0, 0x10000), "a partial fill, one starting past the HTILE base, or no HTILE covers nothing");
// Pipe-aligned SW_64KB_Z_X places a 16x16 surface's HTILE words across its metadata block (a valid word at offset
// 260, AMD PAL), so a fill of the words a linear layout would hold covers nothing.
Require(!HtileFillCovers(0x10000, {16, 16}, 0x10000, 16) && !HtileFillCovers(0x10000, {16, 16}, 0x10000, 264), "a fill of the linear HTILE size of a 16x16 surface misses its pipe-aligned words");
Require(AgcDriver::Graphics::HtileMetadataBytes({1024, 512}) == 0x8000u && AgcDriver::Graphics::HtileMetadataBytes({1025, 512}) == 0x10000u && AgcDriver::Graphics::HtileMetadataBytes({3840, 2160}) == 20u * 0x8000u, "the HTILE metadata is 32 KiB per 1024x512 block");
}
Require(state.renderExtent.width == 4 && state.renderExtent.height == 2, "render extent ignores the depth surface");
Require(!state.depthTest && !state.depthWrite && state.stencilTest, "depth/stencil enables changed");
Require(state.depth->htileAddress == 0, "an HTILE surface without DB_HTILE_DATA_BASE decoded an HTILE address");
queue.context.erase(0x005);
Require(AgcDriver::Graphics::DecodeState(queue).depth->htileAddress == 0, "an HTILE surface without DB_HTILE_DATA_BASE decoded an HTILE address");
queue.context[0x005] = 0x300;
queue.context[0x01e] = 0x01;
log.clear();
@ -1111,6 +1198,12 @@ void DepthBoundsBiasTests() {
queue.context[0x205] = 0x00003a46u;
state = AgcDriver::Graphics::DecodeState(queue);
Require(state.depthBias && state.depthBiasConstant == 4.0f && state.cullMode == VK_CULL_MODE_BACK_BIT, "a triangle draw with the point and line offset enable lost its depth bias");
queue.context[0x205] = 0x00013a46u;
state = AgcDriver::Graphics::DecodeState(queue);
Require(state.depthBias && state.depthBiasConstant == 4.0f && state.cullMode == VK_CULL_MODE_BACK_BIT, "the window offset enable bit changed the raster decode while the window offset is zero");
queue.context[0x80] = 1;
expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "window offset");
queue.context[0x80] = 0;
queue.userConfig[0x242] = 2;
expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "polygon mode, depth bias or nonstandard rasterization");
queue.userConfig[0x242] = 4;
@ -1274,6 +1367,28 @@ void metadataPassTests() {
for (std::size_t offset = 0; offset < tiledBytes; offset += 4) filled = filled && std::memcmp(tiledTexels + offset, &texel, 4) == 0;
Require(filled && stored == expected && std::all_of(tiledKeys, tiledKeys + expected, [](std::uint8_t value) { return value == 0xff; }), std::string("a ") + code + " fast clear eliminate of a 128x128 SW_64KB_R_X target with " + alignment + " DCC stored uncompressed keys over " + std::to_string(stored) + " of its " + std::to_string(keyCount) + " DCC key bytes, expected " + std::to_string(expected));
}
std::vector<std::uint32_t> original(tiledBytes / 4);
for (std::size_t index = 0; index < original.size(); ++index) original[index] = 0x01000000u + static_cast<std::uint32_t>(index);
const auto resetSingle = [&] {
std::memcpy(tiledTexels, original.data(), tiledBytes);
std::memset(tiledKeys, 0x10, keyCount);
};
resetSingle();
expectFailure([&] { AgcDriver::Graphics::RunColorMetadataPass(context, *tiledPass); }, "fast-clear eliminate over comp-to-single");
Require(std::memcmp(tiledTexels, original.data(), tiledBytes) == 0 && std::all_of(tiledKeys, tiledKeys + keyCount, [](std::uint8_t value) { return value == 0x10; }), "a refused fast-clear eliminate over comp-to-single keys changed the " + alignment + " target");
resetSingle();
AgcDriver::Graphics::RunColorMetadataPass(context, {AgcDriver::Graphics::ColorMetadataPass::Mode::DccDecompress, tiledPass->targets});
const AgcDriver::Graphics::ColorTargetLayout layout(tiledSide, tiledSide, AgcDriver::Graphics::ColorTileMode::RenderTarget, 4);
bool expanded = true;
for (std::uint32_t y = 0; y < tiledSide && expanded; ++y) {
for (std::uint32_t x = 0; x < tiledSide && expanded; ++x) {
std::uint32_t texel = 0;
std::memcpy(&texel, tiledTexels + layout.Offset(x, y), 4);
expanded = texel == original[layout.Offset(x & ~7u, y & ~7u) / 4];
}
}
const auto stored = static_cast<std::size_t>(std::count(tiledKeys, tiledKeys + keyCount, std::uint8_t{0xff}));
Require(expanded && stored == expected, "a DCC decompress of comp-to-single keys over a 128x128 SW_64KB_R_X target with " + alignment + " DCC did not give every 8x8 block its first texel (" + std::to_string(stored) + " of " + std::to_string(keyCount) + " keys uncompressed, expected " + std::to_string(expected) + ")");
}
}
@ -1486,6 +1601,26 @@ void DepthClipTests() {
}
}
void AlphaToCoverageTests() {
auto queue = makeState();
queue.context[0x1b3] = 2;
queue.context[0x1b4] = 2;
Require(!AgcDriver::Graphics::DecodeState(queue).alphaToCoverage, "alpha-to-coverage decoded while disabled");
queue.context[0x2dc] = 0xaa01;
Require(AgcDriver::Graphics::DecodeState(queue).alphaToCoverage, "undithered alpha-to-coverage was not decoded");
Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "the precheck rejected undithered alpha-to-coverage");
for (const auto control : {0x8701u, 0x1aa01u, 0xaa03u}) {
queue.context[0x2dc] = control;
expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "alpha-to-coverage");
Require(AgcDriver::Graphics::DrawRejection(queue, false).find("alpha-to-coverage") != std::string::npos, "the precheck accepted unsupported alpha-to-coverage");
}
queue.context[0x2dc] = 0xaa01;
const auto exportFormat = queue.context.at(0x1c5);
queue.context[0x1c5] = (exportFormat & ~0xfu) | 1u;
expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "floating-point alpha");
queue.context[0x1c5] = exportFormat;
}
void InitialContextTests() {
const auto configured = makeState();
AgcDriver::QueueState queue;
@ -1503,7 +1638,7 @@ void InitialContextTests() {
}
queue.context[0x206] = 0x43f;
queue.context[0x2dc] |= 1;
expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "alpha-to-coverage");
Require(AgcDriver::Graphics::DecodeState(queue).alphaToCoverage, "undithered alpha-to-coverage was not decoded");
queue.context.erase(0x2dc);
expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "missing register");
queue.ClearContext();
@ -1971,6 +2106,7 @@ void descriptorSnapshotTests() {
const auto state = AgcDriver::Graphics::DecodeState(makeState());
ShaderResources resources(context, vertex, fragment, state.color, 0, 0);
const auto original = findWrite(0).buffers;
const auto originalThird = findWrite(4).buffers;
std::vector<ShaderResources::MovedBuffer> moved;
for (std::size_t index = 0; index < 2; ++index) {
if ((selected & (1u << index)) == 0) continue;
@ -1980,9 +2116,11 @@ void descriptorSnapshotTests() {
mock.writes.clear();
const auto snapshots = resources.PrepareDrawBindings(recorder, moved);
Require(snapshots != nullptr && snapshots->snapshots.size() == moved.size(), "draw snapshots are missing");
Require(mock.copies.size() == 2 && mock.copies[0].descriptorCount == 2 && mock.copies[1].descriptorCount == 1, "snapshot descriptor set did not copy every original binding");
Require(mock.writes.size() == 1, "snapshot updates must rewrite one complete affected array");
Require(mock.copies.empty(), "snapshot descriptor set must be written without descriptor copies");
Require(mock.writes.size() == 2, "snapshot update must write every binding once");
const auto& write = mock.writes.front();
const auto& untouched = mock.writes.back();
Require(untouched.binding == 4 && untouched.count == 1 && untouched.buffers[0].buffer == originalThird[0].buffer && untouched.buffers[0].offset == originalThird[0].offset && untouched.buffers[0].range == originalThird[0].range, "snapshot update changed a binding without snapshots");
Require(write.binding == 0 && write.arrayElement == 0 && write.count == 2, "snapshot update did not start from array element zero");
std::size_t snapshot = 0;
for (std::size_t index = 0; index < 2; ++index) {
@ -4005,12 +4143,15 @@ int main() {
uint16ExportTests();
uint8x4TargetTests();
ShaderStageTests();
NggVertexWorkgroupTests();
TuningFieldTests();
PixelInputLayoutTests();
ColorPipeBankXorTests();
HeaderlessVertexStageTests();
ComputeScratchTests();
shaderUserDataTailPaddingTests();
InitialContextTests();
AlphaToCoverageTests();
pushConstantTests();
resourceTests();
descriptorSnapshotTests();

13
core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp
View File

@ -288,7 +288,7 @@ void RunGuestSamplerResourceTests() {
Require(result.addressModeU == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, "clamp mode 2 must decode to clamp-to-edge");
Require(!result.anisotropyEnable && result.maxAnisotropy == 1.0f, "non-anisotropic filter must leave anisotropy disabled");
Require(nearlyEqual(result.maxLod, 12.0f), "max LOD decoded incorrectly");
Require(result.borderColor == VK_BORDER_COLOR_INT_TRANSPARENT_BLACK, "border color type 0 must decode to transparent black");
Require(result.borderColor == VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK, "border color type 0 must decode to transparent black");
Fields nearest = base;
nearest.xyMagFilter = 0;
@ -360,7 +360,7 @@ void RunGuestSamplerResourceTests() {
requireUnnormalized(unnormalized, "captured unnormalized S#");
Require(unnormalized.magFilter == VK_FILTER_LINEAR && unnormalized.minFilter == VK_FILTER_LINEAR, "captured unnormalized S# filters decoded incorrectly");
Require(unnormalized.addressModeU == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE && unnormalized.addressModeV == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, "captured unnormalized S# clamp modes decoded incorrectly");
Require(unnormalized.borderColor == VK_BORDER_COLOR_INT_TRANSPARENT_BLACK, "captured unnormalized S# border color decoded incorrectly");
Require(unnormalized.borderColor == VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK, "captured unnormalized S# border color decoded incorrectly");
Require(!DecodeSamplerResource(pack(base)).unnormalizedCoordinates, "a normalized S# decoded to unnormalized coordinates");
Fields unnormalizedBase = base;
@ -371,7 +371,7 @@ void RunGuestSamplerResourceTests() {
const auto point = DecodeSamplerResource(pack(unnormalizedPoint), true);
requireUnnormalized(point, "unnormalized point S#");
Require(point.magFilter == VK_FILTER_NEAREST && point.minFilter == VK_FILTER_NEAREST, "unnormalized point S# filters decoded incorrectly");
const std::array borders{VK_BORDER_COLOR_INT_TRANSPARENT_BLACK, VK_BORDER_COLOR_INT_OPAQUE_BLACK, VK_BORDER_COLOR_INT_OPAQUE_WHITE};
const std::array borders{VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK, VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK, VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE};
for (std::uint32_t type = 0; type < borders.size(); ++type) {
Fields border = unnormalizedBase;
border.clampX = 6;
@ -504,10 +504,13 @@ void RunGuestSamplerResourceTests() {
Fields opaqueBlack = base;
opaqueBlack.borderColorType = 1;
Require(DecodeSamplerResource(pack(opaqueBlack)).borderColor == VK_BORDER_COLOR_INT_OPAQUE_BLACK, "border color type 1 must decode to opaque black");
Require(DecodeSamplerResource(pack(opaqueBlack)).borderColor == VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK, "border color type 1 must decode to opaque black");
Require(DecodeSamplerResource(pack(opaqueBlack), false, false, true).borderColor == VK_BORDER_COLOR_INT_OPAQUE_BLACK, "border color type 1 of a sampler for integer images must decode to integer opaque black");
Fields opaqueWhite = base;
opaqueWhite.borderColorType = 2;
Require(DecodeSamplerResource(pack(opaqueWhite)).borderColor == VK_BORDER_COLOR_INT_OPAQUE_WHITE, "border color type 2 must decode to opaque white");
Require(DecodeSamplerResource(pack(opaqueWhite)).borderColor == VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE, "border color type 2 must decode to opaque white");
Require(DecodeSamplerResource(pack(opaqueWhite), false, false, true).borderColor == VK_BORDER_COLOR_INT_OPAQUE_WHITE, "border color type 2 of a sampler for integer images must decode to integer opaque white");
Require(DecodeSamplerResource(pack(base), false, false, true).borderColor == VK_BORDER_COLOR_INT_TRANSPARENT_BLACK, "border color type 0 of a sampler for integer images must decode to integer transparent black");
Fields positiveBias = base;
positiveBias.lodBiasRaw = 256;

337
core/libs/prx/libSceAgcDriver/tests/MaskedSelect.cpp
View File

@ -77,6 +77,22 @@ struct Builder {
return threadBit(low, high);
}
std::pair<IrValue*, IrValue*> ballotWords(IrValue& predicate) {
auto& ballot = emit(IrOpcode::Ballot, IrType::U32x4, {&predicate});
return {&emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &constant(0u)}), &emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &constant(1u)})};
}
IrValue& wordAnd(IrValue& left, IrValue& right) {
return emit(IrOpcode::BitwiseAnd32, IrType::U32, {&left, &right});
}
IrValue& phi(IrType type, IrBlock& predecessor, IrValue& incoming) {
auto& created = program.CreateValue(IrOpcode::Phi, type);
block->AppendInstruction(&created);
created.AddPhiOperand(&predecessor, &incoming);
return created;
}
void keep(IrValue& value) {
(void)emit(IrOpcode::ReferenceU32, IrType::Void, {&value});
}
@ -162,6 +178,29 @@ int main() {
return b.eliminate() == 2u && removed(first) && removed(second) && !removed(sink);
});
passed &= run("a write read through thousands of lane-local values still loses its select", [] {
Builder b;
auto& exec = b.mask(16u);
auto& old = b.lane();
auto& written = b.select(exec, b.add(old, 1u), old);
IrValue* value = &written;
for (std::uint32_t step = 0; step < 6000u; ++step) value = &b.add(*value, step);
b.keep(b.select(exec, *value, old));
return b.eliminate() == 1u && removed(written);
});
passed &= run("thousands of lane-local values ending in a wider read keep the select", [] {
Builder b;
auto& wide = b.mask(32u);
auto& exec = b.logicalAnd(wide, b.mask(16u));
auto& old = b.lane();
auto& written = b.select(exec, b.add(old, 1u), old);
IrValue* value = &written;
for (std::uint32_t step = 0; step < 6000u; ++step) value = &b.add(*value, step);
b.keep(b.select(wide, *value, old));
return b.eliminate() == 0u && !removed(written);
});
passed &= run("cross-lane reads must keep the select", [] {
bool ok = true;
for (const IrOpcode opcode : {IrOpcode::ReadFirstLane, IrOpcode::DppMoveU32, IrOpcode::BpermuteU32, IrOpcode::Permlane16U32, IrOpcode::SwizzleU32}) {
@ -275,6 +314,121 @@ int main() {
return count == 0u && !gone;
});
const auto mergedExec = [](bool wideBranch, bool loopMerge) {
Builder b;
auto& wide = b.mask(48u);
auto& exec = b.logicalAnd(wide, b.mask(32u));
auto& old = b.lane();
auto& written = b.select(exec, b.add(old, 1u), old);
IrBlock& head = *b.block;
IrBlock& body = b.newBlock();
IrBlock& merge = b.newBlock();
head.AddBranch(&body);
head.AddBranch(&merge);
body.AddBranch(&merge);
b.block = &body;
auto& killed = b.logicalAnd(wideBranch ? wide : exec, b.mask(16u));
auto& phi = b.program.CreateValue(IrOpcode::Phi, IrType::Bool);
merge.AppendInstruction(&phi);
phi.AddPhiOperand(&head, &exec);
phi.AddPhiOperand(&body, &killed);
if (loopMerge) {
IrBlock& exit = b.newBlock();
merge.AddBranch(&merge);
merge.AddBranch(&exit);
phi.AddPhiOperand(&merge, &phi);
b.block = &exit;
} else {
b.block = &merge;
}
b.keep(b.select(phi, b.add(written, 2u), old));
return std::pair{b.eliminate(), removed(written)};
};
passed &= run("an exec merged by a phi whose every incoming exec implies the write's exec drops the select", [&] {
const auto [count, gone] = mergedExec(false, false);
return count == 1u && gone;
});
passed &= run("an exec merged by a phi with a wider incoming exec keeps the select", [&] {
const auto [count, gone] = mergedExec(true, false);
return count == 0u && !gone;
});
passed &= run("an exec carried around a loop whose every incoming exec implies the write's exec drops the select", [&] {
const auto [count, gone] = mergedExec(false, true);
return count == 1u && gone;
});
passed &= run("an exec carried around a loop with a wider incoming exec keeps the select", [&] {
const auto [count, gone] = mergedExec(true, true);
return count == 0u && !gone;
});
const auto mergedWords = [](bool wideBranch) {
Builder b;
auto& wide = b.mask(48u);
auto& exec = b.logicalAnd(wide, b.mask(40u));
auto& old = b.lane();
auto& written = b.select(exec, b.add(old, 1u), old);
auto& saved = b.emit(IrOpcode::Ballot, IrType::U32x4, {&exec});
auto& savedLow = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&saved, &b.constant(0u)});
auto& savedHigh = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&saved, &b.constant(1u)});
IrBlock& head = *b.block;
IrBlock& body = b.newBlock();
IrBlock& merge = b.newBlock();
head.AddBranch(&body);
head.AddBranch(&merge);
body.AddBranch(&merge);
b.block = &body;
auto& survived = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.logicalAnd(wideBranch ? wide : exec, b.mask(24u))});
auto& survivedLow = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&survived, &b.constant(0u)});
auto& survivedHigh = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&survived, &b.constant(1u)});
auto& low = b.program.CreateValue(IrOpcode::Phi, IrType::U32);
auto& high = b.program.CreateValue(IrOpcode::Phi, IrType::U32);
merge.AppendInstruction(&low);
merge.AppendInstruction(&high);
low.AddPhiOperand(&head, &savedLow);
low.AddPhiOperand(&body, &survivedLow);
high.AddPhiOperand(&head, &savedHigh);
high.AddPhiOperand(&body, &survivedHigh);
b.block = &merge;
b.keep(b.select(b.threadBit(low, high), b.add(written, 2u), old));
return std::pair{b.eliminate(), removed(written)};
};
passed &= run("an exec rebuilt from mask words merged by phis implies the write's exec", [&] {
const auto [count, gone] = mergedWords(false);
return count == 1u && gone;
});
passed &= run("an exec rebuilt from merged mask words with a wider incoming ballot keeps the select", [&] {
const auto [count, gone] = mergedWords(true);
return count == 0u && !gone;
});
const auto chained = [](bool wideRoot) {
Builder b;
auto& wide = b.mask(48u);
auto& exec = b.logicalAnd(wide, b.mask(32u));
auto& old = b.lane();
auto& written = b.select(exec, b.add(old, 1u), old);
IrValue* later = wideRoot ? &wide : &exec;
for (std::uint32_t kill = 0; kill < 24u; ++kill) later = &b.logicalAnd(*later, b.mask(31u - kill));
b.keep(b.select(*later, b.add(written, 2u), old));
return std::pair{b.eliminate(), removed(written)};
};
passed &= run("an exec narrowed by a long chain of kills still implies the write's exec", [&] {
const auto [count, gone] = chained(false);
return count == 1u && gone;
});
passed &= run("a long chain of kills rooted at a wider exec keeps the select", [&] {
const auto [count, gone] = chained(true);
return count == 0u && !gone;
});
passed &= run("a write in a loop must keep its select", [] {
Builder b;
auto& exec = b.mask(16u);
@ -292,9 +446,10 @@ int main() {
return b.eliminate() == 0u && !removed(written);
});
passed &= run("a write read inside a later loop must keep its select", [] {
const auto laterLoop = [](bool wideRead) {
Builder b;
auto& exec = b.mask(16u);
auto& wide = b.mask(48u);
auto& exec = b.logicalAnd(wide, b.mask(16u));
auto& old = b.lane();
auto& written = b.select(exec, b.add(old, 1u), old);
IrBlock& head = *b.block;
@ -302,11 +457,142 @@ int main() {
head.AddBranch(&loop);
loop.AddBranch(&loop);
b.block = &loop;
b.keep(b.select(exec, b.add(written, 2u), old));
b.keep(b.select(wideRead ? wide : exec, b.add(written, 2u), old));
return std::pair{b.eliminate(), removed(written)};
};
passed &= run("a write read inside a later loop under its own exec loses its select", [&] {
const auto [count, gone] = laterLoop(false);
return count == 1u && gone;
});
passed &= run("a write read inside a later loop under a wider exec keeps its select", [&] {
const auto [count, gone] = laterLoop(true);
return count == 0u && !gone;
});
enum class Latch { Narrowed, Revived };
const auto carried = [](Latch latch, bool readInside) {
Builder b;
auto& wide = b.mask(48u);
auto& exec = b.logicalAnd(wide, b.mask(40u));
auto& old = b.lane();
auto& written = b.select(exec, b.add(old, 1u), old);
const auto [savedLow, savedHigh] = b.ballotWords(exec);
IrBlock& entry = *b.block;
IrBlock& header = b.newBlock();
IrBlock& exit = b.newBlock();
entry.AddBranch(&header);
header.AddBranch(&header);
header.AddBranch(&exit);
b.block = &header;
auto& low = b.phi(IrType::U32, entry, *savedLow);
auto& high = b.phi(IrType::U32, entry, *savedHigh);
auto& value = b.phi(IrType::U32, entry, written);
auto& loopExec = b.threadBit(low, high);
auto& next = b.select(loopExec, b.add(value, 2u), value);
if (readInside) b.keep(b.select(loopExec, b.add(value, 3u), old));
if (latch == Latch::Narrowed) {
const auto [stayLow, stayHigh] = b.ballotWords(b.mask(24u));
low.AddPhiOperand(&header, &b.wordAnd(low, *stayLow));
high.AddPhiOperand(&header, &b.wordAnd(high, *stayHigh));
} else {
const auto [revivedLow, revivedHigh] = b.ballotWords(wide);
low.AddPhiOperand(&header, revivedLow);
high.AddPhiOperand(&header, revivedHigh);
}
value.AddPhiOperand(&header, &next);
b.block = &exit;
b.keep(b.select(b.threadBit(*savedLow, *savedHigh), b.add(value, 4u), old));
return std::pair{b.eliminate(), removed(written)};
};
passed &= run("a write carried through a loop whose exec is saved, narrowed and restored loses its select", [&] {
const auto [count, gone] = carried(Latch::Narrowed, false);
return count == 1u && gone;
});
passed &= run("a write read inside a loop under the loop's narrowed exec loses its select", [&] {
const auto [count, gone] = carried(Latch::Narrowed, true);
return count == 1u && gone;
});
passed &= run("a lane that becomes active again inside a loop and reads the write keeps the select", [&] {
const auto [count, gone] = carried(Latch::Revived, true);
return count == 0u && !gone;
});
passed &= run("a lane that becomes active again inside a loop but never reads the write lets the select go", [&] {
const auto [count, gone] = carried(Latch::Revived, false);
return count == 1u && gone;
});
passed &= run("an exec proven only by assuming a loop phi is forgotten once that phi turns out wider", [] {
Builder b;
auto& wide = b.mask(48u);
auto& exec = b.logicalAnd(wide, b.mask(40u));
auto& old = b.lane();
auto& written = b.select(exec, b.add(old, 1u), old);
const auto [savedLow, savedHigh] = b.ballotWords(exec);
IrBlock& entry = *b.block;
IrBlock& header = b.newBlock();
IrBlock& kill = b.newBlock();
IrBlock& revive = b.newBlock();
IrBlock& latch = b.newBlock();
IrBlock& exit = b.newBlock();
entry.AddBranch(&header);
header.AddBranch(&kill);
header.AddBranch(&revive);
kill.AddBranch(&latch);
revive.AddBranch(&latch);
latch.AddBranch(&header);
latch.AddBranch(&exit);
b.block = &header;
auto& low = b.phi(IrType::U32, entry, *savedLow);
auto& high = b.phi(IrType::U32, entry, *savedHigh);
(void)b.select(b.threadBit(low, high), written, old);
b.block = &kill;
const auto [stayLow, stayHigh] = b.ballotWords(b.mask(24u));
auto& killedLow = b.wordAnd(low, *stayLow);
auto& killedHigh = b.wordAnd(high, *stayHigh);
b.keep(b.select(b.threadBit(killedLow, killedHigh), written, old));
b.block = &revive;
const auto [revivedLow, revivedHigh] = b.ballotWords(wide);
b.block = &latch;
auto& mergedLow = b.phi(IrType::U32, kill, killedLow);
auto& mergedHigh = b.phi(IrType::U32, kill, killedHigh);
mergedLow.AddPhiOperand(&revive, revivedLow);
mergedHigh.AddPhiOperand(&revive, revivedHigh);
low.AddPhiOperand(&latch, &mergedLow);
high.AddPhiOperand(&latch, &mergedHigh);
return b.eliminate() == 0u && !removed(written);
});
const auto sample = [](IrShaderStage stage, std::uint32_t sampleFlags) {
passed &= run("a write mask computed inside an earlier loop does not let that loop's phis stand for it", [] {
Builder b;
auto& old = b.lane();
IrBlock& entry = *b.block;
IrBlock& header = b.newBlock();
IrBlock& exit = b.newBlock();
entry.AddBranch(&header);
header.AddBranch(&header);
header.AddBranch(&exit);
b.block = &header;
auto& low = b.phi(IrType::U32, entry, b.constant(0u));
auto& high = b.phi(IrType::U32, entry, b.constant(0u));
auto& bound = b.phi(IrType::U32, entry, b.constant(64u));
auto& exec = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.lane(), &bound});
const auto [stayLow, stayHigh] = b.ballotWords(exec);
low.AddPhiOperand(&header, stayLow);
high.AddPhiOperand(&header, stayHigh);
bound.AddPhiOperand(&header, &b.emit(IrOpcode::ISub32, IrType::U32, {&bound, &b.constant(8u)}));
b.block = &exit;
auto& written = b.select(exec, b.add(old, 1u), old);
b.keep(b.select(b.threadBit(low, high), b.add(written, 2u), old));
return b.eliminate() == 0u && !removed(written);
});
const auto sample = [](IrShaderStage stage, std::uint32_t sampleFlags, IrOpcode opcode = IrOpcode::ImageSampleRaw) {
Builder b;
b.program.Resources().stage = stage;
MemoryInfo memory;
@ -322,7 +608,7 @@ int main() {
MemoryFlags flags{0u, 0u};
std::uint64_t bits = 0;
std::memcpy(&bits, &flags, sizeof(flags));
auto& sampled = b.emit(IrOpcode::ImageSampleRaw, IrType::U32x4, {&image, &sampler, &address}, bits);
auto& sampled = b.emit(opcode, IrType::U32x4, {&image, &sampler, &address}, bits);
b.keep(b.select(exec, b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&sampled, &b.constant(0u)}), old));
b.eliminate();
return removed(written);
@ -336,6 +622,26 @@ int main() {
return sample(IrShaderStage::Pixel, RdnaImageSampleFlagLevelZero) && sample(IrShaderStage::Mesh, 0u);
});
passed &= run("a pixel gather without an LOD of its own takes it from the quad and keeps the select", [&] {
return !sample(IrShaderStage::Pixel, 0u, IrOpcode::ImageGatherRaw) && !sample(IrShaderStage::Pixel, RdnaImageSampleFlagBias, IrOpcode::ImageGatherRaw);
});
passed &= run("a gather with an explicit LOD reads only its lane and drops the select", [&] {
return sample(IrShaderStage::Pixel, RdnaImageSampleFlagLevelZero, IrOpcode::ImageGatherRaw) && sample(IrShaderStage::Pixel, RdnaImageSampleFlagLod, IrOpcode::ImageGatherRaw) &&
sample(IrShaderStage::Mesh, 0u, IrOpcode::ImageGatherRaw);
});
passed &= run("an image size query reads only its lane's level and drops the select", [] {
Builder b;
auto& exec = b.mask(16u);
auto& old = b.lane();
auto& written = b.select(exec, b.add(old, 1u), old);
auto& address = b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {&written});
auto& size = b.emit(IrOpcode::ImageQueryDimensions, IrType::U32x4, {&b.emit(IrOpcode::GetImageResource, IrType::ImageResource, {}), &address});
b.keep(b.select(exec, b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&size, &b.constant(0u)}), old));
return b.eliminate() == 1u && removed(written);
});
const auto guarded = [](IrOpcode opcode, bool sameExec, bool asActive) {
Builder b;
auto& wide = b.mask(48u);
@ -350,6 +656,19 @@ int main() {
case IrOpcode::LoadBufferU32:
b.keep(b.select(exec, b.emit(opcode, IrType::U32, {&b.emit(IrOpcode::GetBufferResource, IrType::BufferResource, {}), &written, &b.constant(0u), &b.constant(0u), &active}), old));
break;
case IrOpcode::ImageRead: {
auto& address = b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {&written, &old});
auto& image = b.emit(IrOpcode::GetImageResource, IrType::ImageResource, {});
auto& read = b.emit(opcode, IrType::U32x4, {&image, &address, &active});
b.keep(b.select(exec, b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&read, &b.constant(0u)}), old));
break;
}
case IrOpcode::ImageWrite: {
auto& address = b.emit(IrOpcode::MakeImageAddress, IrType::ImageAddress, {&old, &old});
auto& texel = b.emit(IrOpcode::CompositeConstructU32x4, IrType::U32x4, {&written, &old, &old, &old});
b.emit(opcode, IrType::Void, {&b.emit(IrOpcode::GetImageResource, IrType::ImageResource, {}), &address, &texel, &active});
break;
}
default:
b.emit(opcode, IrType::Void, {&b.emit(IrOpcode::CompositeConstructU32x4, IrType::U32x4, {&written, &old, &old, &old}), &active});
break;
@ -359,11 +678,15 @@ int main() {
};
passed &= run("a store, load or export whose exec implies the write's exec drops the select", [&] {
return guarded(IrOpcode::WriteSharedU32, true, false) && guarded(IrOpcode::LoadBufferU32, true, false) && guarded(IrOpcode::SetAttribute, true, false);
return guarded(IrOpcode::WriteSharedU32, true, false) && guarded(IrOpcode::LoadBufferU32, true, false) && guarded(IrOpcode::ImageRead, true, false) && guarded(IrOpcode::SetAttribute, true, false);
});
passed &= run("a store, load or export under a wider exec keeps the select", [&] {
return !guarded(IrOpcode::WriteSharedU32, false, false) && !guarded(IrOpcode::LoadBufferU32, false, false) && !guarded(IrOpcode::SetAttribute, false, false);
return !guarded(IrOpcode::WriteSharedU32, false, false) && !guarded(IrOpcode::LoadBufferU32, false, false) && !guarded(IrOpcode::ImageRead, false, false) && !guarded(IrOpcode::SetAttribute, false, false);
});
passed &= run("an image write under the write's exec drops the select, under a wider exec or as its exec keeps it", [&] {
return guarded(IrOpcode::ImageWrite, true, false) && !guarded(IrOpcode::ImageWrite, false, false) && !guarded(IrOpcode::ImageWrite, true, true);
});
passed &= run("a value used as a store's exec keeps the select", [&] {

Some files were not shown because too many files have changed in this diff Show More