shifathosn
/
AnyPS5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into perf/shader-compare-constant-flush

This commit is contained in:
Karan Vora 2026-10-10 12:01:08 -04:00
parent 42692dee11 57f96fd2b3
commit bce20b2f35
779 changed files with 49992 additions and 3986 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/area-labels.yml vendored
View File

@ -23,6 +23,8 @@ jobs:
gh pr list -R "$GITHUB_REPOSITORY" --base "${{ github.event.repository.default_branch }}" -L 200 --json number --jq '.[].number'
echo EOF
} >> "$GITHUB_OUTPUT"
- if: steps.prs.outputs.numbers != ''
uses: actions/checkout@v7
- if: steps.prs.outputs.numbers != ''
uses: actions/labeler@v7
with:

85
.github/workflows/build.yml vendored
View File

@ -18,7 +18,7 @@ permissions:
concurrency:
group: build-${{ github.event_name }}-${{ github.event.pull_request.number || inputs.pr || github.ref }}
cancel-in-progress: true
cancel-in-progress: ${{ github.event_name != 'push' }}
env:
CONFIGURE: >-
@ -36,6 +36,8 @@ jobs:
pull-requests: read
outputs:
sha: ${{ steps.gate.outputs.sha }}
pr: ${{ steps.gate.outputs.pr }}
head: ${{ steps.gate.outputs.head }}
relinker: ${{ steps.gate.outputs.relinker }}
full: ${{ steps.gate.outputs.full }}
steps:
@ -50,6 +52,8 @@ jobs:
throw new Error('Automatic full builds only run on main');
}
core.setOutput('sha', context.sha);
core.setOutput('pr', '');
core.setOutput('head', '');
core.setOutput('relinker', true);
core.setOutput('full', true);
await core.summary.addRaw(`Building main at ${context.sha}: all builds and tests enabled`).write();
@ -83,7 +87,7 @@ jobs:
if (!manual && pr.head.sha !== context.payload.pull_request.head.sha) {
throw new Error('The pull request has newer commits; use the latest run');
}
if (pr.mergeable !== true || !pr.merge_commit_sha) {
if (pr.mergeable === false) {
throw new Error('The pull request does not merge cleanly into main; rebase it first');
}
let relinker = manual;
@ -93,11 +97,15 @@ jobs:
});
relinker = commits.length > 0;
}
core.setOutput('sha', pr.merge_commit_sha);
const { data: main } = await github.rest.repos.getBranch({ owner, repo, branch: 'main' });
core.setOutput('sha', main.commit.sha);
core.setOutput('pr', String(pull_number));
core.setOutput('head', pr.head.sha);
core.setOutput('relinker', relinker);
core.setOutput('full', manual);
core.info(`PR #${pull_number} at ${pr.head.sha} merged into main at ${main.commit.sha}`);
await core.summary.addRaw(
`PR #${pull_number} at ${pr.head.sha} merged into main as ${pr.merge_commit_sha}: full builds ${manual ? 'enabled' : 'disabled'}, ` +
`PR #${pull_number} at ${pr.head.sha} merged into main at ${main.commit.sha}: full builds ${manual ? 'enabled' : 'disabled'}, ` +
`relinker ${relinker ? 'enabled' : 'skipped because the author has no commit in main'}`
).write();
@ -127,6 +135,16 @@ jobs:
with:
ref: ${{ needs.eligibility.outputs.sha }}
persist-credentials: false
- name: Merge the pull request into main
if: ${{ needs.eligibility.outputs.head != '' }}
shell: bash
env:
PR_NUMBER: ${{ needs.eligibility.outputs.pr }}
PR_HEAD: ${{ needs.eligibility.outputs.head }}
run: |
git fetch --no-tags --unshallow origin "refs/pull/$PR_NUMBER/head"
if [ "$(git rev-parse FETCH_HEAD)" != "$PR_HEAD" ]; then echo "::error::The pull request has newer commits; start a new build"; exit 1; fi
if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit FETCH_HEAD; then echo "::error::The pull request does not merge cleanly into main; rebase it first"; exit 1; fi
- if: runner.os == 'Windows'
id: toolchain
uses: actions/cache@v6
@ -143,6 +161,33 @@ jobs:
- run: cmake --build build-relinker --parallel 3
- run: ctest --test-dir build-relinker --output-on-failure --timeout 120
register-context:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-15-intel]
configuration: [Debug, Release]
runs-on: ${{ matrix.os }}
timeout-minutes: 5
steps:
- uses: actions/checkout@v7
- run: cmake -S core/libs/tests/register_context -B build/register-context -DCMAKE_BUILD_TYPE=${{ matrix.configuration }}
- run: cmake --build build/register-context --parallel
- run: ctest --test-dir build/register-context --output-on-failure --repeat until-fail:3 --no-tests=error
write-watch:
strategy:
fail-fast: false
matrix:
os: [macos-15-intel, macos-15]
runs-on: ${{ matrix.os }}
timeout-minutes: 5
steps:
- uses: actions/checkout@v7
- run: cmake -S core/libs/tests/write_watch -B build/write-watch -DCMAKE_BUILD_TYPE=Release
- run: cmake --build build/write-watch --parallel
- run: ctest --test-dir build/write-watch --output-on-failure --repeat until-fail:3 --no-tests=error
linux:
needs: eligibility
if: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && needs.eligibility.outputs.full == 'true' }}
@ -154,10 +199,21 @@ jobs:
ref: ${{ needs.eligibility.outputs.sha }}
persist-credentials: false
submodules: true
- name: Merge the pull request into main
if: ${{ needs.eligibility.outputs.head != '' }}
shell: bash
env:
PR_NUMBER: ${{ needs.eligibility.outputs.pr }}
PR_HEAD: ${{ needs.eligibility.outputs.head }}
run: |
git fetch --no-tags --unshallow origin "refs/pull/$PR_NUMBER/head"
if [ "$(git rev-parse FETCH_HEAD)" != "$PR_HEAD" ]; then echo "::error::The pull request has newer commits; start a new build"; exit 1; fi
if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit FETCH_HEAD; then echo "::error::The pull request does not merge cleanly into main; rebase it first"; exit 1; fi
git submodule update --init --recursive
- uses: hendrikmuhs/ccache-action@v1.2.24
with:
key: linux
save: false
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
evict-old-files: job
- name: Install lavapipe for the GPU execution tests
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libvulkan1 mesa-vulkan-drivers
@ -166,10 +222,11 @@ jobs:
run: python3 tools/check_test_registration.py build
- run: cmake --build build
- run: cmake --build build --target libs
- run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log
- run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit ctest.xml
env:
ANYPS5_REQUIRE_VULKAN: 1
GALLIUM_OVERRIDE_CPU_CAPS: avx
- run: python3 tools/check_skips.py build/ctest.xml
- name: Report tests slower than 30 s
if: ${{ !cancelled() }}
shell: bash
@ -196,6 +253,17 @@ jobs:
ref: ${{ needs.eligibility.outputs.sha }}
persist-credentials: false
submodules: true
- name: Merge the pull request into main
if: ${{ needs.eligibility.outputs.head != '' }}
shell: bash
env:
PR_NUMBER: ${{ needs.eligibility.outputs.pr }}
PR_HEAD: ${{ needs.eligibility.outputs.head }}
run: |
git fetch --no-tags --unshallow origin "refs/pull/$PR_NUMBER/head"
if [ "$(git rev-parse FETCH_HEAD)" != "$PR_HEAD" ]; then echo "::error::The pull request has newer commits; start a new build"; exit 1; fi
if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit FETCH_HEAD; then echo "::error::The pull request does not merge cleanly into main; rebase it first"; exit 1; fi
git submodule update --init --recursive
- id: toolchain
uses: actions/cache@v6
with:
@ -209,7 +277,7 @@ jobs:
- uses: hendrikmuhs/ccache-action@v1.2.24
with:
key: windows
save: false
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
evict-old-files: job
- shell: cmd
run: cmake -S . -B build %CONFIGURE%
@ -221,7 +289,8 @@ jobs:
- shell: cmd
run: cmake --build build --target libs
- shell: cmd
run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log
run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit ctest.xml
- run: python tools/check_skips.py build/ctest.xml --job windows
- name: Report tests slower than 30 s
if: ${{ !cancelled() }}
shell: bash

5
.github/workflows/labels.yml vendored
View File

@ -19,11 +19,12 @@ jobs:
GH_REPO: ${{ github.repository }}
steps:
- run: |
label() { gh label create "$1" --color "$2" --description "$3"; }
existing=$(gh label list --limit 1000 --json name --jq '.[].name')
label() { grep -qixF -- "$1" <<< "$existing" || gh label create "$1" --color "$2" --description "$3"; }
force_label() { gh label create "$1" --color "$2" --description "$3" --force; }
force_label 'needs rebase' b60205 'Conflicts with main, rebase needed'
force_label 'intake paused' d93f0b 'Closed while PR intake is limited to authors with at least 20 PRs merged into main'
force_label 'intake paused' d93f0b 'Closed while PR intake is limited to authors with at least 10 PRs merged into main'
force_label ci 5319e7 '.github'
force_label agc 1d76db 'libSceAgc*'
force_label shader 1d76db 'core/shader'

4
.github/workflows/pr-intake.yml vendored
View File

@ -27,7 +27,7 @@ jobs:
if (pr.state !== 'open' || pr.base.ref !== 'main') {
return;
}
const minimumMergedPRs = 20;
const minimumMergedPRs = 10;
const { data: mergedPRs } = await github.rest.search.issuesAndPullRequests({
q: `repo:${owner}/${repo} is:pr is:merged base:main author:${pr.user.login}`,
per_page: 1
@ -48,7 +48,7 @@ jobs:
try {
await github.rest.issues.createLabel({
owner, repo, name, color: 'd93f0b',
description: 'Closed while PR intake is limited to authors with at least 20 PRs merged into main'
description: 'Closed while PR intake is limited to authors with at least 10 PRs merged into main'
});
} catch (createError) {
if (createError.status !== 422) {

10
.github/workflows/release.yml vendored
View File

@ -49,6 +49,8 @@ jobs:
path: C:\winlibs
key: ${{ env.WINLIBS_ZIP }}
- if: matrix.platform == 'windows' && steps.toolchain.outputs.cache-hit != 'true'
env:
SSL_CERT_FILE: 'C:\Program Files\Git\mingw64\etc\ssl\certs\ca-bundle.crt'
run: |
curl -sSfL -o "$RUNNER_TEMP/winlibs.zip" "https://github.com/brechtsanders/winlibs_mingw/releases/download/$WINLIBS_RELEASE/$WINLIBS_ZIP"
7z x "$RUNNER_TEMP/winlibs.zip" -oC:/winlibs
@ -58,12 +60,17 @@ jobs:
with:
key: ${{ matrix.platform }}
save: false
- name: Install lavapipe for the GPU execution tests
if: matrix.platform == 'linux'
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libvulkan1 mesa-vulkan-drivers
- name: Configure Linux
if: matrix.platform == 'linux'
run: cmake -S . -B build $CONFIGURE
- name: Configure Windows
if: matrix.platform == 'windows'
shell: cmd
env:
SSL_CERT_FILE: 'C:\Program Files\Git\mingw64\etc\ssl\certs\ca-bundle.crt'
run: cmake -S . -B build %CONFIGURE%
- name: Build project and tests (Linux)
if: matrix.platform == 'linux'
@ -75,6 +82,9 @@ jobs:
- name: Run tests (Linux)
if: matrix.platform == 'linux'
run: ctest --test-dir build --output-on-failure --timeout 120 --stop-on-failure --no-tests=error
env:
ANYPS5_REQUIRE_VULKAN: 1
GALLIUM_OVERRIDE_CPU_CAPS: avx
- name: Run tests (Windows)
if: matrix.platform == 'windows'
shell: cmd

17
CMakeLists.txt
View File

@ -39,12 +39,24 @@ if(BUILD_TESTING)
if(Python3_Interpreter_FOUND)
add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py)
add_test(NAME pr_overlap_tools COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap.py)
add_test(NAME progress_report COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress.py)
add_test(NAME progress_encoding COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_encoding.py)
add_test(NAME import_audit COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py)
add_test(NAME conventions_function_bodies COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_comments.py)
add_test(NAME test_registration COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_test_registration.py)
add_test(NAME hardware_oracle COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/hw-oracle/test_hw_oracle.py)
add_test(NAME pr_overlap_exports COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_exports.py)
add_test(NAME nid_database_cache COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_nid_names.py)
endif()
endif()
@ -88,6 +100,11 @@ set(FT_DISABLE_BROTLI ON CACHE BOOL "" FORCE)
set(SKIP_INSTALL_ALL ON CACHE BOOL "" FORCE)
add_subdirectory(3rdparty/freetype EXCLUDE_FROM_ALL)
set_target_properties(freetype PROPERTIES POSITION_INDEPENDENT_CODE ON)
if(WIN32)
get_target_property(freetypeSources freetype SOURCES)
list(FILTER freetypeSources EXCLUDE REGEX "ftver\\.rc$")
set_target_properties(freetype PROPERTIES SOURCES "${freetypeSources}")
endif()
add_subdirectory(core/shader/recompiler)
add_subdirectory(core/Decoder/Jpeg)

8
CONTRIBUTING.md
View File

@ -31,7 +31,13 @@ Python 3 is optional; without it some relinker tests are not registered.
Every push to `main`, including a merged pull request, automatically runs full Linux and Windows builds and tests and the relinker matrix on Linux, Windows and macOS, using the pushed commit without contributor checks.
For pull requests, full Linux and Windows builds and tests run only when a collaborator with write access selects Actions > Build > Run workflow on `main` and enters an open pull request number targeting `main`. This manual run also builds and tests the relinker on Linux, Windows and macOS, regardless of the author's previous contributions or the changed files. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main`; full builds never run automatically for pull requests. All build jobs build the selected pull request merged into the current `main` (GitHub's merge commit), so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds.
For pull requests, full Linux and Windows builds and tests run only when a collaborator with write access selects Actions > Build > Run workflow on `main` and enters an open pull request number targeting `main`. A collaborator can start the same manual run from the command line, replacing `1234` with the pull request number:
```
gh workflow run build.yml --repo boykopovar/AnyPS5 --ref main --field pr=1234
```
This manual run also builds and tests the relinker on Linux, Windows and macOS, regardless of the author's previous contributions or the changed files. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main`; full builds never run automatically for pull requests. All build jobs build the selected pull request merged into the current `main` (GitHub's merge commit), so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds.
The Conventions check runs automatically for every contributor, including first-time contributors. It uses the base branch checker and reads pull request Git objects without executing pull request code. The Conventions check runs on every pull request and fails when a rule on this page is broken. It accepts code comments only when the pull request also changes [TechnicalDebt](docs/dev/TechnicalDebt.md), and only UTF-8 text files. Run it locally before pushing:

2
core/Decoder/Png/src/Png.cpp
View File

@ -138,9 +138,9 @@ std::optional<Header> ParseHeader(std::span<const std::uint8_t> png) {
while (png.size() - offset >= CHUNK_OVERHEAD) {
const std::uint32_t length = readBigEndian32(&png[offset]);
const std::uint8_t* type = &png[offset + 4];
if (length > png.size() - offset - CHUNK_OVERHEAD) break;
if (isChunkType(type, "tRNS")) header.hasTransparency = true;
if (isChunkType(type, "tRNS") || isChunkType(type, "IDAT") || isChunkType(type, "IEND")) break;
if (length > png.size() - offset - CHUNK_OVERHEAD) break;
offset += CHUNK_OVERHEAD + length;
}
return header;

6
core/Decoder/Png/tests/Png.cpp
View File

@ -153,6 +153,12 @@ int main() {
Require(paletteHeader.has_value());
Require(paletteHeader->width == 2 && paletteHeader->height == 2 && paletteHeader->bitDepth == 2);
Require(paletteHeader->colorType == Decoder::Png::ColorType::Palette && paletteHeader->hasTransparency);
for (const std::size_t size : {std::size_t{69}, std::size_t{71}}) {
const auto truncated = Decoder::Png::ParseHeader(std::span<const std::uint8_t>(PALETTE_TRNS, size));
Require(truncated.has_value() && !truncated->hasTransparency);
}
const auto completeTransparency = Decoder::Png::ParseHeader(std::span<const std::uint8_t>(PALETTE_TRNS, 73));
Require(completeTransparency.has_value() && completeTransparency->hasTransparency);
const auto palette = Decoder::Png::Decode(PALETTE_TRNS);
Require(palette.has_value());
Require(palette->pixels == std::vector<std::uint8_t>({255, 0, 0, 255, 0, 255, 0, 0, 0, 0, 255, 255, 255, 255, 0, 128}));

24
core/libs/CMakeLists.txt
View File

@ -10,6 +10,7 @@ add_executable(nid_patcher
${CMAKE_CURRENT_SOURCE_DIR}/nid/src/ExportExclusions.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nid/src/ElfNidPatcher.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nid/src/PeNidPatcher.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nid/src/MachONidPatcher.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nid/src/BinaryPatcherFactory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/nid/src/main.cpp
)
@ -131,6 +132,11 @@ endforeach()
add_custom_target(libs DEPENDS ${PATCHED_OUTPUTS})
if(BUILD_TESTING)
find_package(Python3 QUIET COMPONENTS Interpreter)
if(Python3_Interpreter_FOUND)
add_test(NAME nid_patcher_macho COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_macho_nids.py $<TARGET_FILE:nid_patcher>)
endif()
add_test_executable(patched_libraries_tests tests/PatchedLibraries.cpp)
add_dependencies(patched_libraries_tests libs)
if(MINGW)
@ -138,6 +144,19 @@ if(BUILD_TESTING)
target_link_options(patched_libraries_tests PRIVATE -static -static-libgcc -static-libstdc++)
endif()
add_test(NAME patched_libraries COMMAND patched_libraries_tests ${PATCHED_DIR})
add_test_executable(guest_kernel_sys_tests tests/GuestKernelSys.cpp)
target_include_directories(guest_kernel_sys_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_kernel_sys_tests PRIVATE libkernel_sys libkernel libc)
configure_windows_unwind(guest_kernel_sys_tests)
add_test(NAME guest_kernel_sys COMMAND guest_kernel_sys_tests)
if(WIN32)
set_tests_properties(guest_kernel_sys PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libkernel_sys>")
endif()
if(APPLE)
add_subdirectory(tests/write_watch)
endif()
add_test_executable(guest_offline_imports_tests tests/GuestOfflineImports.cpp)
target_include_directories(guest_offline_imports_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(guest_offline_imports_tests PRIVATE libSceHttp libSceNpEntitlementAccess libSceRudp libc)
@ -147,6 +166,11 @@ if(BUILD_TESTING)
set_tests_properties(guest_offline_imports PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libSceHttp>")
endif()
if(UNIX AND (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$" OR CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64"))
add_subdirectory(tests/register_context)
endif()
add_test_executable(savedata_memory_metadata_tests tests/SaveDataMemoryMetadata.cpp)
target_include_directories(savedata_memory_metadata_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(savedata_memory_metadata_tests PRIVATE libSceSaveData.native)

33
core/libs/SceTypes.hpp
View File

@ -205,6 +205,19 @@ struct ModuleSegmentInfo {
std::int32_t prot;
};
struct ModuleInfo {
std::uint64_t st_size;
char name[256];
ModuleSegmentInfo segments[4];
std::uint32_t segment_count;
std::uint8_t fingerprint[20];
};
static_assert(offsetof(ModuleInfo, name) == 0x8);
static_assert(offsetof(ModuleInfo, segments) == 0x108);
static_assert(offsetof(ModuleInfo, segment_count) == 0x148);
static_assert(offsetof(ModuleInfo, fingerprint) == 0x14C);
static_assert(sizeof(ModuleInfo) == 0x160);
struct ModuleInfoEx {
std::uint64_t st_size;
char name[256];
@ -534,6 +547,13 @@ struct Audio3dOpenParameters {
std::uint32_t num_beds;
};
struct Audio3dAttribute {
std::uint32_t attribute_id;
std::uint32_t pad;
const void* value;
std::uint64_t value_size;
};
using AudioPropagationHandle = std::uint64_t;
struct AudioPropagationStructDescriptor {
@ -1827,6 +1847,19 @@ struct VideoOutOutputStatus {
std::uint64_t reserved[3] = {};
};
struct VideoOutResolutionStatus {
std::uint32_t fullWidth = 0;
std::uint32_t fullHeight = 0;
std::uint32_t paneWidth = 0;
std::uint32_t paneHeight = 0;
std::uint64_t refreshRate = 0;
float screenSizeInInch = 0.0f;
std::uint16_t flags = 0;
std::uint16_t reserved0 = 0;
std::uint32_t reserved1[3] = {};
};
static_assert(sizeof(VideoOutResolutionStatus) == 48 && offsetof(VideoOutResolutionStatus, refreshRate) == 16 && offsetof(VideoOutResolutionStatus, screenSizeInInch) == 24);
struct VideoOutOutputOptions { std::uint32_t internalData[16] = {}; };
struct VideoOutColorSettings {

30
core/libs/nid/include/nid/MachONidPatcher.hpp Normal file
View File

@ -0,0 +1,30 @@
#ifndef CORE_LIBS_NID_MACHONIDPATCHER_HPP
#define CORE_LIBS_NID_MACHONIDPATCHER_HPP
#include <nid/IBinaryPatcher.hpp>
#include <cstdint>
#include <string>
#include <vector>
namespace Nid {
constexpr std::uint32_t kMachOMagic64 = 0xfeedfacfu;
struct MachOExport {
std::string Name;
std::uint64_t Flags = 0;
std::uint64_t Address = 0;
std::uint64_t Other = 0;
std::string ImportName;
};
std::vector<MachOExport> ReadMachOExports(const std::vector<std::uint8_t>& binary);
class MachONidPatcher final : public IBinaryPatcher {
public:
void PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const override;
};
}
#endif

3
core/libs/nid/src/BinaryPatcherFactory.cpp
View File

@ -1,5 +1,6 @@
#include <nid/BinaryPatcherFactory.hpp>
#include <nid/ElfPatcher.hpp>
#include <nid/MachONidPatcher.hpp>
#include <nid/PeNidPatcher.hpp>
#include <stdexcept>
@ -10,6 +11,8 @@ std::unique_ptr<IBinaryPatcher> MakePatcher(const std::vector<std::uint8_t>& bin
return std::make_unique<ElfNidPatcher>();
if (binary.size() >= 2 && binary[0] == 'M' && binary[1] == 'Z')
return std::make_unique<PeNidPatcher>();
if (binary.size() >= 4 && binary[0] == 0xcf && binary[1] == 0xfa && binary[2] == 0xed && binary[3] == 0xfe)
return std::make_unique<MachONidPatcher>();
throw std::runtime_error("unrecognized binary format");
}

7
core/libs/nid/src/ExportExclusions.cpp
View File

@ -1,6 +1,7 @@
#include <nid/ExportExclusions.hpp>
#include <nid/ElfPatcher.hpp>
#include <nid/PeNidPatcher.hpp>
#include <nid/MachONidPatcher.hpp>
#include <nid/NidPatcherUtils.hpp>
#include <bit>
#include <cstring>
@ -121,6 +122,12 @@ std::unordered_set<std::string> ReadExportExclusions(const std::string& path) {
exports = ReadPeExports(binary);
} else if (binary.size() >= 4 && binary[0] == 0x7f && binary[1] == 'E' && binary[2] == 'L' && binary[3] == 'F') {
exports = ReadElfExports(binary);
} else if (binary.size() >= 4 && binary[0] == 0xcf && binary[1] == 0xfa && binary[2] == 0xed && binary[3] == 0xfe) {
for (const auto& entry : ReadMachOExports(binary)) {
if (entry.Name.size() < 2 || entry.Name[0] != '_') throw std::runtime_error("invalid reference Mach-O export: " + entry.Name);
if (!exports.insert(entry.Name.substr(1)).second) throw std::runtime_error("duplicate reference export: " + entry.Name);
}
if (exports.empty()) throw std::runtime_error("reference Mach-O has no exports");
} else {
throw std::runtime_error("unrecognized export reference format: " + path);
}

493
core/libs/nid/src/MachONidPatcher.cpp Normal file
View File

@ -0,0 +1,493 @@
#include <nid/MachONidPatcher.hpp>
#include <nid/NidResolver.hpp>
#include <nid/NidPatcherUtils.hpp>
#include <algorithm>
#include <cctype>
#include <cstddef>
#include <cstring>
#include <memory>
#include <optional>
#include <set>
#include <stdexcept>
#include <string_view>
#include <unordered_map>
namespace Nid {
namespace {
using Internal::Read;
using Internal::Write;
constexpr std::uint32_t kLcSegment64 = 0x19;
constexpr std::uint32_t kLcCodeSignature = 0x1d;
constexpr std::uint32_t kLcDyldInfo = 0x22;
constexpr std::uint32_t kLcDyldInfoOnly = 0x80000022u;
constexpr std::uint32_t kLcDyldExportsTrie = 0x80000033u;
constexpr std::uint32_t kLcDyldChainedFixups = 0x80000034u;
constexpr std::uint32_t kLcLoadDylib = 0x0c;
constexpr std::uint32_t kLcLoadWeakDylib = 0x80000018u;
constexpr std::uint32_t kLcReexportDylib = 0x8000001fu;
constexpr std::uint32_t kLcLazyLoadDylib = 0x20;
constexpr std::uint32_t kLcLoadUpwardDylib = 0x80000023u;
constexpr std::uint64_t kExportWeakDefinition = 0x04;
constexpr std::uint64_t kExportReexport = 0x08;
constexpr std::uint64_t kExportStubAndResolver = 0x10;
constexpr std::uint32_t kImportFormatPlain = 1;
constexpr std::uint32_t kImportFormatAddend = 2;
constexpr std::uint32_t kImportFormatAddend64 = 3;
constexpr std::uint64_t kPageSize = 0x1000;
struct MachHeader64 {
std::uint32_t Magic;
std::uint32_t CpuType;
std::uint32_t CpuSubtype;
std::uint32_t FileType;
std::uint32_t CommandCount;
std::uint32_t CommandBytes;
std::uint32_t Flags;
std::uint32_t Reserved;
};
struct LoadCommand {
std::uint32_t Cmd;
std::uint32_t Size;
};
struct SegmentCommand64 {
std::uint32_t Cmd;
std::uint32_t Size;
char Name[16];
std::uint64_t VmAddress;
std::uint64_t VmSize;
std::uint64_t FileOffset;
std::uint64_t FileSize;
std::uint32_t MaxProtection;
std::uint32_t InitProtection;
std::uint32_t SectionCount;
std::uint32_t Flags;
};
struct LinkeditDataCommand {
std::uint32_t Cmd;
std::uint32_t Size;
std::uint32_t DataOffset;
std::uint32_t DataSize;
};
struct DyldInfoCommand {
std::uint32_t Cmd;
std::uint32_t Size;
std::uint32_t RebaseOffset;
std::uint32_t RebaseSize;
std::uint32_t BindOffset;
std::uint32_t BindSize;
std::uint32_t WeakBindOffset;
std::uint32_t WeakBindSize;
std::uint32_t LazyBindOffset;
std::uint32_t LazyBindSize;
std::uint32_t ExportOffset;
std::uint32_t ExportSize;
};
struct ChainedFixupsHeader {
std::uint32_t Version;
std::uint32_t StartsOffset;
std::uint32_t ImportsOffset;
std::uint32_t SymbolsOffset;
std::uint32_t ImportsCount;
std::uint32_t ImportsFormat;
std::uint32_t SymbolsFormat;
};
void RequireRange(const std::vector<std::uint8_t>& binary, std::uint64_t offset, std::uint64_t size) {
if (offset > binary.size() || size > binary.size() - offset) throw std::runtime_error("Mach-O range out of bounds");
}
std::uint64_t ReadUleb(const std::vector<std::uint8_t>& data, std::size_t& cursor, std::size_t end) {
std::uint64_t value = 0;
unsigned shift = 0;
for (;;) {
if (cursor >= end) throw std::runtime_error("truncated uleb128 in export trie");
const auto byte = data[cursor++];
if (shift >= 64 || (shift == 63 && (byte & 0x7e) != 0)) throw std::runtime_error("uleb128 overflow in export trie");
value |= std::uint64_t{byte & 0x7fu} << shift;
if ((byte & 0x80) == 0) return value;
shift += 7;
}
}
void WriteUleb(std::vector<std::uint8_t>& out, std::uint64_t value) {
do {
std::uint8_t byte = value & 0x7f;
value >>= 7;
if (value != 0) byte |= 0x80;
out.push_back(byte);
} while (value != 0);
}
std::size_t UlebSize(std::uint64_t value) {
std::size_t size = 1;
while (value >>= 7) ++size;
return size;
}
std::string ReadTrieString(const std::vector<std::uint8_t>& data, std::size_t& cursor, std::size_t end) {
const auto start = cursor;
while (cursor < end && data[cursor] != 0) ++cursor;
if (cursor >= end) throw std::runtime_error("unterminated string in export trie");
std::string text(reinterpret_cast<const char*>(data.data() + start), cursor - start);
++cursor;
return text;
}
struct Image {
MachHeader64 Header {};
std::vector<std::size_t> Commands;
};
Image ParseImage(const std::vector<std::uint8_t>& binary) {
Image image;
image.Header = Read<MachHeader64>(binary, 0);
if (image.Header.Magic != kMachOMagic64) throw std::runtime_error("not a 64-bit little-endian Mach-O image");
RequireRange(binary, sizeof(MachHeader64), image.Header.CommandBytes);
std::size_t cursor = sizeof(MachHeader64);
const auto end = cursor + image.Header.CommandBytes;
for (std::uint32_t index = 0; index < image.Header.CommandCount; ++index) {
if (cursor + sizeof(LoadCommand) > end) throw std::runtime_error("load command out of bounds");
const auto command = Read<LoadCommand>(binary, cursor);
if (command.Size < sizeof(LoadCommand) || command.Size % 8 != 0 || command.Size > end - cursor) throw std::runtime_error("invalid load command size");
image.Commands.push_back(cursor);
cursor += command.Size;
}
return image;
}
struct TrieLocation {
std::size_t OffsetField;
std::size_t SizeField;
};
std::optional<TrieLocation> FindTrie(const std::vector<std::uint8_t>& binary, const Image& image) {
std::optional<TrieLocation> found;
for (const auto offset : image.Commands) {
const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
std::optional<TrieLocation> here;
if (cmd == kLcDyldExportsTrie) here = TrieLocation{offset + offsetof(LinkeditDataCommand, DataOffset), offset + offsetof(LinkeditDataCommand, DataSize)};
else if (cmd == kLcDyldInfo || cmd == kLcDyldInfoOnly) {
if (Read<DyldInfoCommand>(binary, offset).ExportSize != 0)
here = TrieLocation{offset + offsetof(DyldInfoCommand, ExportOffset), offset + offsetof(DyldInfoCommand, ExportSize)};
}
if (!here) continue;
if (found) throw std::runtime_error("Mach-O image has more than one export trie");
found = here;
}
return found;
}
void ParseTrieNode(const std::vector<std::uint8_t>& data, std::size_t start, std::size_t end, std::size_t node, std::string& prefix, std::set<std::size_t>& visited, std::vector<MachOExport>& out) {
if (node >= end - start) throw std::runtime_error("export trie node out of bounds");
if (!visited.insert(node).second) throw std::runtime_error("export trie has a cycle");
std::size_t cursor = start + node;
const auto terminalSize = ReadUleb(data, cursor, end);
if (terminalSize > end - cursor) throw std::runtime_error("export trie terminal out of bounds");
const auto children = cursor + terminalSize;
if (terminalSize != 0) {
MachOExport entry;
entry.Name = prefix;
entry.Flags = ReadUleb(data, cursor, children);
if (entry.Flags & kExportReexport) {
entry.Other = ReadUleb(data, cursor, children);
entry.ImportName = ReadTrieString(data, cursor, children);
} else {
entry.Address = ReadUleb(data, cursor, children);
if (entry.Flags & kExportStubAndResolver) entry.Other = ReadUleb(data, cursor, children);
}
out.push_back(std::move(entry));
}
cursor = children;
if (cursor >= end) throw std::runtime_error("export trie child count out of bounds");
const auto childCount = data[cursor++];
for (unsigned index = 0; index < childCount; ++index) {
const auto label = ReadTrieString(data, cursor, end);
const auto child = ReadUleb(data, cursor, end);
const auto length = prefix.size();
prefix += label;
ParseTrieNode(data, start, end, static_cast<std::size_t>(child), prefix, visited, out);
prefix.resize(length);
}
}
std::vector<MachOExport> ParseTrie(const std::vector<std::uint8_t>& binary, std::uint32_t offset, std::uint32_t size) {
RequireRange(binary, offset, size);
std::vector<MachOExport> exports;
if (size == 0) return exports;
std::string prefix;
std::set<std::size_t> visited;
ParseTrieNode(binary, offset, std::size_t{offset} + size, 0, prefix, visited, exports);
return exports;
}
class TrieBuilder {
public:
explicit TrieBuilder(const std::vector<MachOExport>& exports) {
for (const auto& entry : exports) Insert(entry);
}
std::vector<std::uint8_t> Build() {
std::vector<Node*> order;
Collect(&root, order);
for (bool changed = true; changed;) {
changed = false;
std::uint64_t offset = 0;
for (auto* node : order) {
if (node->Offset != offset) {
node->Offset = offset;
changed = true;
}
offset += NodeSize(*node);
}
}
std::vector<std::uint8_t> out;
for (auto* node : order) {
if (out.size() != node->Offset) throw std::runtime_error("export trie layout mismatch");
const auto terminal = TerminalBytes(*node);
WriteUleb(out, terminal.size());
out.insert(out.end(), terminal.begin(), terminal.end());
if (node->Children.size() > 255) throw std::runtime_error("export trie node has too many children");
out.push_back(static_cast<std::uint8_t>(node->Children.size()));
for (const auto& [label, child] : node->Children) {
out.insert(out.end(), label.begin(), label.end());
out.push_back(0);
WriteUleb(out, child->Offset);
}
}
while (out.size() % 8 != 0) out.push_back(0);
return out;
}
private:
struct Node {
std::vector<std::pair<std::string, std::unique_ptr<Node>>> Children;
std::optional<MachOExport> Entry;
std::uint64_t Offset = 0;
};
Node root;
void Insert(const MachOExport& entry) {
Node* node = &root;
std::string_view rest = entry.Name;
while (!rest.empty()) {
bool descended = false;
for (auto& [label, child] : node->Children) {
std::size_t common = 0;
while (common < label.size() && common < rest.size() && label[common] == rest[common]) ++common;
if (common == 0) continue;
if (common < label.size()) {
auto middle = std::make_unique<Node>();
middle->Children.emplace_back(label.substr(common), std::move(child));
label.resize(common);
child = std::move(middle);
}
node = child.get();
rest.remove_prefix(common);
descended = true;
break;
}
if (!descended) {
node->Children.emplace_back(std::string(rest), std::make_unique<Node>());
node = node->Children.back().second.get();
rest = {};
}
}
if (node->Entry) throw std::runtime_error("duplicate patched export name: " + entry.Name);
node->Entry = entry;
}
static void Collect(Node* node, std::vector<Node*>& order) {
order.push_back(node);
for (auto& [label, child] : node->Children) Collect(child.get(), order);
}
static std::vector<std::uint8_t> TerminalBytes(const Node& node) {
std::vector<std::uint8_t> bytes;
if (!node.Entry) return bytes;
const auto& entry = *node.Entry;
WriteUleb(bytes, entry.Flags);
if (entry.Flags & kExportReexport) {
WriteUleb(bytes, entry.Other);
bytes.insert(bytes.end(), entry.ImportName.begin(), entry.ImportName.end());
bytes.push_back(0);
} else {
WriteUleb(bytes, entry.Address);
if (entry.Flags & kExportStubAndResolver) WriteUleb(bytes, entry.Other);
}
return bytes;
}
static std::uint64_t NodeSize(const Node& node) {
const auto terminal = TerminalBytes(node).size();
std::uint64_t size = UlebSize(terminal) + terminal + 1;
for (const auto& [label, child] : node.Children) size += label.size() + 1 + UlebSize(child->Offset);
return size;
}
};
std::string StripUnderscore(const std::string& name) {
return !name.empty() && name[0] == '_' ? name.substr(1) : name;
}
bool IsGuestFacingImport(const std::string& name) {
using namespace Internal;
const bool hasNidPostfix = name.size() >= kNidPostfixLen && name.compare(name.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0;
const bool hasScePrefix = name.size() >= 3u &&
std::tolower(static_cast<unsigned char>(name[0])) == 's' &&
std::tolower(static_cast<unsigned char>(name[1])) == 'c' &&
std::tolower(static_cast<unsigned char>(name[2])) == 'e';
return hasNidPostfix || hasScePrefix;
}
std::vector<bool> DylibOrdinalsArePrx(const std::vector<std::uint8_t>& binary, const Image& image) {
std::vector<bool> result;
for (const auto offset : image.Commands) {
const auto command = Read<LoadCommand>(binary, offset);
if (command.Cmd != kLcLoadDylib && command.Cmd != kLcLoadWeakDylib && command.Cmd != kLcReexportDylib && command.Cmd != kLcLazyLoadDylib && command.Cmd != kLcLoadUpwardDylib) continue;
const auto nameOffset = Read<std::uint32_t>(binary, offset + 8);
if (nameOffset >= command.Size) throw std::runtime_error("dylib name out of bounds");
const auto name = Internal::ReadCStr(binary, offset + nameOffset);
result.push_back(name.size() >= 4 && name.compare(name.size() - 4, 4, ".prx") == 0);
}
return result;
}
void PatchChainedImports(std::vector<std::uint8_t>& binary, const Image& image) {
std::optional<LinkeditDataCommand> fixups;
for (const auto offset : image.Commands) {
const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
if (cmd == kLcDyldChainedFixups) fixups = Read<LinkeditDataCommand>(binary, offset);
if ((cmd == kLcDyldInfo || cmd == kLcDyldInfoOnly)) {
const auto info = Read<DyldInfoCommand>(binary, offset);
if (info.BindSize != 0 || info.LazyBindSize != 0 || info.WeakBindSize != 0) throw std::runtime_error("Mach-O bind opcodes are not supported; link with chained fixups");
}
}
if (!fixups || fixups->DataSize == 0) return;
RequireRange(binary, fixups->DataOffset, fixups->DataSize);
const auto base = std::size_t{fixups->DataOffset};
const auto header = Read<ChainedFixupsHeader>(binary, base);
if (header.Version != 0) throw std::runtime_error("unsupported chained fixups version");
if (header.SymbolsFormat != 0) throw std::runtime_error("compressed chained fixup symbols are not supported");
const std::size_t entrySize = header.ImportsFormat == kImportFormatPlain ? 4 : header.ImportsFormat == kImportFormatAddend ? 8 : header.ImportsFormat == kImportFormatAddend64 ? 16 : 0;
if (entrySize == 0) throw std::runtime_error("unsupported chained import format");
if (header.ImportsOffset > fixups->DataSize || std::uint64_t{header.ImportsCount} * entrySize > fixups->DataSize - header.ImportsOffset || header.SymbolsOffset > fixups->DataSize)
throw std::runtime_error("chained imports out of bounds");
const auto symbols = base + header.SymbolsOffset;
const auto symbolsEnd = base + fixups->DataSize;
const auto prx = DylibOrdinalsArePrx(binary, image);
struct Import { std::uint32_t NameOffset; std::uint32_t Ordinal; };
std::vector<Import> imports;
for (std::uint32_t index = 0; index < header.ImportsCount; ++index) {
const auto at = base + header.ImportsOffset + index * entrySize;
if (header.ImportsFormat == kImportFormatAddend64) {
const auto raw = Read<std::uint64_t>(binary, at);
imports.push_back({static_cast<std::uint32_t>(raw >> 32), static_cast<std::uint32_t>(raw & 0xffff)});
} else {
const auto raw = Read<std::uint32_t>(binary, at);
imports.push_back({raw >> 9, raw & 0xff});
}
}
std::set<std::uint32_t> nameOffsets;
for (const auto& item : imports) nameOffsets.insert(item.NameOffset);
std::set<std::uint32_t> rewritten;
for (const auto& item : imports) {
const bool special = header.ImportsFormat == kImportFormatAddend64 ? item.Ordinal >= 0xfff0 : item.Ordinal >= 0xf0;
if (item.Ordinal == 0 || special || item.Ordinal > prx.size() || !prx[item.Ordinal - 1]) continue;
if (rewritten.contains(item.NameOffset)) continue;
const auto at = symbols + item.NameOffset;
if (at >= symbolsEnd) throw std::runtime_error("chained import name out of bounds");
const auto name = Internal::ReadCStr(binary, at);
if (at + name.size() >= symbolsEnd) throw std::runtime_error("unterminated chained import name");
const auto bare = StripUnderscore(name);
if (bare == name || !IsGuestFacingImport(bare)) continue;
const auto patched = "_" + ResolveOneName(bare);
if (patched.size() > name.size()) throw std::runtime_error("import NID longer than original name: " + name);
const auto inside = nameOffsets.upper_bound(item.NameOffset);
if (inside != nameOffsets.end() && *inside <= item.NameOffset + name.size()) throw std::runtime_error("chained import names share storage: " + name);
std::memcpy(binary.data() + at, patched.data(), patched.size());
std::fill(binary.begin() + static_cast<std::ptrdiff_t>(at + patched.size()), binary.begin() + static_cast<std::ptrdiff_t>(at + name.size() + 1), 0);
rewritten.insert(item.NameOffset);
}
}
void WriteTrie(std::vector<std::uint8_t>& binary, const Image& image, const TrieLocation& location, const std::vector<std::uint8_t>& trie) {
const auto oldOffset = Read<std::uint32_t>(binary, location.OffsetField);
const auto oldSize = Read<std::uint32_t>(binary, location.SizeField);
if (trie.size() <= oldSize) {
std::copy(trie.begin(), trie.end(), binary.begin() + oldOffset);
std::fill(binary.begin() + oldOffset + static_cast<std::ptrdiff_t>(trie.size()), binary.begin() + oldOffset + oldSize, 0);
Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
return;
}
std::optional<std::size_t> linkedit;
for (const auto offset : image.Commands) {
const auto cmd = Read<LoadCommand>(binary, offset).Cmd;
if (cmd == kLcCodeSignature) throw std::runtime_error("signed Mach-O images cannot grow their export trie");
if (cmd != kLcSegment64) continue;
const auto segment = Read<SegmentCommand64>(binary, offset);
if (std::strncmp(segment.Name, "__LINKEDIT", sizeof(segment.Name)) == 0) linkedit = offset;
}
if (!linkedit) throw std::runtime_error("Mach-O image has no __LINKEDIT segment");
auto segment = Read<SegmentCommand64>(binary, *linkedit);
if (segment.FileOffset + segment.FileSize != binary.size()) throw std::runtime_error("__LINKEDIT does not end the file");
const auto newOffset = (binary.size() + 7) & ~std::size_t{7};
if (newOffset + trie.size() > 0xffffffffu) throw std::runtime_error("export trie offset overflow");
binary.resize(newOffset, 0);
binary.insert(binary.end(), trie.begin(), trie.end());
std::fill(binary.begin() + oldOffset, binary.begin() + oldOffset + oldSize, 0);
segment.FileSize = binary.size() - segment.FileOffset;
segment.VmSize = std::max(segment.VmSize, (segment.FileSize + kPageSize - 1) & ~(kPageSize - 1));
Write(binary, *linkedit, segment);
Write(binary, location.OffsetField, static_cast<std::uint32_t>(newOffset));
Write(binary, location.SizeField, static_cast<std::uint32_t>(trie.size()));
}
}
std::vector<MachOExport> ReadMachOExports(const std::vector<std::uint8_t>& binary) {
const auto image = ParseImage(binary);
const auto location = FindTrie(binary, image);
if (!location) return {};
return ParseTrie(binary, Read<std::uint32_t>(binary, location->OffsetField), Read<std::uint32_t>(binary, location->SizeField));
}
void MachONidPatcher::PatchNids(std::vector<std::uint8_t>& binary, const std::string& libraryName, const std::unordered_set<std::string>& excludedExports) const {
const auto image = ParseImage(binary);
const auto location = FindTrie(binary, image);
if (!location) throw std::runtime_error("no export trie");
auto exports = ParseTrie(binary, Read<std::uint32_t>(binary, location->OffsetField), Read<std::uint32_t>(binary, location->SizeField));
if (exports.empty()) throw std::runtime_error("no exported names");
std::vector<std::string> names;
names.reserve(exports.size());
for (const auto& entry : exports) {
if (entry.Name.empty() || entry.Name[0] != '_') throw std::runtime_error("exported name without a C underscore: " + entry.Name);
names.push_back(entry.Name.substr(1));
}
const auto nidMap = ResolveNids(names, libraryName, excludedExports);
for (std::size_t index = 0; index < exports.size(); ++index) {
const auto& name = names[index];
const bool marked = name.ends_with(Internal::kNidPostfix) || Internal::IsNidNoPatch(name) || Internal::IsNidNoPatchCut(name);
if ((exports[index].Flags & kExportWeakDefinition) != 0 && !marked) continue;
const auto found = nidMap.find(name);
if (found == nidMap.end()) throw std::runtime_error("symbol not in nid map: " + name);
exports[index].Name = "_" + found->second;
}
WriteTrie(binary, image, *location, TrieBuilder(exports).Build());
PatchChainedImports(binary, ParseImage(binary));
}
}

235
core/libs/nid/tests/test_macho_nids.py Normal file
View File

@ -0,0 +1,235 @@
"""Check nid_patcher on small byte-built Mach-O dylibs: the export trie, the chained imports and what it refuses."""
import hashlib
import struct
import subprocess
import sys
import tempfile
from pathlib import Path
SUFFIX = bytes([0x51, 0x8D, 0x64, 0xA6, 0x35, 0xDE, 0xD8, 0xC1, 0xE6, 0xB0, 0x39, 0xB1, 0xC3, 0xE5, 0x52, 0x30])
BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-"
LC_SEGMENT_64 = 0x19
LC_LOAD_DYLIB = 0x0C
LC_CODE_SIGNATURE = 0x1D
LC_DYLD_INFO_ONLY = 0x80000022
LC_DYLD_EXPORTS_TRIE = 0x80000033
LC_DYLD_CHAINED_FIXUPS = 0x80000034
EXPORT_WEAK = 0x04
PAGE = 0x1000
def nid(name):
digest = hashlib.sha1(name.encode() + SUFFIX).digest()
reversed_bytes = digest[:8][::-1]
text = ""
for index in (0, 3):
triple = (reversed_bytes[index] << 16) | (reversed_bytes[index + 1] << 8) | reversed_bytes[index + 2]
text += BASE64[(triple >> 18) & 63] + BASE64[(triple >> 12) & 63] + BASE64[(triple >> 6) & 63] + BASE64[triple & 63]
tail = (reversed_bytes[6] << 16) | (reversed_bytes[7] << 8)
return text + BASE64[(tail >> 18) & 63] + BASE64[(tail >> 12) & 63] + BASE64[(tail >> 6) & 63]
def uleb(value):
out = bytearray()
while True:
byte = value & 0x7F
value >>= 7
out.append(byte | (0x80 if value else 0))
if not value:
return bytes(out)
def read_uleb(data, position):
value = shift = 0
while True:
byte = data[position]
position += 1
value |= (byte & 0x7F) << shift
if not byte & 0x80:
return value, position
shift += 7
def build_trie(entries):
offsets = [0] * len(entries)
for _ in range(8):
root = bytearray(b"\0") + bytes([len(entries)])
for (name, _, _), offset in zip(entries, offsets):
root += name.encode() + b"\0" + uleb(offset)
nodes = []
for name, flags, address in entries:
terminal = uleb(flags) + uleb(address)
nodes.append(uleb(len(terminal)) + terminal + b"\0")
position, expected = len(root), []
for node in nodes:
expected.append(position)
position += len(node)
if expected == offsets:
break
offsets = expected
trie = bytes(root) + b"".join(nodes)
return trie + bytes(-len(trie) % 8)
def parse_trie(data, start, size):
found = {}
def walk(node, prefix):
position = start + node
terminal, position = read_uleb(data, position)
children = position + terminal
if terminal:
flags, position = read_uleb(data, position)
address, position = read_uleb(data, position)
found[prefix] = (flags, address)
count = data[children]
position = children + 1
for _ in range(count):
end = data.index(0, position)
label = data[position:end].decode()
child, position = read_uleb(data, end + 1)
walk(child, prefix + label)
if size:
walk(0, "")
return found
def build_fixups(imports):
symbols = bytearray(b"\0")
entries = []
for name, ordinal in imports:
entries.append(ordinal | (len(symbols) << 9))
symbols += name.encode() + b"\0"
imports_offset = 32
header = struct.pack("<7I", 0, 28, imports_offset, imports_offset + 4 * len(imports), len(imports), 1, 0)
blob = header + struct.pack("<I", 0) + b"".join(struct.pack("<I", entry) for entry in entries) + bytes(symbols)
return blob + bytes(-len(blob) % 8)
def dylib_command(path):
name = path.encode() + b"\0"
size = (24 + len(name) + 7) & ~7
return struct.pack("<IIIIII", LC_LOAD_DYLIB, size, 24, 2, 0x10000, 0x10000) + name.ljust(size - 24, b"\0")
def build_dylib(exports, imports, signed=False, bind_opcodes=False):
trie = build_trie(exports)
fixups = build_fixups(imports)
linkedit = bytearray(trie + fixups)
trie_command = (struct.pack("<IIII", LC_DYLD_EXPORTS_TRIE, 16, PAGE, len(trie)))
fixups_command = struct.pack("<IIII", LC_DYLD_CHAINED_FIXUPS, 16, PAGE + len(trie), len(fixups))
if bind_opcodes:
trie_command = struct.pack("<IIIIIIIIIIII", LC_DYLD_INFO_ONLY, 48, 0, 0, PAGE, 8, 0, 0, 0, 0, PAGE, len(trie))
extra = [trie_command, fixups_command, dylib_command("@rpath/libkernel.prx"), dylib_command("/usr/lib/libSystem.B.dylib")]
if signed:
extra.append(struct.pack("<IIII", LC_CODE_SIGNATURE, 16, PAGE + len(linkedit), 0))
text = struct.pack("<II16sQQQQiiII", LC_SEGMENT_64, 72, b"__TEXT", 0, PAGE, 0, PAGE, 5, 5, 0, 0)
link = struct.pack("<II16sQQQQiiII", LC_SEGMENT_64, 72, b"__LINKEDIT", PAGE, PAGE, PAGE, len(linkedit), 1, 1, 0, 0)
body = text + link + b"".join(extra)
header = struct.pack("<IIIIIIII", 0xFEEDFACF, 0x01000007, 3, 6, 2 + len(extra), len(body), 0, 0)
image = bytearray(header + body)
image += bytes(PAGE - len(image))
image += linkedit
return bytes(image)
def command_fields(data):
offset, found = 32, {}
for _ in range(struct.unpack_from("<I", data, 16)[0]):
cmd, size = struct.unpack_from("<II", data, offset)
found.setdefault(cmd, []).append(offset)
offset += size
return found
def segment(data, name):
for offset in command_fields(data)[LC_SEGMENT_64]:
fields = struct.unpack_from("<16sQQQQ", data, offset + 8)
if fields[0].rstrip(b"\0").decode() == name:
return offset, fields[1:]
raise AssertionError(name)
def chained_imports(data):
offset = command_fields(data)[LC_DYLD_CHAINED_FIXUPS][0]
base, size = struct.unpack_from("<II", data, offset + 8)
_, _, imports_offset, symbols_offset, count, _, _ = struct.unpack_from("<7I", data, base)
result = []
for index in range(count):
raw, = struct.unpack_from("<I", data, base + imports_offset + 4 * index)
start = base + symbols_offset + (raw >> 9)
result.append((data[start:data.index(0, start)].decode(), raw & 0xFF))
return result
def patch(patcher, directory, name, image):
path = Path(directory) / name
path.write_bytes(image)
result = subprocess.run([str(patcher), "libtest", str(path)], capture_output=True, text=True, timeout=30)
return result, path
def exports_of(data):
offset = command_fields(data)[LC_DYLD_EXPORTS_TRIE][0]
start, size = struct.unpack_from("<II", data, offset + 8)
return parse_trie(data, start, size)
def check_shrinking(patcher, directory):
exports = [("_puts_nid_postfix", 0, 0x100), ("_exit_nid_postfix", 0, 0x108), ("_weak_inline", EXPORT_WEAK, 0x110)]
imports = [("_malloc", 2), ("_puts_nid_postfix", 1), ("_sceKernelFoo", 1), ("_helper", 1), ("_exit_nid_postfix", 2)]
result, path = patch(patcher, directory, "shrink.dylib", build_dylib(exports, imports))
assert result.returncode == 0, (result.stdout, result.stderr)
data = path.read_bytes()
assert exports_of(data) == {"_" + nid("puts"): (0, 0x100), "_" + nid("exit"): (0, 0x108), "_weak_inline": (EXPORT_WEAK, 0x110)}, exports_of(data)
assert chained_imports(data) == [("_malloc", 2), ("_" + nid("puts"), 1), ("_" + nid("sceKernelFoo"), 1), ("_helper", 1), ("_exit_nid_postfix", 2)], chained_imports(data)
assert len(data) == len(build_dylib(exports, imports)), "the image changed size"
def check_growing(patcher, directory):
original = build_dylib([("_ab", 0, 0x100), ("_cd", 0, 0x108)], [])
result, path = patch(patcher, directory, "grow.dylib", original)
assert result.returncode == 0, (result.stdout, result.stderr)
data = path.read_bytes()
assert exports_of(data) == {"_" + nid("ab"): (0, 0x100), "_" + nid("cd"): (0, 0x108)}, exports_of(data)
offset, (vmaddr, vmsize, fileoff, filesize) = segment(data, "__LINKEDIT")
assert fileoff + filesize == len(data) and vmsize >= filesize and len(data) > len(original), (fileoff, filesize, len(data))
trie_offset, trie_size = struct.unpack_from("<II", data, command_fields(data)[LC_DYLD_EXPORTS_TRIE][0] + 8)
assert trie_offset >= len(original) and trie_offset + trie_size <= len(data) and trie_offset % 8 == 0, (trie_offset, trie_size)
def check_preserved(patcher, directory):
reference = Path(directory) / "reference.dylib"
reference.write_bytes(build_dylib([("_puts", 0, 0x100)], []))
path = Path(directory) / "preserve.dylib"
path.write_bytes(build_dylib([("_puts_nid_postfix", 0, 0x100), ("_exit_nid_postfix", 0, 0x108)], []))
result = subprocess.run([str(patcher), "libtest", "--preserve-exports", str(reference), str(path)], capture_output=True, text=True, timeout=30)
assert result.returncode == 0, (result.stdout, result.stderr)
assert exports_of(path.read_bytes()) == {"_puts_nid_postfix": (0, 0x100), "_" + nid("exit"): (0, 0x108)}, exports_of(path.read_bytes())
def check_refusals(patcher, directory):
growing = [("_ab", 0, 0x100), ("_cd", 0, 0x108)]
result, _ = patch(patcher, directory, "signed.dylib", build_dylib(growing, [], signed=True))
assert result.returncode != 0 and "signed Mach-O images cannot grow their export trie" in result.stdout + result.stderr, (result.stdout, result.stderr)
result, _ = patch(patcher, directory, "opcodes.dylib", build_dylib([("_puts_nid_postfix", 0, 0x100)], [], bind_opcodes=True))
assert result.returncode != 0 and "bind opcodes are not supported" in result.stdout + result.stderr, (result.stdout, result.stderr)
result, _ = patch(patcher, directory, "long.dylib", build_dylib([("_puts_nid_postfix", 0, 0x100)], [("_sceXy", 1)]))
assert result.returncode != 0 and "import NID longer than original name" in result.stdout + result.stderr, (result.stdout, result.stderr)
def main():
patcher = Path(sys.argv[1]).resolve()
with tempfile.TemporaryDirectory(prefix="anyps5-macho-nid-") as directory:
check_shrinking(patcher, directory)
check_growing(patcher, directory)
check_preserved(patcher, directory)
check_refusals(patcher, directory)
print("NID patcher Mach-O tests passed")
if __name__ == "__main__":
main()

2
core/libs/prx/libSceAgc/Shader/include/ShaderConstants.hpp
View File

@ -78,7 +78,7 @@ enum class PrimitiveType : std::uint32_t {
};
constexpr std::uint32_t VGT_SHADER_STAGES_GS_BIT = 0x20u;
constexpr std::uint32_t VGT_SHADER_STAGES_NGG_BIT = 0x04u;
constexpr std::uint32_t VGT_SHADER_STAGES_HS_BIT = 0x04u;
enum class AgcDirectResourceType : std::uint32_t {
GdsCounterRange = 0,

3
core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp
View File

@ -110,8 +110,7 @@ int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader*
extern "C" {
APS5_EXPORT("HV4j+E0MBHE", sceAgcCreateInterpolantMapping);
int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps) {
return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValue);
}

8
core/libs/prx/libSceAgc/Shader/src/LinkShaders.cpp
View File

@ -7,7 +7,7 @@
#include <algorithm>
extern "C" int APS5_VABI sceAgcCreatePrimState(ShaderRegister*, ShaderRegister*, const Shader*, const Shader*, std::uint32_t);
extern "C" int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister*, const Shader*, const Shader*);
extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister*, const Shader*, const Shader*);
extern "C" int APS5_VABI sceAgcLinkShaders(ShaderRegister* context, ShaderRegister* primitive,
const void* reserved, const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType) {
@ -26,7 +26,11 @@ extern "C" int APS5_VABI sceAgcLinkShaders(ShaderRegister* context, ShaderRegist
std::array<ShaderRegister, 34> contextValues{};
std::array<ShaderRegister, 3> primitiveValues{};
sceAgcCreatePrimState(contextValues.data(), primitiveValues.data(), nullptr, vertex, primitiveType);
sceAgcCreateInterpolantMapping(contextValues.data() + 2, vertex, pixel);
sceAgcCreateInterpolantMapping_0100(contextValues.data() + 2, vertex, pixel);
if (pixel != nullptr) {
const std::array<const Shader*, 2> stages{vertex, pixel};
AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, contextValues, primitiveValues);
}
AgcDriverResolveGraphicsAbi_nid_postfix(vertex, pixel, primitiveType);
transaction.Commit();
std::copy(contextValues.begin(), contextValues.end(), context);

23
core/libs/prx/libSceAgc/Shader/src/PrimState.cpp
View File

@ -25,31 +25,38 @@ int APS5_VABI sceAgcCreatePrimState(ShaderRegister* cx_regs, ShaderRegister* uc_
throw std::runtime_error(std::string(__func__) + ": invalid shader type or missing special registers");
}
(void)GraphicsPrimTypeToGsOut(prim_type);
const auto valid = [](const Shader* shader) { return shader->specials->vgt_shader_stages_en.offset == ShaderRegs::VGT_SHADER_STAGES_EN && shader->specials->vgt_gs_out_prim_type.offset == ShaderRegs::VGT_GS_OUT_PRIM_TYPE; };
if (!valid(gs) || (hs != nullptr && !valid(hs))) {
const auto validStages = [](const Shader* shader) { return shader->specials->vgt_shader_stages_en.offset == ShaderRegs::VGT_SHADER_STAGES_EN; };
if (!validStages(gs) || (hs != nullptr && !validStages(hs))) {
throw std::runtime_error(std::string(__func__) + ": invalid context register offsets");
}
if (gs->specials->ge_cntl.offset != ShaderRegs::GE_CNTL || gs->specials->ge_user_vgpr_en.offset != ShaderRegs::GE_USER_VGPR_EN || (hs != nullptr && hs->specials->ge_user_vgpr_en.offset != ShaderRegs::GE_USER_VGPR_EN)) {
throw std::runtime_error(std::string(__func__) + ": invalid user configuration register offsets");
}
std::array<ShaderRegister, 2> contextValues{gs->specials->vgt_shader_stages_en, gs->specials->vgt_gs_out_prim_type};
if ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_GS_BIT) == 0) {
contextValues[1] = {ShaderRegs::VGT_GS_OUT_PRIM_TYPE, GraphicsPrimTypeToGsOut(prim_type)};
}
if (hs != nullptr) {
contextValues[0].value |= hs->specials->vgt_shader_stages_en.value;
if ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_GS_BIT) == 0) contextValues[1] = hs->specials->vgt_gs_out_prim_type;
}
if ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_GS_BIT) != 0) {
if (gs->specials->vgt_gs_out_prim_type.offset != ShaderRegs::VGT_GS_OUT_PRIM_TYPE) throw std::runtime_error(std::string(__func__) + ": invalid context register offsets");
} else if (hs != nullptr) {
if (hs->specials->vgt_gs_out_prim_type.offset != ShaderRegs::VGT_GS_OUT_PRIM_TYPE) throw std::runtime_error(std::string(__func__) + ": invalid context register offsets");
contextValues[1] = hs->specials->vgt_gs_out_prim_type;
} else {
contextValues[1] = {ShaderRegs::VGT_GS_OUT_PRIM_TYPE, GraphicsPrimTypeToGsOut(prim_type)};
}
std::array<ShaderRegister, 3> primitiveValues{gs->specials->ge_cntl, hs != nullptr ? hs->specials->ge_user_vgpr_en : gs->specials->ge_user_vgpr_en, ShaderRegister{ShaderRegs::VGT_PRIMITIVE_TYPE, prim_type}};
const std::array<const Shader*, 2> stages{hs, gs};
AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span(stages).subspan(hs == nullptr ? 1u : 0u), contextValues, primitiveValues);
const bool patch = prim_type == static_cast<std::uint32_t>(ShaderRegs::PrimitiveType::Patch);
if (patch == ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_HS_BIT) != 0)) {
AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span(stages).subspan(hs == nullptr ? 1u : 0u), contextValues, primitiveValues);
}
if (cx_regs != nullptr) std::copy(contextValues.begin(), contextValues.end(), cx_regs);
if (uc_regs != nullptr) std::copy(primitiveValues.begin(), primitiveValues.end(), uc_regs);
return 0;
}
int APS5_VABI sceAgcUpdatePrimState(ShaderRegister* cx_regs, ShaderRegister* uc_regs, std::uint32_t prim_type) {
if (cx_regs != nullptr && (cx_regs[0].value & (ShaderRegs::VGT_SHADER_STAGES_GS_BIT | ShaderRegs::VGT_SHADER_STAGES_NGG_BIT)) == 0) {
if (cx_regs != nullptr && (cx_regs[0].value & (ShaderRegs::VGT_SHADER_STAGES_GS_BIT | ShaderRegs::VGT_SHADER_STAGES_HS_BIT)) == 0) {
cx_regs[1].value &= ~0x7u;
cx_regs[1].value |= GraphicsPrimTypeToGsOut(prim_type);
}

6
core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp
View File

@ -11,7 +11,7 @@
#include <stdexcept>
#include <span>
extern "C" int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps);
extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps);
extern "C" int APS5_VABI sceAgcUnknownCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps);
namespace {
@ -100,7 +100,7 @@ void testMapping() {
checkIdentity(split, static_cast<std::uint32_t>(inputs.size()), "unused interpolants are not the identity");
auto regular = filled();
check(sceAgcCreateInterpolantMapping(regular.data(), &gs, &ps) == 0, "mapping failed");
check(sceAgcCreateInterpolantMapping_0100(regular.data(), &gs, &ps) == 0, "mapping failed");
check(std::memcmp(regular.data(), preparedMapping.data(), sizeof(regular)) == 0, "prepared mapping differs from published registers");
check(regular[3].value == split[3].value && regular[4].value == split[4].value, "mappings differ outside the high f16 half mode");
checkIdentity(regular, static_cast<std::uint32_t>(inputs.size()), "unused interpolants are not the identity");
@ -115,7 +115,7 @@ void testRejections() {
std::array<ShaderRegister, 64> regs{};
std::memset(regs.data(), 0xcc, sizeof(regs));
const auto saved = regs;
expectFailure([&] { sceAgcCreateInterpolantMapping(regs.data(), &gs, &ps); });
expectFailure([&] { sceAgcCreateInterpolantMapping_0100(regs.data(), &gs, &ps); });
expectFailure([&] { sceAgcUnknownCreateInterpolantMapping(regs.data(), &gs, &ps); });
check(std::memcmp(regs.data(), saved.data(), sizeof(regs)) == 0, "rejected mapping wrote registers");
ps.num_input_semantics = 1;

65
core/libs/prx/libSceAgc/tests/ShaderHelpers.cpp
View File

@ -11,12 +11,15 @@
extern "C" int APS5_VABI sceAgcLinkShaders(ShaderRegister*, ShaderRegister*, const void*, const Shader*, const Shader*, std::uint32_t);
extern "C" int APS5_VABI sceAgcCreatePrimState(ShaderRegister*, ShaderRegister*, const Shader*, const Shader*, std::uint32_t);
extern "C" int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister*, const Shader*, const Shader*);
extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister*, const Shader*, const Shader*);
namespace {
std::array<ShaderRegister, 2> preparedContext;
std::array<ShaderRegister, 3> preparedPrimitive;
std::size_t stageCount = 0;
std::array<const Shader*, 2> linkedStages{};
std::size_t linkedStageCount = 0;
std::size_t linkedContextCount = 0;
unsigned preparations = 0;
unsigned mappings = 0;
unsigned links = 0;
@ -26,13 +29,39 @@ const Shader* mappedPixel = nullptr;
void Require(bool condition) {
if (!condition) throw std::runtime_error("shader helper preparation mismatch");
}
void OptionalGeometryOutputRegister(ShaderSpecialRegs& special, const Shader& vertex, const Shader& hull) {
using namespace ShaderRegs;
special.vgt_gs_out_prim_type = {};
std::array<ShaderRegister, 2> noGeometryContext{};
Require(sceAgcCreatePrimState(noGeometryContext.data(), nullptr, nullptr, &vertex, 7) == 0);
Require(noGeometryContext[0].offset == VGT_SHADER_STAGES_EN && noGeometryContext[1].offset == VGT_GS_OUT_PRIM_TYPE && noGeometryContext[1].value == static_cast<std::uint32_t>(GsOutputPrimitiveType::Rectangle2D));
bool rejectedMissingHullOutput = false;
try { static_cast<void>(sceAgcCreatePrimState(noGeometryContext.data(), nullptr, &hull, &vertex, 7)); }
catch (const std::runtime_error&) { rejectedMissingHullOutput = true; }
Require(rejectedMissingHullOutput);
special.vgt_shader_stages_en.value = VGT_SHADER_STAGES_GS_BIT;
bool rejectedMissingGeometryOutput = false;
try { static_cast<void>(sceAgcCreatePrimState(noGeometryContext.data(), nullptr, nullptr, &vertex, 7)); }
catch (const std::runtime_error&) { rejectedMissingGeometryOutput = true; }
Require(rejectedMissingGeometryOutput);
special.vgt_shader_stages_en.value = 0;
special.vgt_gs_out_prim_type = {VGT_GS_OUT_PRIM_TYPE, 2};
}
}
extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span<const Shader* const> stages, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive) {
Require(context.size() == 2 && primitive.size() == 3);
std::copy(context.begin(), context.end(), preparedContext.begin());
std::copy(primitive.begin(), primitive.end(), preparedPrimitive.begin());
stageCount = stages.size();
Require((context.size() == 2 || context.size() == 34) && primitive.size() == 3);
if (context.size() == 2) {
std::copy(context.begin(), context.end(), preparedContext.begin());
std::copy(primitive.begin(), primitive.end(), preparedPrimitive.begin());
stageCount = stages.size();
} else {
Require(stages.size() == 2);
std::copy(stages.begin(), stages.end(), linkedStages.begin());
linkedStageCount = stages.size();
linkedContextCount = context.size();
}
++preparations;
}
@ -79,7 +108,7 @@ int main() {
context.back().value = primitive.back().value = 0xdeadbeef;
const auto prepare = [&] { Require(sceAgcCreatePrimState(outputs & 1 ? context.data() : nullptr, outputs & 2 ? primitive.data() : nullptr, hs, &vertex, 7) == 0); };
if (primitiveFirst) prepare();
Require(sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, &pixel) == 0);
Require(sceAgcCreateInterpolantMapping_0100(interpolants.data(), &vertex, &pixel) == 0);
if (!primitiveFirst) prepare();
Require(preparations == 1 && mappings == 1 && links == 1 && stageCount == (hs ? 2u : 1u));
Require(preparedContext[0].offset == VGT_SHADER_STAGES_EN && preparedContext[1].offset == VGT_GS_OUT_PRIM_TYPE);
@ -91,6 +120,19 @@ int main() {
}
}
}
preparations = 0;
std::array<ShaderRegister, 2> patchContext{};
std::array<ShaderRegister, 3> patchPrimitive{};
Require(sceAgcCreatePrimState(patchContext.data(), patchPrimitive.data(), nullptr, &vertex, 9) == 0);
Require(preparations == 0 && patchContext[0].offset == VGT_SHADER_STAGES_EN && patchContext[0].value == 0 && patchPrimitive[2].offset == VGT_PRIMITIVE_TYPE && patchPrimitive[2].value == 9);
ShaderSpecialRegs hullSpecial = special;
hullSpecial.vgt_shader_stages_en.value = VGT_SHADER_STAGES_HS_BIT;
hull.specials = &hullSpecial;
Require(sceAgcCreatePrimState(patchContext.data(), patchPrimitive.data(), &hull, &vertex, 4) == 0);
Require(preparations == 0 && patchContext[0].value == VGT_SHADER_STAGES_HS_BIT && patchPrimitive[2].value == 4);
Require(sceAgcCreatePrimState(patchContext.data(), patchPrimitive.data(), &hull, &vertex, 9) == 0);
Require(preparations == 1 && stageCount == 2 && preparedContext[0].value == VGT_SHADER_STAGES_HS_BIT && preparedPrimitive[2].value == 9 && patchPrimitive[2].value == 9);
hull.specials = &special;
std::array<ShaderRegister, 34> linkedContext{};
std::array<ShaderRegister, 3> linkedPrimitive{};
for (auto& value : linkedContext) value.value = 0xdeadbeef;
@ -104,23 +146,28 @@ int main() {
for (const auto value : linkedContext) Require(value.value == 0xdeadbeef);
for (const auto value : linkedPrimitive) Require(value.value == 0xdeadbeef);
rejected = false;
try { sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, &pixel); }
try { sceAgcCreateInterpolantMapping_0100(interpolants.data(), &vertex, &pixel); }
catch (const std::runtime_error&) { rejected = true; }
Require(rejected);
for (const auto value : interpolants) Require(value.value == 0xdeadbeef);
rejectLink = false;
pixel.num_input_semantics = 33;
rejected = false;
try { sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, &pixel); }
try { sceAgcCreateInterpolantMapping_0100(interpolants.data(), &vertex, &pixel); }
catch (const std::runtime_error&) { rejected = true; }
Require(rejected);
for (const auto value : interpolants) Require(value.value == 0xdeadbeef);
pixel.num_input_semantics = 1;
preparations = mappings = links = 0;
linkedStageCount = linkedContextCount = 0;
Require(sceAgcLinkShaders(linkedContext.data(), linkedPrimitive.data(), nullptr, &vertex, &pixel, 7) == 0);
Require(preparations == 2 && linkedStageCount == 2 && linkedContextCount == linkedContext.size());
Require(linkedStages[0] == &vertex && linkedStages[1] == &pixel);
preparations = mappings = links = 0;
Require(sceAgcCreatePrimState(nullptr, nullptr, nullptr, nullptr, 7) == 0);
Require(sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, nullptr) == 0);
Require(sceAgcCreateInterpolantMapping_0100(interpolants.data(), &vertex, nullptr) == 0);
Require(preparations == 0 && mappings == 0 && links == 0);
OptionalGeometryOutputRegister(special, vertex, hull);
std::cout << "shader helper preparation tests passed\n";
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';

392
core/libs/prx/libSceAgcDriver/CMakeLists.txt
View File

@ -45,6 +45,17 @@ add_custom_command(
DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone
VERBATIM
)
set(agcTextureDetileImageShaderSpv ${agcTextureDetileGeneratedDir}/TextureDetileImage.spv)
set(agcTextureDetileImageShaderHeader ${agcTextureDetileGeneratedDir}/TextureDetileImage_spv.h)
add_custom_command(
OUTPUT "${agcTextureDetileImageShaderHeader}"
COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -DLINEAR_IMAGE=1 -o "${agcTextureDetileImageShaderSpv}" "${agcTextureDetileShaderSource}"
COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureDetileImageShaderSpv} -DOUTPUT=${agcTextureDetileImageShaderHeader}
-DSYMBOL=TEXTURE_DETILE_IMAGE_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone
VERBATIM
)
set(agcMeshArgumentsShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/MeshArguments.comp)
set(agcMeshArgumentsShaderSpv ${agcTextureDetileGeneratedDir}/MeshArguments.spv)
@ -59,6 +70,19 @@ add_custom_command(
VERBATIM
)
set(agcCmaskClearShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/CmaskClear.comp)
set(agcCmaskClearShaderSpv ${agcTextureDetileGeneratedDir}/CmaskClear.spv)
set(agcCmaskClearShaderHeader ${agcTextureDetileGeneratedDir}/CmaskClear_spv.h)
add_custom_command(
OUTPUT "${agcCmaskClearShaderHeader}"
COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -o "${agcCmaskClearShaderSpv}" "${agcCmaskClearShaderSource}"
COMMAND ${CMAKE_COMMAND} -DINPUT=${agcCmaskClearShaderSpv} -DOUTPUT=${agcCmaskClearShaderHeader}
-DSYMBOL=CMASK_CLEAR_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
DEPENDS "${agcCmaskClearShaderSource}" glslang-standalone
VERBATIM
)
set(agcSampleCounterShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/SampleCounter.comp)
set(agcSampleCounterShaderSpv ${agcTextureDetileGeneratedDir}/SampleCounter.spv)
set(agcSampleCounterShaderHeader ${agcTextureDetileGeneratedDir}/SampleCounter_spv.h)
@ -72,6 +96,19 @@ add_custom_command(
VERBATIM
)
set(agcSampleDumpsShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/SampleDumps.comp)
set(agcSampleDumpsShaderSpv ${agcTextureDetileGeneratedDir}/SampleDumps.spv)
set(agcSampleDumpsShaderHeader ${agcTextureDetileGeneratedDir}/SampleDumps_spv.h)
add_custom_command(
OUTPUT "${agcSampleDumpsShaderHeader}"
COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -o "${agcSampleDumpsShaderSpv}" "${agcSampleDumpsShaderSource}"
COMMAND ${CMAKE_COMMAND} -DINPUT=${agcSampleDumpsShaderSpv} -DOUTPUT=${agcSampleDumpsShaderHeader}
-DSYMBOL=SAMPLE_DUMPS_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
DEPENDS "${agcSampleDumpsShaderSource}" glslang-standalone
VERBATIM
)
set(agcDriverSources
${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/DisplayBuffer.cpp
@ -80,6 +117,7 @@ set(agcDriverSources
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Diagnostics.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/DataHitVerification.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/Dispatch.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/DispatchAhead.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/DispatchCache.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/DispatchCacheStatistics.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/DispatchInsert.cpp
@ -137,9 +175,11 @@ set(agcDriverSources
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ColorTargetLayout.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ColorTargetTransfer.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Pipeline.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineLibrary.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineCache.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Resources.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ParallelCompare.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderInputState.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderResources.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderAddressResources.cpp
@ -155,11 +195,15 @@ set(agcDriverSources
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureTiling.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureDetiler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Texture.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PassHazards.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/UnitShadow.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Sampler.cpp
${agcTextureDetileShaderHeader}
${agcCmaskClearShaderHeader}
${agcMeshArgumentsShaderHeader}
${agcTextureDetileImageShaderHeader}
${agcSampleCounterShaderHeader}
${agcSampleDumpsShaderHeader}
${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Event.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Query.cpp
${CMAKE_CURRENT_SOURCE_DIR}/Resource/src/Lifecycle.cpp
@ -178,6 +222,7 @@ add_library(libSceAgcDriver SHARED EXCLUDE_FROM_ALL
target_include_directories(libSceAgcDriver PRIVATE
${LIBS_INCLUDE_DIR}
${agcTextureDetileGeneratedRoot}
${CMAKE_SOURCE_DIR}/core/shader/recompiler/RdnaDecoder/include
${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include
)
@ -264,6 +309,12 @@ target_link_libraries(agc_shader_memory_tests PRIVATE shader_recompiler_agc_driv
set_target_properties(agc_shader_memory_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_shader_memory_tests)
add_test_executable(agc_shared_write_tests EXCLUDE_FROM_ALL tests/SharedWrites.cpp)
target_include_directories(agc_shared_write_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_shared_write_tests PRIVATE libSceAgcDriver libkernel libc)
set_target_properties(agc_shared_write_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_shared_write_tests)
add_test_executable(agc_write_tracking_tests EXCLUDE_FROM_ALL
tests/WriteTracking.cpp
Execution/src/GuestMemory.cpp
@ -287,6 +338,15 @@ target_link_libraries(agc_structurizer_tests PRIVATE shader_recompiler_agc_drive
set_target_properties(agc_structurizer_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_structurizer_tests)
add_test_executable(agc_fragment_barycentric_tests EXCLUDE_FROM_ALL tests/FragmentBarycentric.cpp)
target_include_directories(agc_fragment_barycentric_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
set_target_properties(agc_fragment_barycentric_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_fragment_barycentric_tests)
if(BUILD_TESTING)
set_target_properties(agc_fragment_barycentric_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_fragment_barycentric COMMAND agc_fragment_barycentric_tests)
endif()
add_test_executable(agc_subgroup_clock_tests EXCLUDE_FROM_ALL
tests/SubgroupClock.cpp
Execution/src/SubgroupClock.cpp
@ -299,6 +359,23 @@ if(BUILD_TESTING)
add_test(NAME agc_subgroup_clock COMMAND agc_subgroup_clock_tests)
endif()
add_test_executable(agc_tessellation_factor_ring_tests EXCLUDE_FROM_ALL
tests/TessellationFactorRing.cpp
State/src/Configuration.cpp
)
target_include_directories(agc_tessellation_factor_ring_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_tessellation_factor_ring_tests PRIVATE libc)
set_target_properties(agc_tessellation_factor_ring_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_tessellation_factor_ring_tests)
if(BUILD_TESTING)
set_target_properties(agc_tessellation_factor_ring_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_tessellation_factor_ring COMMAND agc_tessellation_factor_ring_tests)
if(WIN32)
set_tests_properties(agc_tessellation_factor_ring PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_spirv_module_tests EXCLUDE_FROM_ALL tests/SpirvModule.cpp)
target_link_libraries(agc_spirv_module_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_spirv_module_tests PROPERTIES CXX_EXTENSIONS OFF)
@ -316,6 +393,19 @@ if(ANYPS5_ENABLE_SPIRV_TOOLS)
configure_windows_unwind(agc_spirv_optimizer_tests)
endif()
add_test_executable(agc_float16_conversion_tests EXCLUDE_FROM_ALL tests/Float16Conversion.cpp)
target_link_libraries(agc_float16_conversion_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_float16_conversion_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_float16_conversion_tests)
if(BUILD_TESTING)
set_target_properties(agc_float16_conversion_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_float16_conversion COMMAND agc_float16_conversion_tests)
set_tests_properties(agc_float16_conversion PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1")
if(WIN32)
set_tests_properties(agc_float16_conversion PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_dpp_write_tests EXCLUDE_FROM_ALL tests/DppWrite.cpp)
target_link_libraries(agc_dpp_write_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_dpp_write_tests PROPERTIES CXX_EXTENSIONS OFF)
@ -326,6 +416,16 @@ target_link_libraries(agc_masked_select_tests PRIVATE shader_recompiler_agc_driv
set_target_properties(agc_masked_select_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_masked_select_tests)
add_test_executable(agc_constant_bit_test_tests EXCLUDE_FROM_ALL tests/ConstantBitTest.cpp)
target_link_libraries(agc_constant_bit_test_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_constant_bit_test_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_constant_bit_test_tests)
add_test_executable(agc_scalar_select_mask_tests EXCLUDE_FROM_ALL tests/ScalarSelectMask.cpp)
target_link_libraries(agc_scalar_select_mask_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_scalar_select_mask_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_scalar_select_mask_tests)
add_test_executable(agc_wave_control_decode_tests EXCLUDE_FROM_ALL tests/WaveControlDecode.cpp)
target_include_directories(agc_wave_control_decode_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_wave_control_decode_tests PRIVATE shader_recompiler_agc_driver libc)
@ -340,10 +440,41 @@ if(BUILD_TESTING)
endif()
endif()
add_test_executable(agc_denormal_flush_elimination_tests EXCLUDE_FROM_ALL tests/DenormalFlushElimination.cpp)
target_link_libraries(agc_denormal_flush_elimination_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_denormal_flush_elimination_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_denormal_flush_elimination_tests)
if(BUILD_TESTING)
set_target_properties(agc_denormal_flush_elimination_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_denormal_flush_elimination COMMAND agc_denormal_flush_elimination_tests)
if(WIN32)
set_tests_properties(agc_denormal_flush_elimination PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_vop1_control_decode_tests EXCLUDE_FROM_ALL tests/Vop1ControlDecode.cpp)
target_link_libraries(agc_vop1_control_decode_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_vop1_control_decode_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_vop1_control_decode_tests)
add_test_executable(agc_vop2_encoding_decode_tests EXCLUDE_FROM_ALL tests/Vop2EncodingDecode.cpp)
target_link_libraries(agc_vop2_encoding_decode_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_vop2_encoding_decode_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_vop2_encoding_decode_tests)
if(BUILD_TESTING)
set_target_properties(agc_vop2_encoding_decode_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_vop2_encoding_decode COMMAND agc_vop2_encoding_decode_tests)
if(WIN32)
set_tests_properties(agc_vop2_encoding_decode PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_pass_hazards_tests EXCLUDE_FROM_ALL tests/PassHazards.cpp Graphics/src/PassHazards.cpp)
target_include_directories(agc_pass_hazards_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_link_libraries(agc_pass_hazards_tests PRIVATE libc)
set_target_properties(agc_pass_hazards_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_pass_hazards_tests)
add_test_executable(agc_dcc_key_sharing_tests EXCLUDE_FROM_ALL tests/DccKeySharing.cpp)
target_include_directories(agc_dcc_key_sharing_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
@ -359,6 +490,20 @@ if(BUILD_TESTING)
endif()
endif()
add_test_executable(agc_buffer_pool_tests EXCLUDE_FROM_ALL tests/BufferPool.cpp)
target_include_directories(agc_buffer_pool_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_link_libraries(agc_buffer_pool_tests PRIVATE libSceAgcDriver libc)
set_target_properties(agc_buffer_pool_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_buffer_pool_tests)
if(BUILD_TESTING)
set_target_properties(agc_buffer_pool_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_buffer_pool COMMAND agc_buffer_pool_tests)
if(WIN32)
set_tests_properties(agc_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_display_buffer_dcc_tests EXCLUDE_FROM_ALL tests/DisplayBufferDcc.cpp)
target_include_directories(agc_display_buffer_dcc_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
target_link_libraries(agc_display_buffer_dcc_tests PRIVATE libSceAgcDriver libc)
@ -378,6 +523,11 @@ target_link_libraries(agc_validate_program_tests PRIVATE shader_recompiler_agc_d
set_target_properties(agc_validate_program_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_validate_program_tests)
add_test_executable(agc_formatted_buffer_once_tests EXCLUDE_FROM_ALL tests/FormattedBufferOnce.cpp)
target_link_libraries(agc_formatted_buffer_once_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_formatted_buffer_once_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_formatted_buffer_once_tests)
add_test_executable(agc_shader_disk_cache_tests EXCLUDE_FROM_ALL tests/ShaderDiskCache.cpp)
target_include_directories(agc_shader_disk_cache_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_shader_disk_cache_tests PRIVATE shader_recompiler_agc_driver libc)
@ -398,6 +548,12 @@ target_link_libraries(agc_shader_control_flow_tests PRIVATE shader_recompiler_ag
set_target_properties(agc_shader_control_flow_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_shader_control_flow_tests)
add_test_executable(agc_shader_mode_register_tests EXCLUDE_FROM_ALL tests/ShaderModeRegisters.cpp)
target_include_directories(agc_shader_mode_register_tests PRIVATE ${LIBS_INCLUDE_DIR})
target_link_libraries(agc_shader_mode_register_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_shader_mode_register_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_shader_mode_register_tests)
add_test_executable(agc_driver_flip_tests EXCLUDE_FROM_ALL
tests/Flip.cpp
)
@ -441,11 +597,66 @@ add_custom_command(
VERBATIM
)
foreach(agcTriangleStage IN ITEMS vert frag)
set(agcTriangleSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/Triangle.${agcTriangleStage})
set(agcTriangleSpv ${agcTestShaderGeneratedRoot}/Triangle_${agcTriangleStage}.spv)
set(agcTriangleHeader ${agcTestShaderGeneratedRoot}/Triangle_${agcTriangleStage}_spv.h)
add_custom_command(
OUTPUT "${agcTriangleHeader}"
COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTestShaderGeneratedRoot}"
COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -o "${agcTriangleSpv}" "${agcTriangleSource}"
COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTriangleSpv} -DOUTPUT=${agcTriangleHeader}
-DSYMBOL=TRIANGLE_${agcTriangleStage}_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
DEPENDS "${agcTriangleSource}" glslang-standalone
VERBATIM
)
list(APPEND agcTriangleHeaders "${agcTriangleHeader}")
endforeach()
foreach(agcCoverStage IN ITEMS vert frag)
string(TOUPPER "${agcCoverStage}" agcCoverSymbol)
set(agcCoverShaderSpv ${agcTestShaderGeneratedRoot}/Cover_${agcCoverStage}.spv)
set(agcCoverShaderHeader ${agcTestShaderGeneratedRoot}/Cover_${agcCoverStage}_spv.h)
add_custom_command(
OUTPUT "${agcCoverShaderHeader}"
COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTestShaderGeneratedRoot}"
COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -o "${agcCoverShaderSpv}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/Cover.${agcCoverStage}"
COMMAND ${CMAKE_COMMAND} -DINPUT=${agcCoverShaderSpv} -DOUTPUT=${agcCoverShaderHeader}
-DSYMBOL=COVER_${agcCoverSymbol}_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/Cover.${agcCoverStage}" glslang-standalone
VERBATIM
)
list(APPEND agcCoverShaderHeaders ${agcCoverShaderHeader})
endforeach()
add_test_executable(agc_driver_cmask_fast_clear_tests EXCLUDE_FROM_ALL
tests/CmaskFastClear.cpp
${agcCoverShaderHeaders}
)
target_include_directories(agc_driver_cmask_fast_clear_tests PRIVATE
${LIBS_INCLUDE_DIR}
${agcTestShaderGeneratedRoot}
${CMAKE_SOURCE_DIR}/core/shader/recompiler
${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include
)
target_link_libraries(agc_driver_cmask_fast_clear_tests PRIVATE
libSceAgcDriver
SDL2-static
libc
)
set_target_properties(agc_driver_cmask_fast_clear_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_driver_cmask_fast_clear_tests)
add_test_executable(agc_driver_recorder_tests EXCLUDE_FROM_ALL
tests/Recorder.cpp
tests/ResidentPresent.cpp
${agcTriangleHeaders}
${agcSampleLodShaderHeader}
${agcSampleArrayShaderHeader}
${agcCoverShaderHeaders}
)
target_include_directories(agc_driver_recorder_tests PRIVATE
@ -456,6 +667,7 @@ target_include_directories(agc_driver_recorder_tests PRIVATE
)
target_link_libraries(agc_driver_recorder_tests PRIVATE
libkernel
libSceAgcDriver
SDL2-static
libc
@ -463,6 +675,37 @@ target_link_libraries(agc_driver_recorder_tests PRIVATE
set_target_properties(agc_driver_recorder_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_driver_recorder_tests)
if(BUILD_TESTING)
set_target_properties(agc_driver_recorder_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_pipeline_cache COMMAND agc_driver_recorder_tests --pipeline-cache-only)
add_test(NAME agc_pipeline_library COMMAND agc_driver_recorder_tests --pipeline-library-only)
add_test(NAME agc_pipeline_library_off COMMAND agc_driver_recorder_tests --pipeline-library-only)
set_tests_properties(agc_pipeline_library agc_pipeline_library_off PROPERTIES SKIP_RETURN_CODE 77)
set_tests_properties(agc_pipeline_library_off PROPERTIES ENVIRONMENT "APS5_NO_GPL=1")
add_test(NAME agc_pipeline_library_fast_only COMMAND agc_driver_recorder_tests --pipeline-library-only)
set_tests_properties(agc_pipeline_library_fast_only PROPERTIES ENVIRONMENT "APS5_NO_PIPELINE_LTO=1" SKIP_RETURN_CODE 77)
if(WIN32)
set_tests_properties(agc_pipeline_library agc_pipeline_library_off agc_pipeline_library_fast_only PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
set_tests_properties(agc_pipeline_cache PROPERTIES SKIP_RETURN_CODE 77)
if(WIN32)
set_tests_properties(agc_pipeline_cache PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_unchanged_cpu_stamps COMMAND agc_driver_recorder_tests --unchanged-cpu-stamps-only)
set_tests_properties(agc_unchanged_cpu_stamps PROPERTIES SKIP_RETURN_CODE 77)
if(WIN32)
set_tests_properties(agc_unchanged_cpu_stamps PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_host_import_unmap COMMAND agc_driver_recorder_tests --host-import-unmap-only)
set_tests_properties(agc_host_import_unmap PROPERTIES SKIP_RETURN_CODE 77)
if(WIN32)
set_tests_properties(agc_host_import_unmap PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_recorder_kept_bytes_tests EXCLUDE_FROM_ALL tests/RecorderKeptBytes.cpp)
target_include_directories(agc_recorder_kept_bytes_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
@ -506,9 +749,11 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL
tests/GuestTextureResource.cpp
tests/GuestSamplerResource.cpp
tests/TextureDetiler.cpp
tests/DepthSurfaceReuse.cpp
Graphics/src/Draw.cpp
Graphics/src/DepthSurface.cpp
Graphics/src/Pipeline.cpp
Graphics/src/PipelineLibrary.cpp
Graphics/src/State.cpp
Graphics/src/ColorTargetLayout.cpp
Graphics/src/ColorTargetTransfer.cpp
@ -526,13 +771,18 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL
Graphics/src/TextureTiling.cpp
Graphics/src/TextureDetiler.cpp
Graphics/src/Texture.cpp
Graphics/src/PassHazards.cpp
Graphics/src/UnitShadow.cpp
Graphics/src/Sampler.cpp
Graphics/src/Recorder.cpp
Graphics/src/ParallelCompare.cpp
Graphics/src/ShaderInputState.cpp
${agcTextureDetileShaderHeader}
${agcCmaskClearShaderHeader}
${agcMeshArgumentsShaderHeader}
${agcTextureDetileImageShaderHeader}
${agcSampleCounterShaderHeader}
${agcSampleDumpsShaderHeader}
Execution/src/GuestMemory.cpp
Execution/src/Pm4.cpp
)
@ -609,6 +859,28 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
list(APPEND agcExecutionTests ${agcExecutionTest})
endforeach()
target_include_directories(agc_driver_dual_source_blend_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_packed_unorm_color_target_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
foreach(stage IN ITEMS vert frag)
set(agcProvokingShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/ProvokingVertex.${stage})
set(agcProvokingShaderSpv ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}.spv)
set(agcProvokingShaderHeader ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}_spv.h)
string(TOUPPER "PROVOKING_VERTEX_${stage}_SPV" agcProvokingShaderSymbol)
add_custom_command(
OUTPUT "${agcProvokingShaderHeader}"
COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTestShaderGeneratedRoot}"
COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -o "${agcProvokingShaderSpv}" "${agcProvokingShaderSource}"
COMMAND ${CMAKE_COMMAND} -DINPUT=${agcProvokingShaderSpv} -DOUTPUT=${agcProvokingShaderHeader}
-DSYMBOL=${agcProvokingShaderSymbol} -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
DEPENDS "${agcProvokingShaderSource}" glslang-standalone
VERBATIM
)
target_sources(agc_driver_provoking_vertex_tests PRIVATE ${agcProvokingShaderHeader})
endforeach()
target_include_directories(agc_driver_provoking_vertex_tests PRIVATE ${agcTestShaderGeneratedRoot})
target_include_directories(agc_driver_prepared_exports_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
target_include_directories(agc_driver_prepared_publication_tests PRIVATE $<TARGET_PROPERTY:shader_recompiler_agc_driver,INTERFACE_INCLUDE_DIRECTORIES>)
@ -621,12 +893,22 @@ target_include_directories(agc_driver_prepared_shaders_tests PRIVATE
)
if(BUILD_TESTING)
set_target_properties(agc_shader_mode_register_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_shader_mode_register COMMAND agc_shader_mode_register_tests)
set_tests_properties(agc_shader_mode_register PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 20)
if(WIN32)
set_tests_properties(agc_shader_mode_register PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test(NAME agc_shader_bindless COMMAND agc_shader_memory_tests --bindless)
set_tests_properties(agc_shader_bindless PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
add_test(NAME agc_shader_frontend_pair COMMAND agc_shader_memory_tests --frontend-pair)
add_test(NAME agc_shader_frontend_pair_off COMMAND agc_shader_memory_tests --frontend-pair)
set_tests_properties(agc_shader_frontend_pair PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
set_tests_properties(agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_FRONTEND_PAIR=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
if(WIN32)
set_tests_properties(agc_shader_bindless PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
set_tests_properties(agc_shader_bindless agc_shader_frontend_pair agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests ${agcExecutionTests})
foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_constant_bit_test_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests})
if(agcTest STREQUAL "agc_driver_constant_store_tests")
continue()
endif()
@ -639,7 +921,9 @@ if(BUILD_TESTING)
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endforeach()
set_tests_properties(agc_driver_provoking_vertex PROPERTIES TIMEOUT 30)
set_tests_properties(agc_driver_flat_store PROPERTIES RUN_SERIAL TRUE)
set_tests_properties(agc_driver_storage_cache_budget PROPERTIES TIMEOUT 30)
add_test(NAME agc_driver_sampled_cache_budget_memory COMMAND agc_driver_sampled_cache_budget_tests memory)
set_tests_properties(agc_driver_sampled_cache_budget_memory PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77)
if(WIN32)
@ -686,10 +970,14 @@ if(BUILD_TESTING)
endif()
endforeach()
endforeach()
foreach(preparedMode IN ITEMS indirect uncached debug debug_failure)
foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable)
set(preparedTest "agc_driver_prepared_shaders_${preparedMode}")
if(preparedMode STREQUAL "indirect")
add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --indirect)
elseif(preparedMode STREQUAL "deferred")
add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --deferred)
elseif(preparedMode STREQUAL "deferred_undecodable")
add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --deferred-undecodable)
elseif(preparedMode STREQUAL "debug_failure")
add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --fail-before-registration)
set_tests_properties(${preparedTest} PROPERTIES WILL_FAIL TRUE TIMEOUT 30)
@ -706,6 +994,10 @@ if(BUILD_TESTING)
set_tests_properties(${preparedTest} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endforeach()
set_tests_properties(agc_driver_prepared_shaders
agc_driver_prepared_shaders_indirect agc_driver_prepared_shaders_uncached
agc_driver_prepared_shaders_debug agc_driver_prepared_shaders_debug_failure
PROPERTIES RESOURCE_LOCK shader_failure_capture)
endif()
option(AGC_BUILD_VISUAL_TEST "Enable the standalone AGC SPIR-V visual test" OFF)
@ -737,37 +1029,7 @@ if(AGC_BUILD_VISUAL_TEST)
add_test_executable(agc_driver_visual_test EXCLUDE_FROM_ALL
tests/Visual.cpp
Execution/src/VulkanDevice.cpp
Execution/src/DisplayFormat.cpp
Execution/src/AspectFit.cpp
Execution/src/PresentationScaler.cpp
Execution/src/GuestMemory.cpp
Execution/src/BdaFeatures.cpp
Execution/src/SubgroupClock.cpp
Execution/src/ShaderDeviceProfile.cpp
Graphics/src/Draw.cpp
Graphics/src/DepthSurface.cpp
Graphics/src/ColorTargetLayout.cpp
Graphics/src/ColorTargetTransfer.cpp
Graphics/src/Pipeline.cpp
Graphics/src/PipelineCache.cpp
Graphics/src/Resources.cpp
Graphics/src/ShaderResources.cpp
Graphics/src/ShaderAddressResources.cpp
Graphics/src/GuestBufferMemory.cpp
Graphics/src/BdaResources.cpp
Graphics/src/BufferAddress.cpp
Graphics/src/ShaderValidation.cpp
Graphics/src/GuestTextureResource.cpp
Graphics/src/GuestSamplerResource.cpp
Graphics/src/TextureFormat.cpp
Graphics/src/DccMetadata.cpp
Graphics/src/TextureTiling.cpp
Graphics/src/TextureDetiler.cpp
Graphics/src/Texture.cpp
Graphics/src/UnitShadow.cpp
Graphics/src/Sampler.cpp
${agcTextureDetileShaderHeader}
${agcDriverSources}
${visualShaders}
)
@ -783,15 +1045,17 @@ if(AGC_BUILD_VISUAL_TEST)
)
target_link_libraries(agc_driver_visual_test PRIVATE
shader_recompiler_agc_driver
SDL2-static
libc
libkernel
)
set_target_properties(agc_driver_visual_test PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_driver_visual_test)
if(WIN32)
set(visualRuntimeFiles "$<TARGET_FILE:libc>")
set(visualRuntimeFiles "$<TARGET_FILE:libc>" "$<TARGET_FILE:libkernel>")
if(MINGW)
get_filename_component(visualCompilerDirectory "${CMAKE_CXX_COMPILER}" DIRECTORY)
@ -815,7 +1079,7 @@ if(AGC_BUILD_VISUAL_TEST)
COMMAND "${CMAKE_COMMAND}" -E copy_if_different
${visualRuntimeFiles}
"$<TARGET_FILE_DIR:agc_driver_visual_test>"
DEPENDS libc
DEPENDS libc libkernel
COMMAND_EXPAND_LISTS
VERBATIM
)
@ -938,6 +1202,14 @@ if(BUILD_TESTING)
set_tests_properties(vector_fused_multiply_add PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test_executable(vector_constant_factor_tininess_tests ${LIBS_INCLUDE_DIR}/tests/VectorConstantFactorTininess.cpp)
target_link_libraries(vector_constant_factor_tininess_tests PRIVATE shader_recompiler_agc_driver libc)
configure_windows_unwind(vector_constant_factor_tininess_tests)
add_test(NAME vector_constant_factor_tininess COMMAND vector_constant_factor_tininess_tests)
if(WIN32)
set_tests_properties(vector_constant_factor_tininess PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test_executable(scalar_saveexec_tests ${LIBS_INCLUDE_DIR}/tests/ScalarSaveexec.cpp)
target_link_libraries(scalar_saveexec_tests PRIVATE shader_recompiler_agc_driver libc)
configure_windows_unwind(scalar_saveexec_tests)
@ -962,6 +1234,14 @@ if(BUILD_TESTING)
set_tests_properties(scalar_bitcount64 PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test_executable(bitfield_mask_tests ${LIBS_INCLUDE_DIR}/tests/BitfieldMask.cpp)
target_link_libraries(bitfield_mask_tests PRIVATE shader_recompiler_agc_driver libc)
configure_windows_unwind(bitfield_mask_tests)
add_test(NAME bitfield_mask COMMAND bitfield_mask_tests)
if(WIN32)
set_tests_properties(bitfield_mask PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
add_test_executable(scalar_scratch_load_tests ${LIBS_INCLUDE_DIR}/tests/ScalarScratchLoad.cpp)
target_link_libraries(scalar_scratch_load_tests PRIVATE shader_recompiler_agc_driver libc)
configure_windows_unwind(scalar_scratch_load_tests)
@ -1005,6 +1285,19 @@ if(BUILD_TESTING)
endif()
endif()
add_test_executable(agc_embedded_vertex_fetch_tests EXCLUDE_FROM_ALL tests/EmbeddedVertexFetch.cpp)
target_link_libraries(agc_embedded_vertex_fetch_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_embedded_vertex_fetch_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_embedded_vertex_fetch_tests)
if(BUILD_TESTING)
set_target_properties(agc_embedded_vertex_fetch_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_embedded_vertex_fetch COMMAND agc_embedded_vertex_fetch_tests)
if(WIN32)
set_tests_properties(agc_embedded_vertex_fetch PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_rdna_truncated_modifier_tests EXCLUDE_FROM_ALL tests/RdnaTruncatedModifier.cpp)
target_link_libraries(agc_rdna_truncated_modifier_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_rdna_truncated_modifier_tests PROPERTIES CXX_EXTENSIONS OFF)
@ -1018,6 +1311,19 @@ if(BUILD_TESTING)
endif()
endif()
add_test_executable(agc_mubuf_lds_decode_tests EXCLUDE_FROM_ALL tests/MubufLdsDecode.cpp)
target_link_libraries(agc_mubuf_lds_decode_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_mubuf_lds_decode_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_mubuf_lds_decode_tests)
if(BUILD_TESTING)
set_target_properties(agc_mubuf_lds_decode_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_mubuf_lds_decode COMMAND agc_mubuf_lds_decode_tests)
if(WIN32)
set_tests_properties(agc_mubuf_lds_decode PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
add_test_executable(agc_vop1_sdwa_move_decode_tests EXCLUDE_FROM_ALL tests/Vop1SdwaMoveDecode.cpp)
target_link_libraries(agc_vop1_sdwa_move_decode_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_vop1_sdwa_move_decode_tests PROPERTIES CXX_EXTENSIONS OFF)
@ -1045,6 +1351,20 @@ if(BUILD_TESTING)
endif()
endif()
add_test_executable(agc_position_invariance_tests EXCLUDE_FROM_ALL tests/PositionInvariance.cpp)
target_link_libraries(agc_position_invariance_tests PRIVATE shader_recompiler_agc_driver libc)
set_target_properties(agc_position_invariance_tests PROPERTIES CXX_EXTENSIONS OFF)
configure_windows_unwind(agc_position_invariance_tests)
if(BUILD_TESTING)
set_target_properties(agc_position_invariance_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
add_test(NAME agc_position_invariance COMMAND agc_position_invariance_tests)
set_tests_properties(agc_position_invariance PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1")
if(WIN32)
set_tests_properties(agc_position_invariance PROPERTIES ENVIRONMENT_MODIFICATION
"PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
endif()
endif()
if(BUILD_TESTING)
add_test_executable(image_by_tests ${LIBS_INCLUDE_DIR}/tests/ImageBy.cpp)
target_link_libraries(image_by_tests PRIVATE shader_recompiler_agc_driver libc)

2
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp
View File

@ -82,6 +82,8 @@ struct DataMask {
enum class EntryOutcome { Equal, EqualData, Differing, Inaccessible, QueuedLabel, FlushingImage, PublishMoved, PendingMoved, ForgetMoved };
bool CacheableResult(const ShaderRecompiler::RecompileResult& compiled);
}
#endif

2
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/IndirectDispatch.hpp
View File

@ -5,7 +5,7 @@
namespace AgcDriver::DriverDetail {
enum IndirectPath { IndirectGpu = 0, IndirectPendingImage = 1, IndirectCopiedWrite = 2, IndirectNotImported = 3, IndirectThreadDimensions = 4, IndirectFillKernel = 5, IndirectMisaligned = 6, IndirectDisabled = 7, IndirectPaths = 8 };
enum IndirectPath { IndirectGpu = 0, IndirectPendingImage = 1, IndirectCopiedWrite = 2, IndirectNotImported = 3, IndirectThreadDimensions = 4, IndirectFillKernel = 5, IndirectMisaligned = 6, IndirectDisabled = 7, IndirectWorkgroupMemory = 8, IndirectPaths = 9 };
}

53
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp
View File

@ -60,6 +60,9 @@ public:
void ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
void ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std::span<const ShaderRegister> context, std::span<const ShaderRegister> primitive);
void ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType);
static void FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters);
static std::optional<Graphics::IndirectDrawPath> ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr<VulkanDevice>& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect);
static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request);
private:
friend class SampledReadScope;
@ -74,6 +77,10 @@ private:
static bool copySegment(Submission& submission, const std::uint32_t* guest, std::size_t words, std::size_t& budget);
static void readRegisterLists(Submission& submission);
void waitForFlipRoom(const Submission& submission);
void holdFlipBehindWorker(const Submission& submission);
bool awaitsTitle(std::uint64_t awaited) const;
void noteAwaitingTitle(std::uint32_t queue, std::uint64_t awaited);
void releaseFlipHold();
void reserveOutputs(Submission& submission);
void executeRewindTail(const Submission& stalled);
void enqueue(Submission submission);
@ -120,22 +127,20 @@ private:
static bool drawPrecheck();
std::optional<DrawVerdict> precheckDraw(const QueueState& queue, const Submission& submission, std::span<const std::uint32_t> packet, const Pm4::DrawParameters& drawParameters, std::string& rejected, bool& traceIndirect);
static std::uint32_t drawUserWord(const DrawProgram& program, std::int32_t sgpr);
std::optional<Graphics::IndirectDrawPath> classifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr<VulkanDevice>& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect);
DrawVerdict draw(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission, std::string& rejected);
void addDriverPhases(DispatchClass which, const std::array<double, DriverPhaseCount>& ms, bool hit, bool validated);
static PendingDispatchPhases& pendingDispatchPhases();
static std::chrono::steady_clock::time_point& packetStartedAt();
static PendingDrawPhases& pendingDrawPhases();
void addDrawPhases(const std::array<double, DrawDriverPhaseCount>& ms, bool drawn, std::uint64_t captures);
void noteLabelStore(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp);
bool storedSince(std::span<const std::uint32_t> packet, std::uint64_t address, std::size_t bytes, std::uint64_t received);
void noteLabelStore(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue);
bool storedSince(std::span<const std::uint32_t> packet, std::uint64_t address, std::size_t bytes, std::uint64_t received, std::uint32_t* writer = nullptr);
static void traceLabel(std::span<const std::uint32_t> packet, std::uint32_t queue);
static std::array<WriteRecord, 16384>& writeHistory();
static std::size_t& writeCursor();
static void dumpPackets(std::span<const std::uint32_t> commands, const std::uint32_t* guest = nullptr);
static void validate(const Submission& submission, const std::uint32_t* guest = nullptr);
static void reportSkip(const char* kind, const std::string& what);
static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request);
static bool matchesFillKernel(std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute);
static bool fillClearEnabled();
static bool fillClearExactOnly();
@ -177,6 +182,7 @@ private:
template<typename TVisit>
static void forEachWrittenBuffer(const ShaderRecompiler::RecompileResult& compiled, TVisit&& visit);
void noteWrittenBuffers(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled);
static void appendWrittenRanges(const ShaderRecompiler::RecompileResult& compiled, std::vector<std::pair<std::uint64_t, std::uint64_t>>& ranges);
void noteForeignWriter(std::uint64_t begin, std::uint64_t end, std::uint32_t queue);
void noteDrawWriters(std::span<const Graphics::CompiledShader> stages, std::uint32_t queue);
std::optional<WrittenBuffer> newestWriterLocked(std::uint64_t begin, std::uint64_t end) const;
@ -195,7 +201,41 @@ private:
static bool pollReapAll();
static bool pollTryEach();
static bool traceLateLabels();
void waitMemory(std::span<const std::uint32_t> packet, std::uint32_t queue, const PacketHistory& context, std::uint64_t received, bool heldAtSubmit);
void waitMemory(std::span<const std::uint32_t> packet, std::uint32_t queue, const PacketHistory& context, std::uint64_t received, bool heldAtSubmit, bool requireMemory = false);
struct PendingWait {
std::array<std::uint32_t, 8> words;
std::size_t count;
std::uint64_t received;
};
static std::vector<PendingWait>& pendingWaits();
static void notePendingWait(std::span<const std::uint32_t> packet, std::uint32_t queue, std::uint64_t received);
void landPendingWaits(std::uint32_t queue);
struct ResolvedDispatch {
std::uint64_t address = 0, key = 0, indirectArguments = 0;
std::array<std::uint32_t, 5> packet{};
std::shared_ptr<const ShaderSnapshot> registeredShader;
std::shared_ptr<const ShaderRecompiler::RecompileResult> compiledResult;
std::shared_ptr<DispatchVariant> keepVariant, attachVariant;
std::shared_ptr<ShaderMemory> shaderMemory;
std::vector<ShaderRecompiler::MemoryRegion> captured;
std::vector<std::uint32_t> liveWords;
std::shared_ptr<const ShaderRecompiler::ResourceCapture> capture;
bool dataHit = false, cached = false, validated = false;
std::chrono::steady_clock::time_point resolvedAt{};
};
struct GroupCaptureStats {
std::uint64_t runs = 0, resolved = 0, resolvedHits = 0, adopted = 0, addressMismatches = 0, keyMismatches = 0, failures = 0, stale = 0, labelOverlaps = 0, writtenBefore = 0, deviceRejects = 0;
double resolveMs = 0, resolveMaxMs = 0, ageMs = 0, ageMaxMs = 0;
std::chrono::steady_clock::time_point reported = std::chrono::steady_clock::now();
};
static std::map<std::size_t, ResolvedDispatch>& resolvedAhead();
static GroupCaptureStats& groupCaptureStats();
static std::size_t& currentPacketOffset();
static bool& resolvingAhead();
static bool& gateOpened();
void resolveGroupAhead(const Submission& submission, const QueueState& live, std::size_t from);
static void clearResolvedAhead();
static void reportGroupCapture(std::uint32_t queue);
static PollStats& pollStats();
static WaitOutcomes& waitOutcomes();
static Graphics::Recorder::LateStatistics& lateCountsSeen();
@ -302,6 +342,9 @@ private:
std::atomic<std::uint32_t> orderHolders{0};
std::atomic<std::uint32_t> runningWorkers{0};
std::atomic<std::uint64_t> queue0Awaited{0};
std::atomic<std::uint32_t> flipsAhead{0};
std::atomic<std::uint32_t> flipHolders{0};
std::atomic<bool> queue0AwaitsTitle{false};
std::atomic<std::uint64_t> evidenceReads{0};
std::atomic<std::uint64_t> evidenceValidations{0};

54
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferFill.hpp Normal file
View File

@ -0,0 +1,54 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_BUFFERFILL_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_BUFFERFILL_HPP
#include "Recompiler.hpp"
#include <algorithm>
#include <array>
#include <cstddef>
#include <cstdint>
#include <optional>
#include <span>
namespace AgcDriver::DriverDetail {
inline constexpr std::array<std::uint32_t, 18> PatternFillKernel{
0xbfa00001u, 0xd7460000u, 0x04010c0cu, 0xf4201a84u, 0xfa000000u, 0xbf8cc07fu,
0x7da8006au, 0xbf880009u, 0xf4201a84u, 0xfa000004u, 0xbf8cc07fu, 0x3602006au,
0xe0002000u, 0x80000101u, 0xbf8c3f70u, 0xe0102000u, 0x80010100u, 0xbf810000u
};
inline bool MatchesPatternFillKernel(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) {
if (userData.size() != 12 || compute.numThreads != std::array<std::uint32_t, 3>{64, 1, 1} || !compute.groupIdEnable[0] || compute.threadIdComponentCount != 1) return false;
if (code.size() < PatternFillKernel.size() || !std::equal(PatternFillKernel.begin(), PatternFillKernel.end(), code.begin())) return false;
for (const auto first : {0u, 4u, 8u}) {
if ((userData[first + 1] & 0xffff0000u) != (first == 8 ? 0x00100000u : 0x00040000u)) return false;
if (userData[first + 3] != (first == 8 ? 0x0004dfacu : 0x00014004u)) return false;
}
return userData[2] >= 1 && userData[10] >= 1;
}
struct PatternFillRange {
std::uint64_t source, destination, control, invocations;
std::uint32_t records;
[[nodiscard]] std::optional<std::size_t> UniformBytes(std::uint32_t count, std::uint32_t mask) const {
if (mask != 0) return std::nullopt;
const auto bytes = std::min({invocations, std::uint64_t{count}, std::uint64_t{records}}) * 4u;
if (destination % 16 != 0 || bytes % 16 != 0) return std::nullopt;
if (bytes != 0 && ((source < destination + bytes && destination < source + 4) || (control < destination + bytes && destination < control + 8))) return std::nullopt;
return static_cast<std::size_t>(bytes);
}
};
inline std::optional<PatternFillRange> DecodePatternFillRange(std::span<const std::uint32_t> userData, std::span<const std::uint32_t> packet) {
if (userData.size() != 12 || packet.size() < 5 || packet[2] != 1 || packet[3] != 1) return std::nullopt;
const std::uint64_t invocations = (packet[4] & 0x20u) != 0 ? packet[1] : std::uint64_t{packet[1]} * 64u;
if (invocations > (std::uint64_t{1} << 32u)) return std::nullopt;
const auto base = [&](std::size_t word) { return userData[word] | (static_cast<std::uint64_t>(userData[word + 1] & 0xffffu) << 32u); };
if (base(0) % 4 != 0 || base(8) % 4 != 0) return std::nullopt;
return PatternFillRange{base(0), base(4), base(8), invocations, userData[6]};
}
}
#endif

24
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DwordPatternFill.hpp Normal file
View File

@ -0,0 +1,24 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_MEMORY_DWORDPATTERNFILL_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_MEMORY_DWORDPATTERNFILL_HPP
#include "Recompiler.hpp"
#include <array>
#include <cstddef>
#include <cstdint>
#include <optional>
#include <span>
#include <vector>
namespace AgcDriver::DriverDetail {
struct DwordPatternFill {
std::uint64_t base = 0;
std::size_t bytes = 0;
std::array<std::uint32_t, 4> pattern{};
};
std::optional<DwordPatternFill> MatchDwordPatternFill(std::span<const std::uint32_t> packet, std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute);
}
#endif

1
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp
View File

@ -29,6 +29,7 @@ struct Submission {
std::map<std::size_t, std::vector<std::uint32_t>> registerLists;
bool suspend = false;
bool waitFree = false;
bool holdsFlip = false;
std::uint64_t received = 0;
std::vector<std::uint64_t> labelWrites;

18
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp
View File

@ -6,14 +6,24 @@
#include <array>
#include <optional>
#include <atomic>
#include <condition_variable>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
namespace ShaderRecompiler {
class ShaderPreparationContext;
}
namespace AgcDriver {
class VulkanDevice;
}
namespace AgcDriver::DriverDetail {
struct ShaderSnapshot;
@ -46,9 +56,13 @@ struct PreparedShaderState {
std::vector<RectangleProgress> rectangleProgress;
std::vector<std::weak_ptr<const ShaderSnapshot>> fragments;
bool rectangleRequested = false;
bool deferred = false;
};
struct PreparedShaders : PreparedShaderState {
std::mutex mutex;
std::condition_variable settled;
std::exception_ptr failure;
bool pending = false;
};
struct RegisteredShaderState {
@ -70,13 +84,17 @@ struct ShaderSnapshot {
std::shared_ptr<const ShaderSnapshot> ReadRawComputeShader(std::uint64_t address);
std::shared_ptr<const ShaderRecompiler::SourceHandle> PrepareShaderWithDiagnostics(const ShaderRecompiler::RecompileRequest& request, ShaderRecompiler::ShaderPreparationContext* preparation = nullptr);
std::uint64_t NullPixelProgramAddress();
ShaderSnapshot PrepareNullPixelProgram(const VulkanDevice& device);
std::optional<ShaderRecompiler::ShaderFloatMode> RegisteredFloatMode(const ShaderSnapshot& snapshot);
void PublishRegisteredShader(std::shared_ptr<ShaderRegistry>& registry, const std::shared_ptr<const ShaderSnapshot>& snapshot);
void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target);
ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapshot, std::uint64_t vertexId, std::uint64_t fragmentId);
ShaderRecompiler::RectListShaders DrawRectangle(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint64_t vertexId, std::uint64_t fragmentId, const ShaderRecompiler::SpirvTarget& target);
std::shared_ptr<const ShaderRecompiler::SourceHandle> SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request);
ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request);

2
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp
View File

@ -12,7 +12,7 @@ struct SubmissionCosts {
std::atomic<std::uint64_t> submissions{0}, validateNs{0}, copyNs{0}, dequeueNs{0}, completeNs{0}, suspendNs{0}, suspends{0}, suspendsSkipped{0}, endSubmits{0}, endSkipped{0}, notifiesSkipped{0};
};
extern std::atomic<std::uint64_t> gpuLabels, completionLabels, notImportedLabels, noOpLabels, unlockedDrains;
extern std::atomic<std::uint64_t> gpuLabels, completionLabels, noOpLabels, unlockedDrains;
extern std::atomic<std::uint64_t> labelFallbacks[5];

1
core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp
View File

@ -18,6 +18,7 @@ inline constexpr std::chrono::microseconds PollTryInterval{1000};
struct LabelStore {
std::uint64_t stamp = 0;
std::uint32_t value = 0;
std::uint32_t queue = 0xffffffffu;
};
struct WriteRecord {

14
core/libs/prx/libSceAgcDriver/Execution/include/FragmentBarycentric.hpp Normal file
View File

@ -0,0 +1,14 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_FRAGMENTBARYCENTRIC_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_FRAGMENTBARYCENTRIC_HPP
#include <vulkan/vulkan.h>
namespace AgcDriver {
inline bool FragmentShaderBarycentricUsable(VkDriverId driver, bool reported) {
return reported && driver != VK_DRIVER_ID_MOLTENVK;
}
}
#endif

1
core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp
View File

@ -75,6 +75,7 @@ bool UnchangedSinceAll(std::span<const UnchangedQuery> queries);
// UnchangedSince(range, it) holds until the next store over the range.
std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes);
std::uint64_t StoreOwnBytes(std::uint64_t address, std::size_t bytes, const std::function<void()>& store);
std::uint64_t StoreOwnBytes(std::uint64_t address, std::span<const std::byte> source);
bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generation);
// Collect epoch: within one epoch a range already collected is not walked again, CollectWrites
// returns the current generation instead. A guest write landing between two collects of the same

2
core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp
View File

@ -81,7 +81,7 @@ inline bool DrawOpcode(std::uint32_t opcode) { return opcode == 0x27 || opcode =
std::string Name(std::uint32_t header);
// A PM4 type-2 packet is a one-dword filler (command-buffer padding); type 3 and type 0 carry a
// dword count in bits 29:16. Type 1 is undefined.
inline bool FillerPacket(std::uint32_t header) { return (header >> 30u) == 2u; }
inline bool FillerPacket(std::uint32_t header) { return (header >> 30u) == 2u || header == 0xffff1000u; }
inline std::size_t PacketWords(std::uint32_t header) { return FillerPacket(header) ? 1u : static_cast<std::size_t>((header >> 16u) & 0x3fffu) + 2u; }
std::string_view UnsupportedReason(std::uint32_t header);
void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue);

2
core/libs/prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp
View File

@ -57,6 +57,8 @@ public:
return true;
}
void Print(const char* text) { Print("%s", text); }
template<typename... TArgs>
void Print(const char* format, TArgs... args) {
std::array<char, 2048> local{};

1
core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp
View File

@ -156,6 +156,7 @@ inline Registers InitialContextRegisters() {
{0x318, 0}, {0x31b, 0}, {0x31c, 0}, {0x31d, 0},
{0x390, 0}, {0x3b0, 0}, {0x3b8, 0}
};
for (std::uint32_t i = 0; i < 32; ++i) result.emplace(0x191 + i, 0);
for (std::uint32_t i = 0; i < 8; ++i) {
result.emplace(0x1e0 + i, 0x20010001);
result.emplace(0x31c + 0xfu * i, 0);

1
core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp
View File

@ -85,6 +85,7 @@ private:
};
static bool read(void* context, std::uint64_t address, std::uint32_t* value);
static bool accessible(void* context, std::uint64_t address, std::uint64_t bytes);
Page& page(std::uint64_t base);
// Regions given at construction (the registered shader's code and header), referenced as given:

3
core/libs/prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp
View File

@ -2,11 +2,12 @@
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SUBGROUPCLOCK_HPP
#include <vulkan/vulkan.h>
#include <cstdint>
#include <string_view>
namespace AgcDriver {
bool NarrowSubgroupClock(VkDriverId driver, std::string_view deviceName);
bool NarrowSubgroupClock(VkDriverId driver, std::uint32_t deviceId, std::string_view deviceName);
}

24
core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp
View File

@ -35,6 +35,7 @@ public:
VulkanDevice& operator=(const VulkanDevice&) = delete;
std::string DeviceName() const;
ShaderRecompiler::SpirvTarget Target() const;
std::optional<ShaderRecompiler::GeometryStageLimits> GeometryLimits() const;
ShaderRecompiler::SpirvTarget ComputeTarget(std::uint32_t waveSize) const;
// Distinguishes this device from every earlier one in the process (a memo keyed by the device
// cannot rely on the pointer, which a replacement may reuse).
@ -66,11 +67,9 @@ public:
// once that work completed; the batch is submitted by the queue worker's flush rules (or at once
// with APS5_LABEL_SUBMIT_NOW=1). `stamp` is the record-order stamp and `queue` the recording
// queue for the recorder's pending-label table. Returns 0 when recorded on the GPU, 5 when kept
// as a completion action behind pending write-backs and 6 when kept as one because the memory is
// not host-imported so the GPU cannot store it (both land when their batch is reaped), else why
// the CPU must write it:
// as a completion action behind pending write-backs, else why the CPU must write it:
// 1 nothing recorded (the write is already ordered), 2 write-backs pending and 3 memory not
// imported (both only with APS5_DRAIN_COMPLETION_LABELS=1), 4 unsuitable size or alignment.
// imported, 4 unsuitable size or alignment.
// `reapFirst` retires finished batches before the checks; a caller recording a group of labels
// under one lock passes it for the first label only.
int WriteLabelOnGpu(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst = true);
@ -78,6 +77,7 @@ public:
// Pending-label table lookup and open-batch overlap test for WAIT_REG_MEM (see Recorder).
std::optional<Graphics::Recorder::LabelHit> PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, Graphics::Recorder::LabelRefusal* refusal = nullptr) const;
bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const;
bool RecordedWritesSettled(std::uint64_t address, std::size_t bytes) const;
// Fills [address, address + bytes) of host-imported guest memory with a repeating 16-byte pattern,
// recorded behind the open batch; false when the range is not imported (the caller stores it).
// Recorded GPU stores over the range are ordered before the fill by its barrier; finished
@ -159,9 +159,13 @@ public:
void* Window() const;
void Resize(std::uint32_t width, std::uint32_t height);
bool Presentable() const;
bool DualSrcBlend() const;
bool PrimitiveListRestart() const;
bool SamplerFilterMinmax() const;
bool ConservativeRasterization() const;
VkShaderStageFlags SubgroupStages() const;
bool ProvokingVertexLast() const;
bool GraphicsPipelineLibraries() const;
// A presentation is a few steps so the presenter holds GuestMemory::GpuMutex only while it
// touches the queue. Presentations are slots (FlipInFlight() + 1, each with its own command
// buffer, fence, kept resident image and dump buffer): RetirePresents(keep) (no mutex) retires
@ -216,7 +220,7 @@ public:
// records. Null when nothing is prepared: the resource cache may serve the dispatch (its
// Revalidate stays under the mutex), or APS5_LOCKED_BUILD=1 keeps the whole build under it as
// before. `shader` and `snapshots` must outlive the dispatch.
std::shared_ptr<PreparedDispatch> PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span<const Graphics::GuestMemorySnapshot> snapshots);
std::shared_ptr<PreparedDispatch> PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t dispatchThreads = 0);
// APS5_PROFILE_DRAW: the parts of a PrepareDispatch in milliseconds, in the order key, find,
// precollect, presync, stage A (the driver's 'prepare:' rows).
static std::span<const double, 5> PreparePhaseMs(const PreparedDispatch& prepared);
@ -224,16 +228,6 @@ public:
// variant (design_cpu_final M4): only when the resource cache served or took the object
// (reusable, cacheable) and recipes are on (APS5_NO_DISPATCH_RECIPE=1 builds none); else null.
void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span<const Graphics::GuestMemorySnapshot> snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr<PreparedDispatch> prepared = nullptr, std::shared_ptr<const Recipe>* recipe = nullptr);
// A dispatch whose group counts are the three dwords at `arguments` in guest memory
// (DISPATCH_INDIRECT): the GPU reads them in place from the host import, ordered after everything
// recorded before, so the CPU never waits for the shader that wrote them. When the GPU could not
// see the current bytes the counts are read on the CPU instead (through the flush hook, as the
// driver resolved every indirect dispatch before) and the dispatch is recorded as a direct one:
// cpuReason 1 storage-image results were pending over them, 2 a recorded dispatch writes them
// through a copied buffer (its CPU write-back lands only when the batch is reaped) or a label
// over those dwords is pending, 3 the memory is not host-imported; 0 when recorded GPU-side. argumentReadMs is
// what that CPU read (its sync) took. The device limit on group counts is not checked GPU-side.
// Debug aid: APS5_NO_GPU_INDIRECT=1 (in the driver) keeps every indirect dispatch on the CPU path.
struct IndirectOutcome {
int cpuReason;
double argumentReadMs;

104
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp
View File

@ -5,7 +5,9 @@
#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp"
#include "Optimization/ResourceProgram.hpp"
#include <algorithm>
#include <cstdlib>
#include <stdexcept>
@ -29,6 +31,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
userData.push_back(readUserData(queue.shader, 0x240 + i));
}
auto compute = Graphics::DecodeComputeStageInfo(queue.shader, snapshot.header);
const bool resolving = resolvingAhead();
std::vector<ShaderRecompiler::MemoryRegion> memory{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}};
if (!snapshot.header.empty()) memory.push_back({snapshot.headerAddress, snapshot.header});
@ -42,7 +45,23 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
}
const auto codeOffset = static_cast<std::size_t>((address - snapshot.codeAddress) / sizeof(std::uint32_t));
std::array<std::uint32_t, 5> resolved{};
if (indirectArguments != 0 && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) {
ResolvedDispatch ahead;
bool fromAhead = false;
auto& groupStats = groupCaptureStats();
if (!resolving) {
auto& parked = resolvedAhead();
if (const auto found = parked.find(currentPacketOffset()); found != parked.end()) {
ahead = std::move(found->second);
parked.erase(found);
fromAhead = ahead.address == address;
if (!fromAhead) ++groupStats.addressMismatches;
}
}
if (fromAhead) {
resolved = ahead.packet;
packet = resolved;
indirectArguments = ahead.indirectArguments;
} else if (indirectArguments != 0 && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) {
recordQueuedLabelsBeforeRead(submission.queue);
const auto readStart = std::chrono::steady_clock::now();
@ -51,13 +70,19 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
packet = resolved;
indirectArguments = 0;
}
if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) {
pendingDispatchPhases().outcome = DispatchOutcome::FillHle;
return;
}
if (indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) {
pendingDispatchPhases().outcome = DispatchOutcome::CopyHle;
return;
if (resolving) {
if (matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return;
if (indirectArguments == 0 && matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return;
} else {
if (const auto htile = Graphics::HtileDepthClearAddress(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads); htile != 0) Graphics::NoteHtileDepthClear(htile);
if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) {
pendingDispatchPhases().outcome = DispatchOutcome::FillHle;
return;
}
if (indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) {
pendingDispatchPhases().outcome = DispatchOutcome::CopyHle;
return;
}
}
if (indirectArguments == 0 && (packet[4] & 0x20u) != 0) {
const std::array<std::uint32_t, 3> threads{packet[1], packet[2], packet[3]};
@ -180,14 +205,36 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
}
mix(reinterpret_cast<std::uintptr_t>(registeredShader.get()));
if (fromAhead && key != ahead.key) {
fromAhead = false;
++groupStats.keyMismatches;
}
phaseTiming.Phase(PhaseKey);
timing.Mark("decode_key_device_setup");
lookupDispatch(address, submission, key, noDispatchCache, traceCache, profile, memory, phaseTiming, phaseMs, compiledResult, keepVariant, captured, liveWords, dataHit, cached, validated, missedEntry, missedDiffering);
if (fromAhead) {
compiledResult = std::move(ahead.compiledResult);
keepVariant = std::move(ahead.keepVariant);
attachVariant = std::move(ahead.attachVariant);
shaderMemory = std::move(ahead.shaderMemory);
captured = std::move(ahead.captured);
liveWords = std::move(ahead.liveWords);
capture = std::move(ahead.capture);
dataHit = ahead.dataHit;
cached = ahead.cached;
validated = ahead.validated;
++groupStats.adopted;
const auto age = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - ahead.resolvedAt).count();
groupStats.ageMs += age;
groupStats.ageMaxMs = std::max(groupStats.ageMaxMs, age);
phaseTiming.Phase(PhaseLookup);
} else {
lookupDispatch(address, submission, key, noDispatchCache, traceCache, profile, memory, phaseTiming, phaseMs, compiledResult, keepVariant, captured, liveWords, dataHit, cached, validated, missedEntry, missedDiffering);
}
timing.Mark("lookup_cache");
if (cached) {
require(keepVariant != nullptr && keepVariant->shader == registeredShader, "dispatch cache belongs to another registered shader");
captureMs += phaseTiming.Elapsed();
} else {
} else if (!fromAhead) {
shaderMemory = std::make_shared<ShaderMemory>(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter());
std::uint64_t forgetAtCapture = 0;
@ -231,7 +278,30 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
}
recompileMs += phaseTiming.Elapsed();
phaseTiming.Phase(PhaseRecompile);
insertDispatch(address, key, noDispatchCache, profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming);
insertDispatch(address, key, noDispatchCache || !CacheableResult(*compiledResult), profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming);
}
if (resolving) {
auto& parked = resolvedAhead()[currentPacketOffset()];
parked = ResolvedDispatch{};
parked.address = address;
parked.key = key;
parked.indirectArguments = indirectArguments;
std::copy_n(packet.begin(), std::min(packet.size(), parked.packet.size()), parked.packet.begin());
parked.registeredShader = registeredShader;
parked.compiledResult = compiledResult;
parked.keepVariant = keepVariant;
parked.attachVariant = attachVariant;
parked.shaderMemory = shaderMemory;
parked.captured = captured;
parked.liveWords = liveWords;
parked.capture = capture;
parked.dataHit = dataHit;
parked.cached = cached;
parked.validated = validated;
parked.resolvedAt = std::chrono::steady_clock::now();
++groupStats.resolved;
if (cached) ++groupStats.resolvedHits;
return;
}
if (verifyDataHits() && dataHit) verifyDataHit(snapshot, codeOffset, request, memory, address, *keepVariant, liveWords, *compiledResult);
@ -251,6 +321,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
groups[axis] = (groups[axis] + threads - 1) / threads;
}
}
const std::uint64_t dispatchThreads = indirectArguments != 0 ? 0 : std::uint64_t{groups[0]} * groups[1] * groups[2] * compute.numThreads[0] * compute.numThreads[1] * compute.numThreads[2];
static const bool traceIo = std::getenv("APS5_TRACE_DISPATCH_IO") != nullptr;
if (traceIo) {
@ -266,6 +337,11 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
}
const auto rethrow = [&](const std::exception& error) {
if (fromAhead) {
++groupStats.deviceRejects;
dispatch(queue, packet, submission, indirectArguments);
return true;
}
char where[64];
std::snprintf(where, sizeof(where), "compute shader 0x%llx: ", static_cast<unsigned long long>(address));
throw std::runtime_error(where + std::string(error.what()));
@ -288,9 +364,9 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
std::shared_ptr<PreparedDispatch> prepared;
if (recipeHit == nullptr || VulkanDevice::VerifyRecipes()) {
try {
prepared = localDevice->PrepareDispatch(compiled, snapshots);
prepared = localDevice->PrepareDispatch(compiled, snapshots, dispatchThreads);
} catch (const std::exception& error) {
rethrow(error);
if (rethrow(error)) return;
}
if (profile) {
const auto now = std::chrono::steady_clock::now();
@ -338,7 +414,7 @@ void Driver::dispatch(QueueState& queue, std::span<const std::uint32_t> packet,
localDevice->Dispatch(compiled, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), attachTo != nullptr ? &builtRecipe : nullptr);
}
} catch (const std::exception& error) {
rethrow(error);
if (rethrow(error)) return;
}
if (builtRecipe != nullptr) {
if (dataHit) {

188
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp Normal file
View File

@ -0,0 +1,188 @@
#include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp"
#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
#include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp"
#include <algorithm>
#include <cstdlib>
#include <cstring>
namespace AgcDriver::DriverDetail {
std::map<std::size_t, Driver::ResolvedDispatch>& Driver::resolvedAhead() {
static thread_local std::map<std::size_t, ResolvedDispatch> resolved;
return resolved;
}
Driver::GroupCaptureStats& Driver::groupCaptureStats() {
static thread_local GroupCaptureStats stats;
return stats;
}
std::size_t& Driver::currentPacketOffset() {
static thread_local std::size_t offset = 0;
return offset;
}
bool& Driver::resolvingAhead() {
static thread_local bool resolving = false;
return resolving;
}
bool& Driver::gateOpened() {
static thread_local bool opened = false;
return opened;
}
void Driver::clearResolvedAhead() {
auto& resolved = resolvedAhead();
if (resolved.empty()) return;
groupCaptureStats().stale += resolved.size();
resolved.clear();
}
namespace {
struct RunLabel {
std::uint64_t address;
std::size_t size;
std::uint64_t value;
};
bool InCode(const ShaderRecompiler::MemoryRegion& region, const std::shared_ptr<const ShaderSnapshot>& code) {
if (code == nullptr) return false;
if (region.guestAddress >= code->codeAddress && region.guestAddress < code->codeAddress + code->code.size() * sizeof(std::uint32_t)) return true;
return !code->header.empty() && region.guestAddress >= code->headerAddress && region.guestAddress < code->headerAddress + code->header.size();
}
bool OverlapsRange(std::span<const ShaderRecompiler::MemoryRegion> regions, const std::vector<std::pair<std::uint64_t, std::uint64_t>>& ranges, const std::shared_ptr<const ShaderSnapshot>& code) {
for (const auto& region : regions) {
if (InCode(region, code)) continue;
for (const auto& [begin, end] : ranges) {
if (begin < region.guestAddress + region.bytes.size() && region.guestAddress < end) return true;
}
}
return false;
}
bool WrittenByRun(std::span<const std::uint32_t> wait, const std::vector<RunLabel>& labels) {
const std::uint64_t awaited = wait[2] | (static_cast<std::uint64_t>(wait[3]) << 32u);
const auto bytes = Pm4::WaitAwaitedBytes(wait);
for (auto it = labels.rbegin(); it != labels.rend(); ++it) {
if (it->address != awaited || it->size < bytes) continue;
return Pm4::WaitComparesValue(wait, bytes == 8 ? it->value : (it->value & 0xffffffffu));
}
return false;
}
void NoteLabel(std::vector<RunLabel>& labels, std::uint64_t address, std::span<const std::byte> bytes) {
if (bytes.size() != 4 && bytes.size() != 8) return;
std::uint64_t value = 0;
std::memcpy(&value, bytes.data(), bytes.size());
labels.push_back({address, bytes.size(), value});
}
bool OverlapsLabel(std::span<const ShaderRecompiler::MemoryRegion> regions, const std::vector<RunLabel>& labels) {
for (const auto& label : labels) {
for (const auto& region : regions) {
if (label.address < region.guestAddress + region.bytes.size() && region.guestAddress < label.address + label.size) return true;
}
}
return false;
}
}
void Driver::resolveGroupAhead(const Submission& submission, const QueueState& live, std::size_t from) {
static const bool allQueues = std::getenv("APS5_GROUP_CAPTURE_ALL_QUEUES") != nullptr;
if (!allQueues && submission.queue == 0) return;
auto& resolved = resolvedAhead();
if (resolved.contains(from) || device.Load() == nullptr) return;
auto& stats = groupCaptureStats();
const auto begin = std::chrono::steady_clock::now();
++stats.runs;
QueueState scratch = live;
std::vector<RunLabel> labels;
std::vector<std::pair<std::uint64_t, std::uint64_t>> writes;
for (const auto& label : deferredLabels().labels) NoteLabel(labels, label.address, std::span<const std::byte>(label.bytes).first(label.size));
struct Resolving {
std::size_t offset;
Resolving() : offset(currentPacketOffset()) { resolvingAhead() = true; }
~Resolving() {
resolvingAhead() = false;
currentPacketOffset() = offset;
}
} resolving;
const auto& commands = submission.commands;
for (std::size_t cursor = from; cursor < commands.size();) {
const auto header = commands[cursor];
if (Pm4::FillerPacket(header)) {
++cursor;
continue;
}
const auto count = Pm4::PacketWords(header);
if (cursor + count > commands.size()) break;
const auto packet = std::span(commands).subspan(cursor, count);
const auto opcode = (header >> 8u) & 0xffu;
const auto next = cursor + count;
if (header == FlipPacketHeader || header == RenderingWaitPacketHeader || opcode == 0x22 || (Pm4::Predicated(header) && scratch.predication.operation != 0)) break;
if (opcode == 0x3c || opcode == 0x93) {
if (!Pm4::WaitSatisfiedUnchecked(packet) && !WrittenByRun(packet, labels)) break;
} else if (opcode == 0x15 || opcode == 0x16) {
if (resolved.contains(cursor)) break;
currentPacketOffset() = cursor;
try {
if (opcode == 0x15) dispatch(scratch, packet, submission);
else dispatchIndirect(scratch, packet, submission);
} catch (const std::exception&) {
++stats.failures;
break;
}
if (const auto found = resolved.find(cursor); found != resolved.end()) {
if (OverlapsLabel(found->second.captured, labels)) {
resolved.erase(found);
++stats.labelOverlaps;
break;
}
if (OverlapsRange(found->second.captured, writes, found->second.registeredShader)) {
resolved.erase(found);
++stats.writtenBefore;
break;
}
if (found->second.compiledResult != nullptr) appendWrittenRanges(*found->second.compiledResult, writes);
}
} else if (opcode == 0x37 || opcode == 0x49) {
const auto label = Pm4::DecodeLabelWrite(packet);
if (!label) break;
NoteLabel(labels, label->address, label->Bytes());
} else if (opcode != 0x3f && opcode != 0x42 && opcode != 0x46 && opcode != 0x58) {
if (Pm4::AccessesMemory(header)) break;
try {
Pm4::Execute(packet, scratch);
} catch (const std::exception&) {
break;
}
}
cursor = next;
}
const auto ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - begin).count();
stats.resolveMs += ms;
stats.resolveMaxMs = std::max(stats.resolveMaxMs, ms);
reportGroupCapture(submission.queue);
}
void Driver::reportGroupCapture(std::uint32_t queue) {
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
if (!profile) return;
auto& stats = groupCaptureStats();
const auto now = std::chrono::steady_clock::now();
if (now - stats.reported < std::chrono::seconds(10)) return;
AgcDriver::ProfilePrint_nid_no_patch("[groupcap] queue 0x%x (10 s): %llu runs resolved %llu dispatches (%llu cache hits) in %.1f ms (max %.2f ms); %llu adopted %.2f ms after the resolve on average, max %.2f ms; mismatches: address %llu key %llu; %llu resolve failures, %llu stale, %llu label overlaps, %llu read an earlier dispatch's output, %llu rejected by the device\n",
queue, static_cast<unsigned long long>(stats.runs), static_cast<unsigned long long>(stats.resolved), static_cast<unsigned long long>(stats.resolvedHits), stats.resolveMs, stats.resolveMaxMs,
static_cast<unsigned long long>(stats.adopted), stats.adopted != 0 ? stats.ageMs / static_cast<double>(stats.adopted) : 0.0, stats.ageMaxMs, static_cast<unsigned long long>(stats.addressMismatches), static_cast<unsigned long long>(stats.keyMismatches),
static_cast<unsigned long long>(stats.failures), static_cast<unsigned long long>(stats.stale), static_cast<unsigned long long>(stats.labelOverlaps), static_cast<unsigned long long>(stats.writtenBefore), static_cast<unsigned long long>(stats.deviceRejects));
stats = GroupCaptureStats{};
stats.reported = now;
}
}

4
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp
View File

@ -6,6 +6,10 @@
namespace AgcDriver::DriverDetail {
bool CacheableResult(const ShaderRecompiler::RecompileResult& compiled) {
return compiled.poisonedSrtReads == 0u;
}
void Driver::insertDispatch(std::uint64_t address, std::uint64_t key, bool noDispatchCache, bool profile, const std::shared_ptr<const ShaderSnapshot>& registeredShader, std::uint64_t forgetAtCapture, std::span<const ShaderRecompiler::MemoryRegion> memory, const std::shared_ptr<ShaderMemory>& shaderMemory, const std::vector<ShaderRecompiler::MemoryRegion>& captured, const std::shared_ptr<const ShaderRecompiler::ResourceCapture>& capture, const std::shared_ptr<const ShaderRecompiler::RecompileResult>& compiledResult, const std::shared_ptr<DispatchEntry>& missedEntry, bool missedDiffering, std::shared_ptr<DispatchVariant>& attachVariant, DispatchPhaseTiming& phaseTiming) {
if (!noDispatchCache) {
auto fresh = std::make_shared<DispatchVariant>();

10
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp
View File

@ -20,7 +20,7 @@ void Driver::countIndirect(int path, double readMs) {
lastReport = now;
std::uint64_t cpu = 0;
for (int i = 1; i < IndirectPaths; ++i) cpu += counts[i];
std::fprintf(stderr, "[indirect] gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu); CPU argument reads took %.1f s\n", static_cast<unsigned long long>(counts[IndirectGpu]), static_cast<unsigned long long>(cpu), static_cast<unsigned long long>(counts[IndirectThreadDimensions]), static_cast<unsigned long long>(counts[IndirectFillKernel]), static_cast<unsigned long long>(counts[IndirectPendingImage]), static_cast<unsigned long long>(counts[IndirectCopiedWrite]), static_cast<unsigned long long>(counts[IndirectNotImported]), static_cast<unsigned long long>(counts[IndirectMisaligned]), static_cast<unsigned long long>(counts[IndirectDisabled]), readWaitedMs / 1000);
std::fprintf(stderr, "[indirect] gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu, device memory LDS %llu); CPU argument reads took %.1f s\n", static_cast<unsigned long long>(counts[IndirectGpu]), static_cast<unsigned long long>(cpu), static_cast<unsigned long long>(counts[IndirectThreadDimensions]), static_cast<unsigned long long>(counts[IndirectFillKernel]), static_cast<unsigned long long>(counts[IndirectPendingImage]), static_cast<unsigned long long>(counts[IndirectCopiedWrite]), static_cast<unsigned long long>(counts[IndirectNotImported]), static_cast<unsigned long long>(counts[IndirectMisaligned]), static_cast<unsigned long long>(counts[IndirectDisabled]), static_cast<unsigned long long>(counts[IndirectWorkgroupMemory]), readWaitedMs / 1000);
}
void Driver::dispatchIndirect(QueueState& queue, std::span<const std::uint32_t> packet, const Submission& submission) {
@ -32,6 +32,14 @@ void Driver::dispatchIndirect(QueueState& queue, std::span<const std::uint32_t>
else if ((initiator & 0x20u) != 0) path = IndirectThreadDimensions;
else if (arguments % 4 != 0) path = IndirectMisaligned;
if (path != IndirectGpu) {
if (!resolvingAhead()) {
const auto& resolved = resolvedAhead();
if (const auto found = resolved.find(currentPacketOffset()); found != resolved.end() && found->second.indirectArguments == 0 && found->second.packet[0] == 0xc0031500u) {
const auto direct = found->second.packet;
dispatch(queue, direct, submission);
return;
}
}
recordQueuedLabelsBeforeRead(submission.queue);
const auto readStart = std::chrono::steady_clock::now();
const auto direct = Pm4::ReadDispatchArguments(arguments, initiator);

27
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp
View File

@ -144,7 +144,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
ShaderMemory shaderMemory(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter());
std::vector<ShaderRecompiler::RecompileResult> results;
std::vector<Graphics::CompiledShader> stages;
results.reserve(programs.size() + (graphics.rectList ? 2u : 0u));
results.reserve(programs.size() + 2u);
stages.reserve(programs.size());
std::uint32_t pushCursorBytes = 0;
@ -179,17 +179,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
phaseTiming.Phase(DrawRowDecode);
}
static const bool indxOffsetSkipFold = std::getenv("APS5_INDX_OFFSET_SKIP_FOLD") != nullptr;
static const bool indexedOffsetFold = std::getenv("APS5_NO_INDEXED_OFFSET_FOLD") == nullptr;
const auto fold = [&](const ShaderRecompiler::RecompileResult& main, Pm4::DrawParameters& parameters) {
if (parameters.indexed && !indexedOffsetFold) return;
if (main.vertexOffsetSgpr >= 0 && (parameters.firstVertex == 0 || !indxOffsetSkipFold)) {
const auto offset = drawUserWord(programs.front(), main.vertexOffsetSgpr);
require(offset <= std::numeric_limits<std::uint32_t>::max() - parameters.firstVertex, "draw vertex offset overflow");
parameters.firstVertex += offset;
}
if (main.instanceOffsetSgpr >= 0) parameters.firstInstance = drawUserWord(programs.front(), main.instanceOffsetSgpr);
};
const auto fold = [&](const ShaderRecompiler::RecompileResult& main, Pm4::DrawParameters& parameters) { FoldDrawOffsets(main, programs.front(), parameters); };
std::optional<Graphics::IndirectDrawPath> indirectCpu;
std::vector<std::uint32_t> pushOffsets(programs.size(), 0);
@ -197,6 +187,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
for (std::size_t i = 0; i < programs.size(); ++i) {
if (roles[i] == Role::GeometryBack) continue;
const auto& program = programs[i];
pushCursorBytes = Graphics::StagePushOffset(pushCursorBytes, program.binary.stage, localDevice->GraphicsPipelineLibraries());
pushOffsets[i] = pushCursorBytes;
if (drawHit) {
@ -210,11 +201,12 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
}
const auto& result = *programResults[i];
if (i == 0 && drawParameters.indirect) {
indirectCpu = classifyIndirectDraw(result, graphics, programs.front(), localDevice, drawParameters, traceIndirect);
indirectCpu = ClassifyIndirectDraw(result, graphics, programs.front(), localDevice, drawParameters, traceIndirect);
} else if (i == 0) {
fold(result, drawParameters);
}
require(result.pushConstants.size() <= Graphics::PipelinePushConstantBytes - pushCursorBytes, "stage push constants exceed the pipeline push constant block");
const auto slotEnd = (pushCursorBytes / Graphics::PipelinePushSlotBytes + 1u) * Graphics::PipelinePushSlotBytes;
require(result.pushConstants.size() <= slotEnd - pushCursorBytes, "stage push constants exceed the pipeline push constant block");
stages.push_back({program.binary.stage, &result, result.pushConstants.empty() ? 0u : pushCursorBytes});
pushCursorBytes += static_cast<std::uint32_t>(result.pushConstants.size());
}
@ -234,7 +226,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
const auto buildRectList = [&] {
phaseTiming.Phase(DrawRowVectors);
require(programs.size() == 2 && programResults[0] != nullptr && programResults[1] != nullptr, "rect-list requires vertex and fragment programs");
auto rectangle = PreparedRectangle(*programs[0].snapshot, programResults[0]->variantId, programResults[1]->variantId);
auto rectangle = DrawRectangle(*programs[0].snapshot, programs[1].snapshot, programResults[0]->variantId, programResults[1]->variantId, localDevice->Target());
if (rectListBuilt) {
results[rectIndex] = std::move(rectangle.control);
results[rectIndex + 1] = std::move(rectangle.evaluation);
@ -250,6 +242,11 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
phaseTiming.Phase(DrawRowRectList);
};
if (graphics.rectList) buildRectList();
if (!programs.empty() && programResults.back() != nullptr && programResults.back()->barycentricEmulation.active) {
require(!graphics.rectList && graphics.stages.path == Graphics::ShaderPath::Vertex && programs.size() == 2 && programResults[0] != nullptr && stages.size() == 2, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric needs a vertex shader before it");
results.push_back(ShaderRecompiler::BuildBarycentricGeometryShader(*programResults[0], *programResults[1], localDevice->Target(), localDevice->GeometryLimits()));
stages.insert(stages.begin() + 1, Graphics::CompiledShader{Stage::Geometry, &results.back(), 0});
}
std::vector<Graphics::GuestMemorySnapshot> snapshots;
const auto snapshot = [&] {
snapshots.clear();

4
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp
View File

@ -20,7 +20,7 @@ ShaderRecompiler::RecompileResult Driver::materializeDrawStage(std::size_t i, st
program.binary,
{waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(pixel) : std::nullopt, vertexInfos[i], memory, RegisteredFloatMode(*program.snapshot)},
localDevice->Target(),
{0, 0, pushOffset, (graphics.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : Graphics::PipelinePushConstantBytes) - pushOffset},
{0, 0, pushOffset, (graphics.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : Graphics::PipelinePushSlotBytes) - pushOffset % Graphics::PipelinePushSlotBytes},
ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, graphics.stages.mesh, graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}}
};
const auto waitedBefore = traceCapSync() || profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0;
@ -88,7 +88,7 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP
std::uint64_t unstable = 0, mismatches = 0;
for (std::size_t i = 0; i < programs.size(); ++i) {
const auto& stageCapture = stageCaptures[i];
if (stageCapture.compiled == nullptr) continue;
if (stageCapture.compiled == nullptr || !CacheableResult(*stageCapture.compiled)) continue;
auto variant = std::make_shared<DispatchVariant>();
variant->compiled = stageCapture.compiled;
variant->shader = programs[i].snapshot;

6
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp
View File

@ -35,8 +35,8 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const
require(address - snapshot.codeAddress < snapshot.code.size() * sizeof(std::uint32_t), "graphics program is outside registered shader code");
require((address - snapshot.codeAddress) % sizeof(std::uint32_t) == 0, "graphics entry point is not dword aligned");
require(snapshot.type == type, "graphics program refers to an incompatible shader binary type");
Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, rsrc2);
const auto resources = nullPixel && !queue.shader.contains(rsrc2) ? 0u : ReadGraphicsRegister(queue.shader, rsrc2);
if (!nullPixel) Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, rsrc2);
const auto resources = nullPixel ? 0u : ReadGraphicsRegister(queue.shader, rsrc2);
const auto userCount = ((resources >> 1u) & 0x1fu) | (((resources >> 27u) & 1u) << 5u);
require(userCount <= 32, "graphics user SGPR count exceeds the register bank");
const auto codeOffset = static_cast<std::size_t>((address - snapshot.codeAddress) / sizeof(std::uint32_t));
@ -116,6 +116,8 @@ std::shared_ptr<DrawDecode> Driver::decodeDraw(const QueueState& queue, const Su
product->state = Graphics::DecodeState(queue);
DecodeGraphicsPrograms(*product, queue, *submission.shaders, false, true);
product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(product->state), Graphics::PixelProgramSkipped(queue));
product->pixel.targetExportPacking = Graphics::ExportPackings(product->state);
product->pixel.dualSourceBlend = product->state.dualSourceBlend;
return product;
}

2
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp
View File

@ -66,6 +66,8 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr<Vulk
mix(pixel.orderedPixelShader);
for (const auto value : pixel.targetOutputMode) mix(value);
for (const auto value : pixel.targetExportMapping) mix(value);
for (const auto value : pixel.targetExportPacking) mix(static_cast<std::uint64_t>(value));
mix(pixel.dualSourceBlend);
std::lock_guard cacheLock(drawCacheMutex);
++drawEntryCounters.lookups;
const auto found = drawCache.find(drawKey);

2
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp
View File

@ -35,8 +35,6 @@ std::optional<DrawVerdict> Driver::precheckDraw(const QueueState& queue, const S
if (!colorWrites && !queue.shader.contains(0x8)) {
const auto word = [&](std::uint32_t offset) { const auto it = queue.context.find(offset); return it == queue.context.end() ? 0u : it->second; };
if ((word(0x200) & 3u) == 0 || ((word(0x010) & 3u) == 0 && (word(0x011) & 1u) == 0)) return DrawVerdict::Nothing;
rejected = "AGC graphics: depth/stencil-only draws without a pixel shader are not implemented";
return DrawVerdict::Rejected;
}
}
if (drawParameters.indexed) {

1
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp
View File

@ -105,6 +105,7 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) {
const auto& y = b.programs[i];
if (x.binary.stage != y.binary.stage || x.binary.codeAddress != y.binary.codeAddress || x.userDataBase != y.userDataBase || x.firstUserSgpr != y.firstUserSgpr || x.userData != y.userData || x.snapshot != y.snapshot || x.codeOffset != y.codeOffset) return false;
}
if (p.targetExportPacking != q.targetExportPacking || p.dualSourceBlend != q.dualSourceBlend) return false;
return true;
}

11
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp
View File

@ -12,7 +12,16 @@ std::uint32_t Driver::drawUserWord(const DrawProgram& program, std::int32_t sgpr
return program.userData[index];
}
std::optional<Graphics::IndirectDrawPath> Driver::classifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr<VulkanDevice>& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect) {
void Driver::FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters) {
if (result.vertexOffsetSgpr >= 0) {
const auto offset = drawUserWord(program, result.vertexOffsetSgpr);
require(offset <= std::numeric_limits<std::uint32_t>::max() - parameters.firstVertex, "draw vertex offset overflow");
parameters.firstVertex += offset;
}
if (result.instanceOffsetSgpr >= 0) parameters.firstInstance = drawUserWord(program, result.instanceOffsetSgpr);
}
std::optional<Graphics::IndirectDrawPath> Driver::ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr<VulkanDevice>& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect) {
std::optional<Graphics::IndirectDrawPath> indirectCpu;
auto& indirect = *drawParameters.indirect;
using Rule = Pm4::DrawParameters::IndirectDraw::Rule;

87
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp
View File

@ -1,16 +1,22 @@
#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferFill.hpp"
#include "ThreadOwned.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DwordPatternFill.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include <algorithm>
#include <bit>
#include <cstdlib>
#include <optional>
#include <cstring>
namespace AgcDriver::DriverDetail {
bool Driver::matchesFillKernel(std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) {
static const bool enabled = std::getenv("APS5_NO_FILL_HLE") == nullptr;
static const bool patternEnabled = std::getenv("APS5_NO_PATTERN_FILL_HLE") == nullptr;
if (!enabled || userData.size() < 8 || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1) return false;
if (patternEnabled && MatchesPatternFillKernel(code, userData, compute)) return true;
static constexpr std::array<std::uint32_t, 9> fillKernel{0xd7460004u, 0x04010c08u, 0x7e000204u, 0x7e020205u, 0x7e040206u, 0x7e060207u, 0xe01c2000u, 0x80000004u, 0xbf810000u};
if (code.size() < fillKernel.size() || !std::equal(fillKernel.begin(), fillKernel.end(), code.begin())) return false;
@ -23,6 +29,34 @@ bool Driver::matchesFillKernel(std::span<const std::uint32_t> code, const std::v
return type == 0 && stride == 16 && !swizzled && !addTid && dstSel == 0xfacu && format == 0x4bu;
}
std::optional<DwordPatternFill> MatchDwordPatternFill(std::span<const std::uint32_t> packet, std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) {
static const bool enabled = std::getenv("APS5_NO_FILL_HLE") == nullptr;
if (!enabled || userData.size() != 10 || packet.size() < 5 || !compute.groupIdEnable[0] || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1) return std::nullopt;
static constexpr std::array<std::uint32_t, 69> kernel{
0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u, 0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu,
0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u, 0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u,
0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u, 0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u,
0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d860609u, 0x7d8c02f9u, 0x06068c02u, 0x87ea6a0cu, 0x50000080u,
0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u, 0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u,
0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au, 0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u,
0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au, 0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u,
0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u, 0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u,
0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u, 0xbf810000u};
if (code.size() < kernel.size() || !std::equal(kernel.begin(), kernel.end(), code.begin())) return std::nullopt;
const auto stride = (userData[1] >> 16u) & 0x3fffu;
const bool swizzled = ((userData[1] >> 31u) & 1u) != 0;
const bool addTid = ((userData[3] >> 23u) & 1u) != 0;
const auto type = userData[3] >> 30u;
const auto period = userData[9];
if (type != 0 || stride != 4 || swizzled || addTid || (period != 1u && period != 2u && period != 4u) || (packet[4] & 0x20u) != 0 || packet[2] != 1 || packet[3] != 1) return std::nullopt;
const auto dwords = std::min<std::uint64_t>({static_cast<std::uint64_t>(packet[1]) * 64u, userData[8], userData[2]});
const auto base = userData[0] | (static_cast<std::uint64_t>(userData[1] & 0xffffu) << 32u);
if (dwords == 0 || dwords % 4u != 0 || base % 16u != 0) return std::nullopt;
DwordPatternFill fill{base, static_cast<std::size_t>(dwords * 4u), {}};
for (std::uint32_t i = 0; i < 4; ++i) fill.pattern[i] = userData[4 + i % period];
return fill;
}
bool Driver::fillClearEnabled() {
static const bool enabled = std::getenv("APS5_NO_FILL_CLEAR") == nullptr;
return enabled;
@ -88,20 +122,49 @@ void Driver::fillClearCount(const Graphics::StorageTexture::FillCoverage& covera
}
bool Driver::fillBuffer(QueueState& queue, std::uint32_t queueId, std::span<const std::uint32_t> packet, std::span<const std::uint32_t> code, const std::vector<std::uint32_t>& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, const std::shared_ptr<VulkanDevice>& localDevice) {
if (!matchesFillKernel(code, userData, compute)) return false;
const auto numRecords = userData[2];
std::array<std::uint32_t, 3> groups{packet[1], packet[2], packet[3]};
if ((packet[4] & 0x20u) != 0) {
for (std::uint32_t axis = 0; axis < 3; ++axis) {
const auto threads = std::max(readRegister(queue.shader, 0x207 + axis) & 0xffffu, 1u);
groups[axis] = (groups[axis] + threads - 1) / threads;
const auto dwordFill = MatchDwordPatternFill(packet, code, userData, compute);
if (!dwordFill && !matchesFillKernel(code, userData, compute)) return false;
std::uint64_t base;
std::size_t bytes;
std::array<std::uint32_t, 4> pattern;
std::unique_lock inputLock(GuestMemory::GpuMutex(), std::defer_lock);
if (dwordFill) {
base = dwordFill->base;
bytes = dwordFill->bytes;
pattern = dwordFill->pattern;
} else if (MatchesPatternFillKernel(code, userData, compute)) {
const auto range = DecodePatternFillRange(userData, packet);
if (!range) return false;
if (range->invocations == 0) return true;
GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill);
inputLock.lock();
recordLabelsForPacket(localDevice.get(), queueId);
if (!GuestMemory::Accessible(reinterpret_cast<const void*>(range->control), 8)) return false;
std::array<std::uint32_t, 2> control;
GuestMemory::Read(range->control, std::as_writable_bytes(std::span(control)), 4);
const auto count = range->UniformBytes(control[0], control[1]);
if (!count) return false;
if (*count == 0) return true;
if (!GuestMemory::Accessible(reinterpret_cast<const void*>(range->source), 4)) return false;
GuestMemory::Read(range->source, std::as_writable_bytes(std::span(pattern).first(1)), 4);
std::fill(pattern.begin() + 1, pattern.end(), pattern[0]);
base = range->destination;
bytes = *count;
} else {
const auto numRecords = userData[2];
std::array<std::uint32_t, 3> groups{packet[1], packet[2], packet[3]};
if ((packet[4] & 0x20u) != 0) {
for (std::uint32_t axis = 0; axis < 3; ++axis) {
const auto threads = std::max(readRegister(queue.shader, 0x207 + axis) & 0xffffu, 1u);
groups[axis] = (groups[axis] + threads - 1) / threads;
}
}
if (groups[1] != 1 || groups[2] != 1) return false;
const auto records = std::min<std::uint64_t>(static_cast<std::uint64_t>(groups[0]) * 64u, numRecords);
base = userData[0] | (static_cast<std::uint64_t>(userData[1] & 0xffffu) << 32u);
bytes = static_cast<std::size_t>(records * 16u);
pattern = {userData[4], userData[5], userData[6], userData[7]};
}
if (groups[1] != 1 || groups[2] != 1) return false;
const auto records = std::min<std::uint64_t>(static_cast<std::uint64_t>(groups[0]) * 64u, numRecords);
const auto base = userData[0] | (static_cast<std::uint64_t>(userData[1] & 0xffffu) << 32u);
const auto bytes = static_cast<std::size_t>(records * 16u);
const std::array<std::uint32_t, 4> pattern{userData[4], userData[5], userData[6], userData[7]};
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
static std::atomic<std::uint64_t> fills{0}, filledBytes{0}, cpuFills{0};

4
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp
View File

@ -20,6 +20,10 @@ void Driver::forEachWrittenBuffer(const ShaderRecompiler::RecompileResult& compi
}
}
void Driver::appendWrittenRanges(const ShaderRecompiler::RecompileResult& compiled, std::vector<std::pair<std::uint64_t, std::uint64_t>>& ranges) {
forEachWrittenBuffer(compiled, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { ranges.emplace_back(begin, end); });
}
void Driver::noteWrittenBuffers(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled) {
std::lock_guard lock(writtenBuffersMutex);
const auto serial = ++writtenBufferSerial;

44
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp
View File

@ -31,6 +31,16 @@ void Driver::timed(double WorkerProfile::*bucket, TWork&& work) {
void Driver::execute(const Submission& submission) {
auto* submissionTiming = includeTimingSubmission(submission, true);
struct FlipHold {
Driver& driver;
bool held;
void Release() {
if (!held) return;
held = false;
driver.releaseFlipHold();
}
~FlipHold() { Release(); }
} flipHold{*this, submission.holdsFlip};
if (submission.suspend) {
PerformanceContext timingContext(submissionTiming);
PerformanceTimer timing("Driver.Suspend");
@ -44,8 +54,9 @@ void Driver::execute(const Submission& submission) {
const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush);
std::lock_guard gpuLock(GuestMemory::GpuMutex());
if (const auto localDevice = device.Load()) {
recordDeferredLabels(localDevice.get(), submission.queue);
const auto localDevice = device.Load();
recordDeferredLabels(localDevice.get(), submission.queue);
if (localDevice != nullptr) {
if (suspendDrain) localDevice->WaitIdle();
else localDevice->SubmitRecorded(submission.queue == 0);
}
@ -98,6 +109,11 @@ void Driver::execute(const Submission& submission) {
++packetProfile.submissions;
bumpEpoch(&EpochBumps::submissions);
clearResolvedAhead();
gateOpened() = true;
struct ClearAhead {
~ClearAhead() { clearResolvedAhead(); }
} clearAhead;
for (std::size_t cursor = 0; cursor < submission.commands.size();) {
if (packetEpoch()) bumpEpoch(&EpochBumps::packets);
CheckFailure();
@ -117,6 +133,7 @@ void Driver::execute(const Submission& submission) {
GuestMemory::SetCurrentPacket(header == FlipPacketHeader ? 0xffffu : opcode, submission.queue);
CaptureTrace::Log("packet submission=%llu queue=%x offset=%zu header=%08x words=%zu", static_cast<unsigned long long>(submission.serial), submission.queue, cursor, header, packet.size());
if (Pm4::Predicated(header) && queue.predication.operation != 0) {
landPendingWaits(submission.queue);
recordQueuedLabelsBeforeRead(submission.queue);
if (!Pm4::PredicationPasses(queue)) {
cursor = opcode == 0x3f ? submission.conditionalEnds.at(cursor) : nextCursor;
@ -211,19 +228,24 @@ void Driver::execute(const Submission& submission) {
if (APS5_ENABLE_TIMING_LOG) frame->CollectBackground();
frame->NoteFlipBatches(batchesAtFlip, unsignaledAtFlip);
CaptureTrace::Log("flip frame=%llu submission=%llu offset=%zu batch=%llu unsignaled=%llu", static_cast<unsigned long long>(frameSerial), static_cast<unsigned long long>(submission.serial), cursor, static_cast<unsigned long long>(batchesAtFlip), static_cast<unsigned long long>(unsignaledAtFlip));
flipHold.Release();
submission.flips.at(cursor)->GpuReady(frame);
} else if (opcode == 0x15) {
timed(&WorkerProfile::dispatchMs, [&] { dispatch(queue, packet, submission); });
} else if (opcode == 0x15 || opcode == 0x16) {
landPendingWaits(submission.queue);
currentPacketOffset() = cursor;
if (gateOpened()) {
gateOpened() = false;
timed(&WorkerProfile::dispatchMs, [&] { resolveGroupAhead(submission, queue, cursor); });
}
if (opcode == 0x15) timed(&WorkerProfile::dispatchMs, [&] { dispatch(queue, packet, submission); });
else timed(&WorkerProfile::dispatchMs, [&] { dispatchIndirect(queue, packet, submission); });
Graphics::Recorder::CountRecordedWork();
finishDispatchPacket(false);
} else if (opcode == 0x16) {
timed(&WorkerProfile::dispatchMs, [&] { dispatchIndirect(queue, packet, submission); });
Graphics::Recorder::CountRecordedWork();
finishDispatchPacket(true);
finishDispatchPacket(opcode == 0x16);
} else if (opcode == 0x3c || opcode == 0x93) {
static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr;
const auto waitStart = std::chrono::steady_clock::now();
timed(&WorkerProfile::waitMs, [&] { waitMemory(packet, submission.queue, recent, submission.received, submission.heldAtSubmit.contains(cursor)); });
gateOpened() = true;
if (traceGpu && std::chrono::steady_clock::now() - waitStart > std::chrono::milliseconds(200)) {
for (std::size_t next = cursor + count, shown = 0; next < submission.commands.size() && shown < 48; ++shown) {
@ -237,6 +259,7 @@ void Driver::execute(const Submission& submission) {
}
}
} else if (drawPacket) {
landPendingWaits(submission.queue);
bool drawn = false;
timed(&WorkerProfile::drawMs, [&] {
static const bool traceDraws = std::getenv("APS5_TRACE_DRAWS") != nullptr;
@ -268,6 +291,7 @@ void Driver::execute(const Submission& submission) {
});
finishDrawPacket(drawn);
} else if (opcode == 0x22) {
landPendingWaits(submission.queue);
recordQueuedLabelsBeforeRead(submission.queue);
const auto condition = Pm4::ReadCondition(packet);
if (condition == 0) nextCursor = submission.conditionalEnds.at(cursor);
@ -279,7 +303,7 @@ void Driver::execute(const Submission& submission) {
if (Pm4::IndirectRegisterOpcode(opcode)) Pm4::ExecuteIndirectRegisters(packet, submission.registerLists.at(cursor), queue);
else Pm4::Execute(packet, queue);
if (opcode == 0x49 || opcode == 0x37) {
if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial);
if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial, submission.queue);
}
}
if (endOfPipeInterrupt && !interruptDeferred) AgcDriverDeliverEopInterrupt(submission.queue);

18
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp
View File

@ -18,8 +18,8 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
bool orderedAlready = false;
const auto interruptSelect = (packet[2] >> 24u) & 7u;
endOfPipeInterrupt = opcode == 0x49 && interruptSelect != 0 && interruptSelect != 3;
const auto interruptSelect = opcode == 0x49 ? (packet[2] >> 24u) & 7u : 0u;
endOfPipeInterrupt = interruptSelect != 0 && interruptSelect != 3;
interruptDeferred = false;
if (!drainAll && endOfPipeInterrupt) {
const auto label = Pm4::DecodeLabelWrite(packet);
@ -37,8 +37,8 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
const auto stamp = ++eventSerial;
reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4;
if (reason == 1) GuestMemory::Write(label->address, bytes, 4);
if (reason == 0 || reason == 1 || reason == 5 || reason == 6) {
noteLabelStore(label->address, bytes, stamp);
if (reason == 0 || reason == 1 || reason == 5) {
noteLabelStore(label->address, bytes, stamp, submission.queue);
Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration());
}
countLabelOutcome(reason);
@ -46,7 +46,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
} else {
++noOpLabels;
}
if (reason == 0 || reason == 5 || reason == 6) {
if (reason == 0 || reason == 5) {
const auto queueId = submission.queue;
interruptDeferred = localDevice->AfterRecordedWork([queueId] { AgcDriverDeliverEopInterrupt(queueId); }, submission.queue == 0);
if (interruptDeferred && workOpen) localDevice->SubmitRecorded(submission.queue == 0);
@ -77,13 +77,13 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
recordDeferredLabels(localDevice.get(), submission.queue);
const auto stamp = ++eventSerial;
const auto reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4;
wroteOnGpu = reason == 0 || reason == 5 || reason == 6;
wroteOnGpu = reason == 0 || reason == 5;
if (reason == 1) {
GuestMemory::Write(label->address, bytes, 4);
wroteOnGpu = true;
}
if (wroteOnGpu) noteLabelStore(label->address, bytes, stamp);
if (wroteOnGpu) noteLabelStore(label->address, bytes, stamp, submission.queue);
Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration());
countLabelOutcome(reason);
++immediateLabels;
@ -117,8 +117,8 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
recordDeferredLabels(localDevice.get(), submission.queue);
const auto stamp = ++eventSerial;
const auto reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(store->address, bytes, stamp, submission.queue) : 4;
if (reason == 0 || reason == 1 || reason == 5 || reason == 6) noteLabelStore(store->address, bytes, stamp);
if (reason == 0 || reason == 5 || reason == 6) {
if (reason == 0 || reason == 1 || reason == 5) noteLabelStore(store->address, bytes, stamp, submission.queue);
if (reason == 0 || reason == 5) {
if (reason == 0) ++storesOnGpu;
else ++storesBehindCompletions;
wroteOnGpu = true;

2
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp
View File

@ -165,8 +165,8 @@ void Driver::run(std::uint32_t id) noexcept {
submission = Submission{};
} catch (...) {
const auto error = std::current_exception();
ReportFailure(error);
for (const auto& [offset, flip] : submission.flips) flip->Fail(error);
ReportFailure(error);
}
}

35
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp
View File

@ -131,6 +131,34 @@ void Driver::waitForFlipRoom(const Submission& submission) {
}
}
bool Driver::awaitsTitle(std::uint64_t awaited) const {
return runningWorkers.load(std::memory_order_acquire) == 0 && orderHolders.load(std::memory_order_acquire) == 0 && !completionsPending() && !Graphics::Recorder::SnapshotWriteOverlaps(awaited, 4);
}
void Driver::noteAwaitingTitle(std::uint32_t queue, std::uint64_t awaited) {
if (queue != 0 || flipHolders.load(std::memory_order_acquire) == 0 || queue0AwaitsTitle.load(std::memory_order_acquire) || !awaitsTitle(awaited)) return;
std::lock_guard lock(mutex);
queue0AwaitsTitle.store(true, std::memory_order_release);
changed.notify_all();
}
void Driver::holdFlipBehindWorker(const Submission& submission) {
if (submission.queue != 0 || onWorkerThread()) return;
bool flips = false;
for (std::size_t cursor = 0; cursor < submission.commands.size() && !flips; cursor += Pm4::PacketWords(submission.commands[cursor])) flips = submission.commands[cursor] == FlipPacketHeader;
if (!flips) return;
std::unique_lock lock(mutex);
flipHolders.fetch_add(1, std::memory_order_acq_rel);
changed.wait(lock, [&] { return failure != nullptr || stopping.load(std::memory_order_acquire) || shutdownToken.stop_requested() || flipsAhead.load(std::memory_order_acquire) == 0 || queue0AwaitsTitle.load(std::memory_order_acquire); });
flipHolders.fetch_sub(1, std::memory_order_acq_rel);
}
void Driver::releaseFlipHold() {
std::lock_guard lock(mutex);
flipsAhead.fetch_sub(1, std::memory_order_acq_rel);
if (flipHolders.load(std::memory_order_acquire) != 0) changed.notify_all();
}
void Driver::reserveOutputs(Submission& submission) {
for (std::size_t cursor = 0; cursor < submission.commands.size();) {
const auto* words = submission.commands.data() + cursor;
@ -167,6 +195,7 @@ void Driver::executeRewindTail(const Submission& stalled) {
while ((control.load(std::memory_order_acquire) & 0x80000000u) == 0) {
CheckFailure();
checkStopping();
noteAwaitingTitle(stalled.queue, reinterpret_cast<std::uint64_t>(stalled.rewindTail - 1));
PollSleep();
}
}
@ -218,6 +247,7 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) {
readRegisterLists(submission);
if (APS5_ENABLE_TIMING_LOG) submission.validatedAt = std::chrono::steady_clock::now();
waitForFlipRoom(submission);
holdFlipBehindWorker(submission);
if (APS5_ENABLE_TIMING_LOG) submission.roomReadyAt = std::chrono::steady_clock::now();
static const bool trace = std::getenv("APS5_TRACE_GPU") != nullptr;
if (trace) std::fprintf(stderr, "[gpu] %.1f submit queue=0x%x dwords=%zu at %p\n", TraceMs(), queue, submission.commands.size(), static_cast<const void*>(descriptor.addr));
@ -240,6 +270,10 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) {
costs.copyNs += static_cast<std::uint64_t>(std::chrono::duration_cast<std::chrono::nanoseconds>((copied - start) + (now - validated)).count());
}
if (APS5_ENABLE_TIMING_LOG) submission.enqueuedAt = std::chrono::steady_clock::now();
if (queue == 0 && !submission.flips.empty()) {
submission.holdsFlip = true;
flipsAhead.fetch_add(1, std::memory_order_acq_rel);
}
enqueue(std::move(submission));
++accepted;
}
@ -299,6 +333,7 @@ void Driver::noteWaitBlocked(std::uint32_t queue, std::uint64_t awaited, bool bl
if (blocked) runningWorkers.fetch_sub(1, std::memory_order_acq_rel);
else runningWorkers.fetch_add(1, std::memory_order_acq_rel);
if (queue == 0) queue0Awaited.store(blocked ? awaited : 0, std::memory_order_release);
if (queue == 0 && !blocked) queue0AwaitsTitle.store(false, std::memory_order_release);
if (blocked && orderHolders.load(std::memory_order_acquire) != 0) {
std::lock_guard lock(mutex);
changed.notify_all();

358
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp
View File

@ -1,6 +1,5 @@
#include "prx/libc/include/general/LogMacros.hpp"
#include "ControlFlow/GraphBuilder.hpp"
#include "ControlFlow/Structurizer.hpp"
#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
@ -11,14 +10,33 @@
#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp"
#include "CompiledVariant.hpp"
#include "Optimization/ResourceProgram.hpp"
#include <algorithm>
#include <condition_variable>
#include <cstdio>
#include <cstddef>
#include <cstdlib>
#include <cstring>
#include <deque>
#include <functional>
#include <list>
#include <pthread.h>
#include <stdexcept>
#include <thread>
#include "RdnaDecoder/RdnaInstructionDecoder.hpp"
namespace AgcDriver::DriverDetail {
std::shared_ptr<const ShaderRecompiler::SourceHandle> PrepareShaderWithDiagnostics(const ShaderRecompiler::RecompileRequest& request, ShaderRecompiler::ShaderPreparationContext* preparation) {
try {
return ShaderRecompiler::PrepareShader(request, preparation);
} catch (...) {
try {
if (std::getenv("APS5_DUMP_SHADERS") != nullptr) static_cast<void>(Driver::dumpRequest(request.shader.codeAddress, request));
} catch (...) {}
throw;
}
}
std::shared_ptr<const ShaderSnapshot> ReadRawComputeShader(std::uint64_t address) {
GuestMemory::CheckRange(reinterpret_cast<const void*>(address), sizeof(std::uint32_t), 256);
static std::mutex cacheMutex;
@ -85,11 +103,22 @@ std::shared_ptr<const ShaderSnapshot> ReadRawComputeShader(std::uint64_t address
throw std::runtime_error("AGC driver: raw compute program has no reachable end within mapped code or the size limit");
}
namespace {
void AwaitRegisteredPreparation(PreparedShaders& prepared, std::unique_lock<std::mutex>& lock) {
prepared.settled.wait(lock, [&] { return !prepared.pending; });
if (prepared.failure != nullptr) std::rethrow_exception(prepared.failure);
}
}
struct ShaderPreparationTransaction::State {
struct Change {
std::shared_ptr<PreparedShaders> destination;
PreparedShaderState prepared;
bool edited = false;
std::size_t baseEntries = 0;
std::size_t baseAbis = 0;
};
static std::mutex writers;
static thread_local State* active;
@ -118,15 +147,23 @@ PreparedShaderState& ShaderPreparationTransaction::Edit(const ShaderSnapshot& sn
require(!committed, "shader preparation transaction is already committed");
auto& changes = root->changes;
const auto found = changes.find(snapshot.prepared.get());
if (found != changes.end()) return found->second.prepared;
std::lock_guard lock(snapshot.prepared->mutex);
return changes.emplace(snapshot.prepared.get(), State::Change{snapshot.prepared, *snapshot.prepared}).first->second.prepared;
if (found != changes.end()) {
found->second.edited = true;
return found->second.prepared;
}
std::unique_lock lock(snapshot.prepared->mutex);
AwaitRegisteredPreparation(*snapshot.prepared, lock);
return changes.emplace(snapshot.prepared.get(), State::Change{snapshot.prepared, *snapshot.prepared, true, snapshot.prepared->entries.size(), snapshot.prepared->registeredAbis.size()}).first->second.prepared;
}
const PreparedShaderState& ShaderPreparationTransaction::Read(const ShaderSnapshot& snapshot) const {
require(!committed, "shader preparation transaction is already committed");
const auto found = root->changes.find(snapshot.prepared.get());
return found != root->changes.end() ? found->second.prepared : *snapshot.prepared;
auto& changes = root->changes;
const auto found = changes.find(snapshot.prepared.get());
if (found != changes.end()) return found->second.prepared;
std::unique_lock lock(snapshot.prepared->mutex);
AwaitRegisteredPreparation(*snapshot.prepared, lock);
return changes.emplace(snapshot.prepared.get(), State::Change{snapshot.prepared, *snapshot.prepared, false, snapshot.prepared->entries.size(), snapshot.prepared->registeredAbis.size()}).first->second.prepared;
}
void ShaderPreparationTransaction::Commit() {
@ -136,7 +173,16 @@ void ShaderPreparationTransaction::Commit() {
locks.reserve(state->changes.size());
for (const auto& [key, change] : state->changes) locks.emplace_back(change.destination->mutex);
for (auto& [key, change] : state->changes) {
if (!change.edited) continue;
auto& destination = *change.destination;
for (auto index = std::min(change.baseEntries, destination.entries.size()); index < destination.entries.size(); ++index) {
const auto& entry = destination.entries[index];
if (std::ranges::none_of(change.prepared.entries, [&](const auto& existing) { return existing.codeOffset == entry.codeOffset && existing.handle == entry.handle; })) change.prepared.entries.push_back(entry);
}
for (auto index = std::min(change.baseAbis, destination.registeredAbis.size()); index < destination.registeredAbis.size(); ++index) {
const auto& abi = destination.registeredAbis[index];
if (std::ranges::find(change.prepared.registeredAbis, abi) == change.prepared.registeredAbis.end()) change.prepared.registeredAbis.push_back(abi);
}
destination.entries.swap(change.prepared.entries);
destination.registeredAbis.swap(change.prepared.registeredAbis);
destination.graphicsAbis.swap(change.prepared.graphicsAbis);
@ -149,6 +195,18 @@ void ShaderPreparationTransaction::Commit() {
committed = true;
}
bool SameHeader(const ShaderSnapshot& snapshot, const Shader* shader) {
if (snapshot.headerAddress == reinterpret_cast<std::uintptr_t>(shader)) return true;
if (snapshot.header.size() < sizeof(Shader)) return false;
GuestMemory::CheckRange(shader, sizeof(Shader), 1);
Shader used;
Shader registered;
std::memcpy(&used, static_cast<const void*>(shader), sizeof(Shader));
std::memcpy(&registered, snapshot.header.data(), sizeof(Shader));
used.user_data = registered.user_data;
return std::memcmp(&used, &registered, offsetof(Shader, num_sh_registers) + sizeof(Shader::num_sh_registers)) == 0;
}
void PublishRegisteredShader(std::shared_ptr<ShaderRegistry>& registry, const std::shared_ptr<const ShaderSnapshot>& snapshot) {
ShaderPreparationTransaction transaction;
if (registry != nullptr) {
@ -167,18 +225,32 @@ void PublishRegisteredShader(std::shared_ptr<ShaderRegistry>& registry, const st
transaction.Commit();
}
namespace {
bool PreparedAtUse(const ShaderSnapshot& snapshot, const ShaderRecompiler::RecompileRequest& request) {
if (snapshot.codeAddress == NullPixelProgramAddress() && request.shader.stage == ShaderRecompiler::ShaderStage::Fragment && request.context.waveSize == 32u) {
APS5_LOG_ERR("The null pixel program has no wave%u artifact for this draw; preparing it at draw", request.context.waveSize);
return true;
}
if (!snapshot.header.empty()) return snapshot.prepared->deferred;
if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader");
APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast<unsigned long long>(snapshot.codeAddress));
return true;
}
}
std::shared_ptr<const ShaderRecompiler::SourceHandle> SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request) {
struct PreparedKeyStorage {};
auto& key = HostThreadLocal<std::vector<std::uint64_t>, PreparedKeyStorage>();
ShaderRecompiler::BuildPreparedShaderKey(request, key);
std::lock_guard lock(snapshot.prepared->mutex);
std::unique_lock lock(snapshot.prepared->mutex);
AwaitRegisteredPreparation(*snapshot.prepared, lock);
for (const auto& entry : snapshot.prepared->entries) {
if (entry.codeOffset == codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) return entry.handle;
}
if (snapshot.header.empty()) {
if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader");
APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast<unsigned long long>(snapshot.codeAddress));
auto handle = ShaderRecompiler::PrepareShader(request);
if (PreparedAtUse(snapshot, request)) {
auto handle = PrepareShaderWithDiagnostics(request);
snapshot.prepared->entries.push_back({codeOffset, handle});
return handle;
}
@ -202,6 +274,17 @@ ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapsh
throw std::runtime_error("AGC driver: prepared rectangle artifacts are missing");
}
ShaderRecompiler::RectListShaders DrawRectangle(const ShaderSnapshot& front, const std::shared_ptr<const ShaderSnapshot>& fragment, std::uint64_t vertexId, std::uint64_t fragmentId, const ShaderRecompiler::SpirvTarget& target) {
const auto prepared = [&] {
std::lock_guard lock(front.prepared->mutex);
return std::ranges::any_of(front.prepared->rectangles, [&](const auto& entry) { return entry.vertexId == vertexId && entry.fragmentId == fragmentId; });
};
if (prepared()) return PreparedRectangle(front, vertexId, fragmentId);
APS5_LOG_ERR("Rect-list draw of shader 0x%llx has no rectangle for fragment shader 0x%llx; preparing it at draw", static_cast<unsigned long long>(front.codeAddress), static_cast<unsigned long long>(fragment->codeAddress));
ResolvePreparedGraphics(front, fragment, 17, target);
return PreparedRectangle(front, vertexId, fragmentId);
}
ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request) {
require(codeOffset < snapshot.code.size(), "prepared shader code offset is outside the snapshot");
const auto code = std::span(snapshot.code).subspan(codeOffset);
@ -210,16 +293,15 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s
struct PreparedKeyStorage {};
auto& key = HostThreadLocal<std::vector<std::uint64_t>, PreparedKeyStorage>();
ShaderRecompiler::BuildPreparedShaderKey(request, key);
std::lock_guard lock(snapshot.prepared->mutex);
std::unique_lock lock(snapshot.prepared->mutex);
AwaitRegisteredPreparation(*snapshot.prepared, lock);
for (const auto& entry : snapshot.prepared->entries) {
if (entry.codeOffset != codeOffset) continue;
invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle);
if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) return std::move(*invocation);
}
if (snapshot.header.empty()) {
if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader");
APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast<unsigned long long>(snapshot.codeAddress));
auto handle = ShaderRecompiler::PrepareShader(request);
if (PreparedAtUse(snapshot, request)) {
auto handle = PrepareShaderWithDiagnostics(request);
invocationRequest = request;
invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*handle);
auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, handle, key);
@ -284,7 +366,7 @@ RegisteredShaderState DecodeRegisteredState(const ShaderSnapshot& snapshot) {
if (header.specials != nullptr) {
const auto special = ReadHeaderArray(snapshot, header.specials, 1).front();
state.context[special.vgt_shader_stages_en.offset] = special.vgt_shader_stages_en.value;
state.context[special.vgt_gs_out_prim_type.offset] = special.vgt_gs_out_prim_type.value;
if (special.vgt_gs_out_prim_type.offset != 0) state.context[special.vgt_gs_out_prim_type.offset] = special.vgt_gs_out_prim_type.value;
state.userConfig[special.ge_cntl.offset] = special.ge_cntl.value;
state.userConfig[special.ge_user_vgpr_en.offset] = special.ge_user_vgpr_en.value;
}
@ -307,7 +389,17 @@ void BuildRegisteredAbiKey(const QueueState& state, const VulkanDevice& device,
}
}
std::vector<PreparedShaders::Entry> PrepareRegistered(const ShaderSnapshot& snapshot, const VulkanDevice& device, const QueueState& state, bool registration) {
struct RegisteredPreparation {
std::size_t codeOffset = 0;
ShaderRecompiler::RdnaProgram decoded;
ShaderRecompiler::SwappcInfo swappc{};
std::vector<std::uint32_t> userData;
std::array<ShaderRecompiler::MemoryRegion, 2> memory{};
std::optional<ShaderRecompiler::RecompileRequest> request;
std::vector<std::uint64_t> abiKey;
};
std::unique_ptr<RegisteredPreparation> PlanRegistered(const ShaderSnapshot& snapshot, const VulkanDevice& device, const QueueState& state, bool registration, bool* deferred = nullptr) {
using Stage = ShaderRecompiler::ShaderStage;
const auto header = ReadHeader(snapshot);
std::uint32_t programRegister;
@ -322,15 +414,33 @@ std::vector<PreparedShaders::Entry> PrepareRegistered(const ShaderSnapshot& snap
case 5: stage = Stage::Local; programRegister = 0x148; resourceRegister = 0x10b; firstUser = 8; break;
case 6: stage = Stage::Mesh; programRegister = 0x088; resourceRegister = 0x08b; break;
case 7: stage = Stage::TessellationControl; programRegister = 0x108; resourceRegister = 0x10b; break;
default: throw std::runtime_error("AGC driver: unsupported registered shader type");
default:
if (registration) {
APS5_LOG_ERR("Shader 0x%llx has unsupported registered shader type %u; it is not prepared at registration", static_cast<unsigned long long>(snapshot.codeAddress), static_cast<unsigned>(snapshot.type));
return nullptr;
}
throw std::runtime_error("AGC driver: unsupported registered shader type " + std::to_string(snapshot.type));
}
const auto defer = [&](const char* reason) {
APS5_LOG_ERR("Shader 0x%llx could not be prepared at registration (%s); preparing it when it is used", static_cast<unsigned long long>(snapshot.codeAddress), reason);
*deferred = true;
return std::unique_ptr<RegisteredPreparation>{};
};
if (deferred != nullptr && (!state.shader.contains(programRegister) || !state.shader.contains(programRegister + 1))) return defer("its program address is not in the shader registers");
const auto high = RegisterValue(state.shader, programRegister + 1);
if ((high & ~0xffu) != 0) throw std::runtime_error("AGC driver: invalid registered program address");
const auto address = (static_cast<std::uint64_t>(RegisterValue(state.shader, programRegister)) << 8u) | (static_cast<std::uint64_t>(high) << 40u);
if (address < snapshot.codeAddress || address - snapshot.codeAddress >= snapshot.code.size() * 4u) throw std::runtime_error("AGC driver: registered entry point is outside shader code");
const auto codeOffset = static_cast<std::size_t>((address - snapshot.codeAddress) / 4u);
const auto code = std::span(snapshot.code).subspan(codeOffset);
const auto decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code);
auto plan = std::make_unique<RegisteredPreparation>();
plan->codeOffset = codeOffset;
try {
plan->decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code);
} catch (const ShaderRecompiler::UnsupportedInstructionError& error) {
if (deferred == nullptr) throw;
return defer(error.what());
}
std::optional<ShaderRecompiler::ShaderComputeStageInfo> compute;
std::optional<ShaderRecompiler::ShaderPixelStageInfo> pixel;
std::optional<ShaderRecompiler::ShaderVertexStageInfo> vertex;
@ -342,18 +452,22 @@ std::vector<PreparedShaders::Entry> PrepareRegistered(const ShaderSnapshot& snap
if (userCount > 32) throw std::runtime_error("AGC driver: static user SGPR count exceeds the register bank");
if (stage == Stage::Compute) {
compute = Graphics::DecodeComputeStageInfo(state.shader, snapshot.header);
if (header.specials == nullptr) {
if (deferred != nullptr) return defer("its header has no specials to give its wave size");
return {};
}
const auto special = ReadHeaderArray(snapshot, header.specials, 1).front();
wave = (special.dispatch_modifier & 0x8000u) != 0 ? 32u : 64u;
} else if (stage == Stage::Fragment) {
const auto count = RegisterValue(state.context, 0x1b6) & 0x3fu;
if (registration && count != 0) return {};
if (registration && count != 0) return nullptr;
pixel = Graphics::DecodePixelStageInfo(state.context, {});
wave = pixel->wave32 ? 32u : 64u;
} else {
const auto routing = RegisterValue(state.context, 0x2d5);
wave = (routing & 0x00400000u) != 0 ? 32u : 64u;
if ((routing & 0x20u) != 0 || stage == Stage::Mesh || (routing & 4u) != 0) {
if (registration) return {};
if (registration) return nullptr;
auto stageState = state;
stageState.context[0x1b6] = 0;
const auto stages = Graphics::DecodeShaderStages(stageState);
@ -363,30 +477,133 @@ std::vector<PreparedShaders::Entry> PrepareRegistered(const ShaderSnapshot& snap
}
if (firstUser == 0) userCount += 8;
}
std::vector<std::uint32_t> userData(userCount);
if (stage != Stage::Compute && stage != Stage::Fragment && snapshot.type != 6) vertex = Graphics::DecodeVertexStageInfo(snapshot.header, snapshot.headerAddress, userData, nullptr, true);
const ShaderRecompiler::SwappcInfo swappc{vertex.has_value(), firstUser, userCount};
auto graph = ShaderRecompiler::GraphBuilder{}.Build(decoded, &swappc);
ShaderRecompiler::Structurizer{}.Structurize(graph);
const std::array<ShaderRecompiler::MemoryRegion, 2> memory{{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}};
ShaderRecompiler::RecompileRequest request{{stage, address, code, snapshot.headerAddress, snapshot.header}, {wave, firstUser, userData, compute, pixel, vertex, memory, RegisteredFloatMode(snapshot)}, stage == Stage::Compute ? device.ComputeTarget(wave) : device.Target(), {0, 0, 0, 128}, graphics};
plan->userData.resize(userCount);
if (stage != Stage::Compute && stage != Stage::Fragment && snapshot.type != 6) vertex = Graphics::DecodeVertexStageInfo(snapshot.header, snapshot.headerAddress, plan->userData, nullptr, true);
plan->swappc = {vertex.has_value(), firstUser, userCount};
if (deferred != nullptr) {
if (const auto counter = ShaderRecompiler::UnresolvableSwappcTarget(plan->decoded, &plan->swappc)) {
char reason[96];
std::snprintf(reason, sizeof(reason), "its s_swappc_b64 at program counter 0x%x calls a computed target", *counter);
return defer(reason);
}
}
plan->memory = {{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}};
auto& request = plan->request.emplace(ShaderRecompiler::RecompileRequest{{stage, address, code, snapshot.headerAddress, snapshot.header}, {wave, firstUser, plan->userData, compute, pixel, vertex, plan->memory, RegisteredFloatMode(snapshot)}, stage == Stage::Compute ? device.ComputeTarget(wave) : device.Target(), {0, 0, 0, 128}, graphics});
if (graphics && graphics->mesh) request.layout.pushConstantSizeBytes = ShaderRecompiler::MeshDrawPushOffsetBytes;
if (pixel && device.GraphicsPipelineLibraries()) request.layout.pushConstantOffsetBytes = Graphics::FixedPushOffset(Stage::Fragment);
return plan;
}
std::vector<PreparedShaders::Entry> PrepareRegistered(RegisteredPreparation& plan) {
auto& request = *plan.request;
std::vector<PreparedShaders::Entry> entries;
ShaderRecompiler::ShaderPreparationContext preparation;
const auto append = [&] {
PerformanceTimer timing("Shader.PrepareArtifact");
entries.push_back({codeOffset, ShaderRecompiler::PrepareShader(request)});
entries.push_back({plan.codeOffset, PrepareShaderWithDiagnostics(request, &preparation)});
};
append();
if (compute) {
if (request.context.compute) {
request.context.compute->partialThreads = {1, 1, 1};
append();
} else if (pixel) {
} else if (request.context.pixel) {
request.layout.pushConstantSizeBytes = 0;
append();
}
return entries;
}
std::vector<PreparedShaders::Entry> PrepareRegistered(const ShaderSnapshot& snapshot, const VulkanDevice& device, const QueueState& state, bool registration, bool* deferred = nullptr) {
const auto plan = PlanRegistered(snapshot, device, state, registration, deferred);
return plan != nullptr ? PrepareRegistered(*plan) : std::vector<PreparedShaders::Entry>{};
}
class RegistrationPreparePool {
public:
static RegistrationPreparePool& Instance() {
static auto* pool = new RegistrationPreparePool();
return *pool;
}
void Submit(std::function<void()> task) {
if (threads == 0) {
task();
return;
}
{
std::lock_guard lock(mutex);
tasks.push_back(std::move(task));
}
ready.notify_one();
}
private:
RegistrationPreparePool() {
const auto count = std::max(2u, std::thread::hardware_concurrency() / 2u);
pthread_attr_t attributes;
pthread_attr_init(&attributes);
pthread_attr_setstacksize(&attributes, 64u << 20u);
for (unsigned index = 0; index < count; ++index) {
pthread_t thread;
if (pthread_create(&thread, &attributes, &RegistrationPreparePool::Main, this) == 0) {
pthread_detach(thread);
++threads;
}
}
pthread_attr_destroy(&attributes);
}
static void* Main(void* argument) {
auto& pool = *static_cast<RegistrationPreparePool*>(argument);
for (;;) {
std::function<void()> task;
{
std::unique_lock lock(pool.mutex);
pool.ready.wait(lock, [&] { return !pool.tasks.empty(); });
task = std::move(pool.tasks.front());
pool.tasks.pop_front();
}
task();
}
return nullptr;
}
std::mutex mutex;
std::condition_variable ready;
std::deque<std::function<void()>> tasks;
unsigned threads = 0;
};
bool AsyncRegistrationPrepare() {
static const bool sync = std::getenv("APS5_SYNC_SHADER_PREPARE") != nullptr;
return !sync;
}
void PrepareRegisteredInBackground(std::shared_ptr<const ShaderSnapshot> snapshot, std::shared_ptr<RegisteredPreparation> plan) {
RegistrationPreparePool::Instance().Submit([snapshot = std::move(snapshot), plan = std::move(plan)] {
PerformanceContext timingContext(FrameTiming::Preparation());
std::vector<PreparedShaders::Entry> entries;
std::exception_ptr failure;
try {
entries = PrepareRegistered(*plan);
} catch (const std::exception& error) {
APS5_LOG_ERR("Shader 0x%llx could not be prepared after registration: %s", static_cast<unsigned long long>(snapshot->codeAddress), error.what());
failure = std::current_exception();
} catch (...) {
failure = std::current_exception();
}
auto& prepared = *snapshot->prepared;
{
std::lock_guard lock(prepared.mutex);
prepared.entries.insert(prepared.entries.end(), std::make_move_iterator(entries.begin()), std::make_move_iterator(entries.end()));
if (failure == nullptr && !plan->abiKey.empty()) prepared.registeredAbis.push_back(std::move(plan->abiKey));
prepared.failure = failure;
prepared.pending = false;
}
prepared.settled.notify_all();
});
}
}
std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target) {
@ -413,7 +630,8 @@ std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decod
}
std::vector<PreparedGraphicsStage> prepared;
std::uint32_t pushOffset = 0;
const auto capacity = decoded.state.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : Graphics::PipelinePushConstantBytes;
const bool fixedSlots = target.fixedPushSlots;
const auto capacity = decoded.state.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : Graphics::PipelinePushSlotBytes;
for (std::size_t index = 0; index < decoded.programs.size(); ++index) {
if (decoded.roles[index] == ShaderRecompiler::ProgramRole::GeometryBack) continue;
const auto& program = decoded.programs[index];
@ -421,7 +639,10 @@ std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decod
const auto wave = fragment ? decoded.state.stages.fragmentWaveSize : decoded.state.stages.vertexWaveSize;
std::optional<ShaderRecompiler::ShaderVertexStageInfo> vertex;
if (!fragment) vertex = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, nullptr, true);
ShaderRecompiler::RecompileRequest request{program.binary, {wave, program.firstUserSgpr, program.userData, {}, fragment ? std::optional(decoded.pixel) : std::nullopt, vertex, memory, RegisteredFloatMode(*program.snapshot)}, target, {0, 0, pushOffset, capacity - pushOffset}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, decoded.state.stages.mesh, decoded.state.stages.tessellation, {}}};
pushOffset = Graphics::StagePushOffset(pushOffset, program.binary.stage, fixedSlots);
const auto offset = pushOffset;
const auto room = (fixedSlots && fragment ? Graphics::PipelinePushSlotBytes : capacity) - pushOffset % Graphics::PipelinePushSlotBytes;
ShaderRecompiler::RecompileRequest request{program.binary, {wave, program.firstUserSgpr, program.userData, {}, fragment ? std::optional(decoded.pixel) : std::nullopt, vertex, memory, RegisteredFloatMode(*program.snapshot)}, target, {0, 0, offset, room}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, decoded.state.stages.mesh, decoded.state.stages.tessellation, {}}};
std::vector<std::uint64_t> key;
ShaderRecompiler::BuildPreparedShaderKey(request, key);
std::shared_ptr<const ShaderRecompiler::SourceHandle> handle;
@ -433,10 +654,10 @@ std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decod
}
if (handle == nullptr) {
PerformanceTimer timing("Shader.PrepareArtifact");
handle = ShaderRecompiler::PrepareShader(request);
handle = PrepareShaderWithDiagnostics(request);
}
const auto bytes = handle->artifact->bindings.pushConstantSizeBytes;
require(bytes <= capacity - pushOffset, "prepared graphics stages exceed the push constant block");
require(bytes <= room, "prepared graphics stages exceed the push constant block");
prepared.push_back({program.snapshot, {program.codeOffset, handle}});
pushOffset += bytes;
}
@ -462,7 +683,7 @@ void Driver::ResolveGraphicsStagesAbi(std::span<const Shader* const> stages, std
require(registry != nullptr && registry->contains(address), "graphics ABI refers to an unregistered shader");
if (owner == nullptr) owner = registry->at(address);
const auto& snapshot = *registry->at(address);
require(snapshot.headerAddress == reinterpret_cast<std::uintptr_t>(shader), "graphics ABI refers to a replaced shader header");
require(SameHeader(snapshot, shader), "graphics ABI refers to a replaced shader header");
require(snapshot.registeredState != nullptr, "registered shader state is missing");
const auto& registered = *snapshot.registeredState;
for (const auto& [offset, value] : registered.shader) state.shader.insert_or_assign(offset, value);
@ -527,7 +748,7 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::span<const ShaderRegist
const auto address = reinterpret_cast<std::uintptr_t>(const_cast<const void*>(shader->code));
require(shaders != nullptr && shaders->contains(address), "static ABI refers to an unregistered shader");
snapshot = shaders->at(address);
require(snapshot->headerAddress == reinterpret_cast<std::uintptr_t>(shader), "static ABI refers to a replaced shader header");
require(SameHeader(*snapshot, shader), "static ABI refers to a replaced shader header");
}
require(snapshot->registeredState != nullptr, "registered shader state is missing");
QueueState state{};
@ -589,7 +810,7 @@ void Driver::ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::
const auto address = reinterpret_cast<std::uintptr_t>(const_cast<const void*>(shader->code));
require(shaders != nullptr && shaders->contains(address), "rectangle ABI refers to an unregistered shader");
const auto snapshot = shaders->at(address);
require(snapshot->headerAddress == reinterpret_cast<std::uintptr_t>(shader), "rectangle ABI refers to a replaced shader header");
require(SameHeader(*snapshot, shader), "rectangle ABI refers to a replaced shader header");
return snapshot;
};
front = lookup(vertex);
@ -659,6 +880,26 @@ void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr<
transaction.Commit();
}
ShaderSnapshot PrepareNullPixelProgram(const VulkanDevice& device) {
ShaderSnapshot null{NullPixelProgramAddress(), reinterpret_cast<std::uintptr_t>(&NullPixelShader), NullPixelShader.type, {}, {}};
null.code.assign(std::begin(NullPixelCode), std::end(NullPixelCode));
null.header.resize(sizeof(Shader));
std::memcpy(null.header.data(), &NullPixelShader, sizeof(Shader));
auto nullRegisteredState = DecodeRegisteredState(null);
nullRegisteredState.shader.insert_or_assign(0x008u, static_cast<std::uint32_t>(null.codeAddress >> 8u));
nullRegisteredState.shader.insert_or_assign(0x009u, static_cast<std::uint32_t>(null.codeAddress >> 40u));
nullRegisteredState.shader.insert_or_assign(0x00bu, 0u);
nullRegisteredState.context.insert_or_assign(0x1b3u, 0x2u);
nullRegisteredState.context.insert_or_assign(0x1b4u, 0x2u);
null.registeredState = std::make_shared<const RegisteredShaderState>(std::move(nullRegisteredState));
QueueState nullState{};
nullState.shader = null.registeredState->shader;
nullState.context = null.registeredState->context;
nullState.userConfig = null.registeredState->userConfig;
null.prepared->entries = PrepareRegistered(null, device, nullState, true);
return null;
}
void Driver::RegisterShader(const Shader* shader) {
PerformanceContext timingContext(FrameTiming::Preparation());
PerformanceTimer timing("Shader.Register");
@ -705,11 +946,18 @@ void Driver::RegisterShader(const Shader* shader) {
registered.shader = snapshot.registeredState->shader;
registered.context = snapshot.registeredState->context;
registered.userConfig = snapshot.registeredState->userConfig;
snapshot.prepared->entries = PrepareRegistered(snapshot, *localDevice, registered, true);
if ((snapshot.type == 0 || snapshot.type == 1) && !snapshot.prepared->entries.empty()) {
std::vector<std::uint64_t> key;
BuildRegisteredAbiKey(registered, *localDevice, key);
snapshot.prepared->registeredAbis.push_back(std::move(key));
std::shared_ptr<RegisteredPreparation> preparation;
if (AsyncRegistrationPrepare()) {
preparation = PlanRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred);
if (preparation != nullptr && (snapshot.type == 0 || snapshot.type == 1)) BuildRegisteredAbiKey(registered, *localDevice, preparation->abiKey);
snapshot.prepared->pending = preparation != nullptr;
} else {
snapshot.prepared->entries = PrepareRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred);
if ((snapshot.type == 0 || snapshot.type == 1) && !snapshot.prepared->entries.empty()) {
std::vector<std::uint64_t> key;
BuildRegisteredAbiKey(registered, *localDevice, key);
snapshot.prepared->registeredAbis.push_back(std::move(key));
}
}
static const char* traceRegs = std::getenv("APS5_TRACE_SHADER_REGS");
if (traceRegs != nullptr && (std::string(traceRegs) == "all" || std::strtoull(traceRegs, nullptr, 16) == snapshot.codeAddress)) {
@ -726,29 +974,15 @@ void Driver::RegisterShader(const Shader* shader) {
print(fields.sh_registers, fields.num_sh_registers);
std::fprintf(stderr, "\n");
}
const auto published = std::make_shared<const ShaderSnapshot>(std::move(snapshot));
std::lock_guard lock(mutex);
rethrowFailure();
PublishRegisteredShader(shaders, std::make_shared<const ShaderSnapshot>(std::move(snapshot)));
PublishRegisteredShader(shaders, published);
if (shaders->find(NullPixelProgramAddress()) == shaders->end()) {
ShaderSnapshot null{NullPixelProgramAddress(), reinterpret_cast<std::uintptr_t>(&NullPixelShader), NullPixelShader.type, {}, {}};
null.code.assign(std::begin(NullPixelCode), std::end(NullPixelCode));
null.header.resize(sizeof(Shader));
std::memcpy(null.header.data(), &NullPixelShader, sizeof(Shader));
auto nullRegisteredState = DecodeRegisteredState(null);
nullRegisteredState.shader.emplace(0x008u, static_cast<std::uint32_t>(null.codeAddress >> 8u));
nullRegisteredState.shader.emplace(0x009u, static_cast<std::uint32_t>(null.codeAddress >> 40u));
nullRegisteredState.shader.emplace(0x00bu, 0u);
nullRegisteredState.context.emplace(0x1b3u, 0x2u);
nullRegisteredState.context.emplace(0x1b4u, 0x2u);
null.registeredState = std::make_shared<const RegisteredShaderState>(std::move(nullRegisteredState));
QueueState nullState{};
nullState.shader = null.registeredState->shader;
nullState.context = null.registeredState->context;
nullState.userConfig = null.registeredState->userConfig;
null.prepared->entries = PrepareRegistered(null, *localDevice, nullState, true);
PublishRegisteredShader(shaders, std::make_shared<const ShaderSnapshot>(std::move(null)));
PublishRegisteredShader(shaders, std::make_shared<const ShaderSnapshot>(PrepareNullPixelProgram(*localDevice)));
}
transaction.Commit();
if (preparation != nullptr) PrepareRegisteredInBackground(published, std::move(preparation));
}
}

4
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp
View File

@ -117,11 +117,11 @@ void Driver::recordDeferredLabels(VulkanDevice* localDevice, std::uint32_t queue
const int reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label.address, bytes, stamp, queue, first) : 4;
first = false;
countLabelOutcome(reason);
if (reason != 0 && reason != 5 && reason != 6) {
if (reason != 0 && reason != 5) {
if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle();
GuestMemory::Write(label.address, bytes, 4);
}
noteLabelStore(label.address, bytes, stamp);
noteLabelStore(label.address, bytes, stamp, queue);
}
++labelGroups;
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;

10
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/LabelHistory.cpp
View File

@ -6,18 +6,19 @@
namespace AgcDriver::DriverDetail {
void Driver::noteLabelStore(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp) {
void Driver::noteLabelStore(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue) {
if (bytes.empty() || bytes.size() > 64 || bytes.size() % 4 != 0 || address % 4 != 0) return;
std::lock_guard lock(labelStoresMutex);
for (std::size_t offset = 0; offset < bytes.size(); offset += 4) {
auto& history = labelStores[address + offset];
std::shift_right(history.begin(), history.end(), 1);
history[0].stamp = stamp;
history[0].queue = queue;
std::memcpy(&history[0].value, bytes.data() + offset, 4);
}
}
bool Driver::storedSince(std::span<const std::uint32_t> packet, std::uint64_t address, std::size_t bytes, std::uint64_t received) {
bool Driver::storedSince(std::span<const std::uint32_t> packet, std::uint64_t address, std::size_t bytes, std::uint64_t received, std::uint32_t* writer) {
std::lock_guard lock(labelStoresMutex);
const auto low = labelStores.find(address);
if (low == labelStores.end()) return false;
@ -31,7 +32,10 @@ bool Driver::storedSince(std::span<const std::uint32_t> packet, std::uint64_t ad
if (upper == high->second.end()) continue;
value |= static_cast<std::uint64_t>(upper->value) << 32u;
}
if (Pm4::WaitComparesValue(packet, value)) return true;
if (Pm4::WaitComparesValue(packet, value)) {
if (writer != nullptr) *writer = store.queue;
return true;
}
}
return false;
}

5
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/SynchronizationStatistics.cpp
View File

@ -5,7 +5,7 @@
namespace AgcDriver::DriverDetail {
std::atomic<std::uint64_t> gpuLabels{0}, completionLabels{0}, notImportedLabels{0}, noOpLabels{0}, unlockedDrains{0};
std::atomic<std::uint64_t> gpuLabels{0}, completionLabels{0}, noOpLabels{0}, unlockedDrains{0};
std::atomic<std::uint64_t> labelFallbacks[5] = {};
@ -39,13 +39,12 @@ SubmissionCosts& submissionCosts(std::uint32_t queue) {
void reportSync() {
std::string report;
for (const auto& [code, count] : drainCounts) report += " " + (code == 0xffffu ? std::string("flip") : Pm4::Name(code << 8u)) + "=" + std::to_string(count);
AgcDriver::ProfilePrint_nid_no_patch("[sync] %llu device drains by packet:%s (%llu waited without the GPU mutex); %llu labels written on the GPU, %llu deferred behind completions, %llu deferred not imported, %llu no-op, fallbacks: idle %llu, completions %llu, not imported %llu, undecodable %llu; %llu labels queued per worker, recorded in %llu groups, %llu recorded at once; stores: %llu recorded on the GPU, %llu behind completions, %llu on the CPU (idle), %llu synced (drained)\n", static_cast<unsigned long long>(drainTotal), report.c_str(), static_cast<unsigned long long>(unlockedDrains.load()), static_cast<unsigned long long>(gpuLabels.load()), static_cast<unsigned long long>(completionLabels.load()), static_cast<unsigned long long>(notImportedLabels.load()), static_cast<unsigned long long>(noOpLabels.load()), static_cast<unsigned long long>(labelFallbacks[1].load()), static_cast<unsigned long long>(labelFallbacks[2].load()), static_cast<unsigned long long>(labelFallbacks[3].load()), static_cast<unsigned long long>(labelFallbacks[4].load()), static_cast<unsigned long long>(queuedLabels.load()), static_cast<unsigned long long>(labelGroups.load()), static_cast<unsigned long long>(immediateLabels.load()), static_cast<unsigned long long>(storesOnGpu.load()), static_cast<unsigned long long>(storesBehindCompletions.load()), static_cast<unsigned long long>(storesOnCpu.load()), static_cast<unsigned long long>(storesDrained.load()));
AgcDriver::ProfilePrint_nid_no_patch("[sync] %llu device drains by packet:%s (%llu waited without the GPU mutex); %llu labels written on the GPU, %llu deferred behind completions, %llu no-op, fallbacks: idle %llu, completions %llu, not imported %llu, undecodable %llu; %llu labels queued per worker, recorded in %llu groups, %llu recorded at once; stores: %llu recorded on the GPU, %llu behind completions, %llu on the CPU (idle), %llu synced (drained)\n", static_cast<unsigned long long>(drainTotal), report.c_str(), static_cast<unsigned long long>(unlockedDrains.load()), static_cast<unsigned long long>(gpuLabels.load()), static_cast<unsigned long long>(completionLabels.load()), static_cast<unsigned long long>(noOpLabels.load()), static_cast<unsigned long long>(labelFallbacks[1].load()), static_cast<unsigned long long>(labelFallbacks[2].load()), static_cast<unsigned long long>(labelFallbacks[3].load()), static_cast<unsigned long long>(labelFallbacks[4].load()), static_cast<unsigned long long>(queuedLabels.load()), static_cast<unsigned long long>(labelGroups.load()), static_cast<unsigned long long>(immediateLabels.load()), static_cast<unsigned long long>(storesOnGpu.load()), static_cast<unsigned long long>(storesBehindCompletions.load()), static_cast<unsigned long long>(storesOnCpu.load()), static_cast<unsigned long long>(storesDrained.load()));
}
void countLabelOutcome(int reason) {
if (reason == 0) ++gpuLabels;
else if (reason == 5) ++completionLabels;
else if (reason == 6) ++notImportedLabels;
else ++labelFallbacks[std::min(reason, 4)];
}

89
core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp
View File

@ -1,3 +1,4 @@
#include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp"
#include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp"
@ -11,6 +12,9 @@
#include <windows.h>
#endif
#include <immintrin.h>
#include <algorithm>
#include <tuple>
#include <vector>
namespace AgcDriver::DriverDetail {
@ -49,7 +53,57 @@ bool Driver::traceLateLabels() {
return trace;
}
void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t queue, const PacketHistory& context, std::uint64_t received, bool heldAtSubmit) {
std::vector<Driver::PendingWait>& Driver::pendingWaits() {
static thread_local std::vector<PendingWait> waits;
return waits;
}
void Driver::notePendingWait(std::span<const std::uint32_t> packet, std::uint32_t queue, std::uint64_t received) {
if (packet.size() > std::tuple_size_v<decltype(PendingWait::words)>) return;
static const bool allQueues = std::getenv("APS5_LAND_WAITS_ALL_QUEUES") != nullptr;
if (!allQueues && queue == 0) return;
if (Pm4::WaitSatisfiedUnchecked(packet)) return;
auto& waits = pendingWaits();
for (const auto& wait : waits) {
if (wait.count == packet.size() && std::equal(packet.begin(), packet.end(), wait.words.begin())) return;
}
PendingWait wait{};
std::copy(packet.begin(), packet.end(), wait.words.begin());
wait.count = packet.size();
wait.received = received;
waits.push_back(wait);
}
void Driver::landPendingWaits(std::uint32_t queue) {
auto& waits = pendingWaits();
if (waits.empty()) return;
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
struct LandStats {
std::uint64_t lands = 0, waited = 0;
double waitedMs = 0, maxMs = 0;
std::chrono::steady_clock::time_point reported = std::chrono::steady_clock::now();
};
static thread_local LandStats stats;
const PacketHistory none{};
for (const auto& wait : waits) {
const auto packet = std::span<const std::uint32_t>(wait.words).first(wait.count);
++stats.lands;
if (Pm4::WaitSatisfiedUnchecked(packet)) continue;
const auto started = std::chrono::steady_clock::now();
waitMemory(packet, queue, none, wait.received, false, true);
const auto ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - started).count();
++stats.waited;
stats.waitedMs += ms;
stats.maxMs = std::max(stats.maxMs, ms);
}
waits.clear();
if (profile && std::chrono::steady_clock::now() - stats.reported > std::chrono::seconds(10)) {
AgcDriver::ProfilePrint_nid_no_patch("[landwait] queue 0x%x (10 s): %llu waits landed before a guest read, %llu had to wait for the GPU: %.1f ms in total, max %.2f ms\n", queue, static_cast<unsigned long long>(stats.lands), static_cast<unsigned long long>(stats.waited), stats.waitedMs, stats.maxMs);
stats = LandStats{};
}
}
void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t queue, const PacketHistory& context, std::uint64_t received, bool heldAtSubmit, bool requireMemory) {
auto start = std::chrono::steady_clock::now();
auto lastDone = packetsDone.load();
@ -82,9 +136,14 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
++outcomes.atEntry;
return;
}
if (heldAtSubmit || storedSince(packet, awaited, awaitedBytes, received)) return;
if (heldAtSubmit) return;
if (std::uint32_t writer = queue; !requireMemory && storedSince(packet, awaited, awaitedBytes, received, &writer)) {
if (writer != queue) notePendingWait(packet, queue, received);
return;
}
static const bool labelShortcut = std::getenv("APS5_NO_LABEL_SHORTCUT") == nullptr;
const bool labelTable = labelShortcut && !requireMemory;
static const bool overlapSubmit = std::getenv("APS5_NO_WAIT_OVERLAP_SUBMIT") == nullptr;
static const bool waitLock = std::getenv("APS5_WAIT_LOCK") != nullptr;
@ -121,6 +180,8 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
if (hit->queue == queue) {
++outcomes.fromRecorderSameQueue;
if (!hit->late) epochPoint.bump = false;
} else {
notePendingWait(packet, queue, received);
}
return true;
};
@ -129,7 +190,7 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
const bool unlocked = !waitLock && overlapSubmit;
if (unlocked) {
if (labelShortcut) {
if (labelTable) {
Graphics::Recorder::LabelRefusal refusal{};
if (takeLabel(Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, &refusal), refusal, false, true)) return;
}
@ -152,7 +213,7 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
}
if (const auto localDevice = gpuLock.owns_lock() ? device.Load() : std::shared_ptr<VulkanDevice>{}) {
if (labelShortcut) {
if (labelTable) {
Graphics::Recorder::LabelRefusal refusal{};
if (takeLabel(localDevice->PendingLabel(awaited, awaitedBytes, received, &refusal), refusal, false, false)) return;
}
@ -182,7 +243,7 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
if (now - lastTry < PollTryInterval) return false;
lastTry = now;
}
if (changed && unlocked && labelShortcut) {
if (changed && unlocked && labelTable) {
Graphics::Recorder::LabelRefusal refusal{};
if (takeLabel(Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, &refusal), refusal, true, true)) {
@ -220,7 +281,7 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
if (changed) {
seenGeneration = generation;
recheck = false;
if (labelShortcut) {
if (labelTable) {
Graphics::Recorder::LabelRefusal refusal{};
if (takeLabel(localDevice->PendingLabel(awaited, awaitedBytes, received, &refusal), refusal, true, false)) {
if (pollProfile) ++poll.tableHits;
@ -268,7 +329,10 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
bool spinning = pauseSpin;
std::uint32_t polls = 0;
while (!Pm4::WaitSatisfiedUnchecked(packet)) {
if (storedSince(packet, awaited, awaitedBytes, received)) return;
if (std::uint32_t writer = queue; !requireMemory && storedSince(packet, awaited, awaitedBytes, received, &writer)) {
if (writer != queue) notePendingWait(packet, queue, received);
return;
}
++polls;
if (spinning) {
_mm_pause();
@ -281,7 +345,14 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
}
CheckFailure();
if (pollService(!spinning)) return;
if (const auto done = packetsDone.load(); done != lastDone || packetsInFlight.load() != 0) {
noteAwaitingTitle(queue, awaited);
if (requireMemory && (polls & 15u) == 0) {
std::unique_lock settleLock(GuestMemory::GpuMutex(), std::try_to_lock);
if (settleLock.owns_lock()) {
if (const auto localDevice = device.Load(); localDevice != nullptr && localDevice->RecordedWritesSettled(awaited, awaitedBytes)) return;
}
}
if (const auto done = packetsDone.load(); done != lastDone || (!requireMemory && packetsInFlight.load() != 0)) {
lastDone = done;
start = std::chrono::steady_clock::now();
}
@ -301,7 +372,7 @@ void Driver::waitMemory(std::span<const std::uint32_t> packet, std::uint32_t que
const std::uint64_t reference = wide ? packet[4] | (static_cast<std::uint64_t>(packet[5]) << 32u) : packet[4];
const std::uint64_t mask = wide ? packet[6] | (static_cast<std::uint64_t>(packet[7]) << 32u) : packet[5];
const std::uint64_t current = wide ? *reinterpret_cast<const volatile std::uint64_t*>(awaited) : *reinterpret_cast<const volatile std::uint32_t*>(awaited);
std::fprintf(stderr, "[gpu] queue 0x%x WAIT_REG_MEM%s at 0x%llx timed out after %dms (function %u ref 0x%llx mask 0x%llx value 0x%llx)\n", queue, wide ? "_64" : "",
std::fprintf(stderr, "[gpu] queue 0x%x WAIT_REG_MEM%s%s at 0x%llx timed out after %dms (function %u ref 0x%llx mask 0x%llx value 0x%llx)\n", queue, wide ? "_64" : "", requireMemory ? " (landing)" : "",
static_cast<unsigned long long>(awaited), waitTimeoutMs(), packet[1] & 7u, static_cast<unsigned long long>(reference), static_cast<unsigned long long>(mask), static_cast<unsigned long long>(current));
const auto address = packet[2] | (static_cast<std::uint64_t>(packet[3]) << 32u);
for (const auto& record : writeHistory()) {

105
core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp
View File

@ -747,7 +747,7 @@ constexpr std::size_t WriteBlockBytes = 65536;
constexpr std::size_t WritePageBytes = 4096;
constexpr std::size_t WritePagesPerBlock = WriteBlockBytes / WritePageBytes;
enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow };
enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow, Fresh };
#ifdef _WIN32
void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_t generation);
@ -901,6 +901,25 @@ struct WriteTracker {
#endif
}
bool unchanged(std::uint64_t first, std::uint64_t last, std::uint64_t since) const {
if (first == last) return stampOf(first) <= since;
const auto newer = [since](std::uint32_t stamp) { return stamp > since; };
#ifdef _WIN32
return std::none_of(blocks.begin() + first, blocks.begin() + last + 1, newer);
#else
while (first <= last) {
const auto index = first / LeafBlocks;
const auto end = std::min<std::uint64_t>(last + 1, (index + 1) * LeafBlocks);
if (const auto& leaf = leaves[index]; leaf != nullptr) {
const auto begin = leaf->blocks.begin() + first % LeafBlocks;
if (std::any_of(begin, begin + (end - first), newer)) return false;
}
first = end;
}
return true;
#endif
}
std::uint32_t cpuStampOf(std::uint64_t block) const {
#ifdef _WIN32
return cpuBlocks[block];
@ -920,7 +939,7 @@ struct WriteTracker {
}
void stamp(std::uint64_t block, std::uint32_t stampGeneration, StampKind kind) {
const bool cpu = kind != StampKind::Driver;
const bool cpu = kind == StampKind::Cpu || kind == StampKind::ImportWindow;
const bool written = kind != StampKind::ImportWindow;
#ifdef _WIN32
blocks[block] = stampGeneration;
@ -962,6 +981,7 @@ void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_
if (!tracker.watched || bytes == 0 || bytes > tracker.size || address < tracker.base || address - tracker.base > tracker.size - bytes) return;
tracker.coverage.Restore(address, bytes, generation);
++tracker.generation;
tracker.noteCpuStore(address, address + bytes, tracker.generation);
for (auto block = tracker.blockOf(address); block <= tracker.blockOf(address + bytes - 1); ++block) {
tracker.driverPieces.erase(block);
tracker.stamp(block, tracker.generation, StampKind::Cpu);
@ -1007,11 +1027,15 @@ bool sharedCollectMemo() {
struct StampRuns {
WriteTracker& tracker;
StampKind kind;
std::size_t stamped = 0;
};
void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) {
auto& [tracker, kind] = *static_cast<StampRuns*>(context);
auto& runs = *static_cast<StampRuns*>(context);
auto& tracker = runs.tracker;
const auto kind = runs.kind;
if (end <= begin) return;
++runs.stamped;
const auto generation = tracker.generation.load(std::memory_order_relaxed);
if (kind != StampKind::Driver) tracker.noteCpuStore(begin, end, generation);
for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, kind);
@ -1019,9 +1043,10 @@ void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) {
}
#endif
bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind) {
bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind, bool armedOnly = false) {
++tracker.generation;
#ifdef _WIN32
static_cast<void>(armedOnly);
constexpr std::uint64_t page = 4096;
// One resetting walk: the kernel reports and clears a page's dirty bit together, a write landing
// after the walk passed a page is reported by the next walk, and a clean page is not touched by the
@ -1067,7 +1092,8 @@ bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop,
#else
const auto dirtyRuns = collectDirtyRuns.load(std::memory_order_relaxed);
StampRuns runs{tracker, kind};
const bool complete = GuestWriteWatch::GuestWriteWatchCollect_nid_postfix(static_cast<std::uintptr_t>(first), static_cast<std::size_t>(stop - first), &stampWrittenRun, &runs);
const auto collect = armedOnly ? &GuestWriteWatch::GuestWriteWatchCollectArmed_nid_postfix : &GuestWriteWatch::GuestWriteWatchCollect_nid_postfix;
const bool complete = collect(static_cast<std::uintptr_t>(first), static_cast<std::size_t>(stop - first), &stampWrittenRun, &runs);
if (collectDirtyRuns.load(std::memory_order_relaxed) != dirtyRuns) collectDirty.fetch_add(1, std::memory_order_relaxed);
return complete;
#endif
@ -1077,8 +1103,8 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi
auto& tracker = Tracker();
// Whole pages, so a page shared with the next range is collected with either.
constexpr std::uint64_t page = 4096;
const auto first = address & ~(page - 1);
const auto stop = (address + bytes + page - 1) & ~(page - 1);
auto first = address & ~(page - 1);
auto stop = (address + bytes + page - 1) & ~(page - 1);
// One epoch for the lookup and the entry made after the walk: an unbumped thread's fresh epoch
// must not differ between them.
const auto epoch = currentCollectEpoch();
@ -1110,6 +1136,16 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi
}
}
const TimedAccess timed(CounterCollect, bytes);
#ifndef _WIN32
if (useMemo && threadCollectEpoch != 0 && bytes <= WriteBlockBytes) {
const auto begin = first & ~(WriteBlockBytes - 1);
const auto end = (stop + WriteBlockBytes - 1) & ~(WriteBlockBytes - 1);
if (end > begin && tracker.covers(begin, end - begin)) {
first = begin;
stop = end;
}
}
#endif
if (!walkWrites(tracker, first, stop, StampKind::Cpu)) return 0;
// Only a completed walk is remembered; a failed one (uncommitted pages) returned 0 above.
if (collectMemoEnabled()) {
@ -1177,9 +1213,9 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function
const auto first = address & ~(page - 1);
const auto stop = (address + bytes + page - 1) & ~(page - 1);
const bool covered = tracker.covers(first, static_cast<std::size_t>(stop - first));
const bool before = covered && walkWrites(tracker, first, stop, StampKind::Cpu);
const bool before = covered && walkWrites(tracker, first, stop, StampKind::Cpu, true);
if (!import()) return false;
if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow)) unwatchLocked(tracker, first, static_cast<std::size_t>(stop - first));
if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow, true)) unwatchLocked(tracker, first, static_cast<std::size_t>(stop - first));
return true;
}
@ -1204,10 +1240,7 @@ bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t gene
if (!tracker.watched || generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false;
const auto first = tracker.blockOf(address);
const auto last = tracker.blockOf(address + bytes - 1);
for (auto block = first; block <= last; ++block) {
if (tracker.stampOf(block) > generation) return false;
}
return true;
return tracker.unchanged(first, last, generation);
}
bool UnchangedSinceAll(std::span<const UnchangedQuery> queries) {
@ -1219,9 +1252,7 @@ bool UnchangedSinceAll(std::span<const UnchangedQuery> queries) {
if (generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false;
const auto first = tracker.blockOf(address);
const auto last = tracker.blockOf(address + bytes - 1);
for (auto block = first; block <= last; ++block) {
if (tracker.stampOf(block) > generation) return false;
}
if (!tracker.unchanged(first, last, generation)) return false;
}
return true;
}
@ -1261,6 +1292,15 @@ std::uint64_t StoreOwnBytes(std::uint64_t address, std::size_t bytes, const std:
});
}
std::uint64_t StoreOwnBytes(std::uint64_t address, std::span<const std::byte> source) {
return StoreOwnBytes(address, source.size(), [&] {
#ifndef _WIN32
if (GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix() && GuestArena::GuestArenaWriteSharedBacking_nid_no_patch(address, source.data(), source.size())) return;
#endif
std::memcpy(reinterpret_cast<void*>(address), source.data(), source.size());
});
}
std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
auto& tracker = Tracker();
const auto lock = lockTracker(tracker);
@ -1268,6 +1308,18 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) {
if (bytes == 0) return 0;
++tracker.generation;
if (!tracker.watched || !tracker.covers(address, bytes)) return 0;
#ifndef _WIN32
{
const auto blockFirst = address & ~(std::uint64_t{WriteBlockBytes} - 1);
const auto blockStop = (address + bytes + WriteBlockBytes - 1) & ~(std::uint64_t{WriteBlockBytes} - 1);
StampRuns runs{tracker, StampKind::Fresh};
if (!GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(static_cast<std::uintptr_t>(blockFirst), static_cast<std::size_t>(blockStop - blockFirst), &stampWrittenRun, &runs)) {
unwatchLocked(tracker, blockFirst, static_cast<std::size_t>(blockStop - blockFirst));
return 0;
}
if (runs.stamped != 0) ++tracker.generation;
}
#endif
const auto first = tracker.blockOf(address);
const auto last = tracker.blockOf(address + bytes - 1);
for (auto block = first; block <= last; ++block) {
@ -1287,15 +1339,15 @@ bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generati
const auto last = tracker.blockOf(end - 1);
for (auto block = first; block <= last; ++block) {
if (tracker.writtenStampOf(block) <= generation) continue;
if (tracker.cpuStampOf(block) > generation) {
if (const auto found = tracker.cpuPages.find(block); found != tracker.cpuPages.end()) {
const auto begin = tracker.blockBegin(block);
const auto from = std::max(address, begin) - begin;
const auto to = std::min<std::uint64_t>(end, begin + WriteBlockBytes) - begin;
const auto found = tracker.cpuPages.find(block);
if (found == tracker.cpuPages.end()) return true;
for (auto page = from / WritePageBytes; page <= (to - 1) / WritePageBytes; ++page) {
if (found->second.pages[page] > generation) return true;
}
} else if (tracker.cpuStampOf(block) > generation) {
return true;
}
const auto found = tracker.driverPieces.find(block);
if (found == tracker.driverPieces.end()) return true;
@ -1326,8 +1378,18 @@ bool UnchangedSinceCollected(std::uint64_t address, std::size_t bytes, std::uint
if (!tracker.watched || generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false;
const auto first = tracker.blockOf(address);
const auto last = tracker.blockOf(address + bytes - 1);
const auto end = address + bytes;
for (auto block = first; block <= last; ++block) {
if (tracker.cpuStampOf(block) > generation) return false;
if (const auto found = tracker.cpuPages.find(block); found != tracker.cpuPages.end()) {
const auto begin = tracker.blockBegin(block);
const auto from = std::max(address, begin) - begin;
const auto to = std::min<std::uint64_t>(end, begin + WriteBlockBytes) - begin;
for (auto page = from / WritePageBytes; page <= (to - 1) / WritePageBytes; ++page) {
if (found->second.pages[page] > generation) return false;
}
} else if (tracker.cpuStampOf(block) > generation) {
return false;
}
}
return true;
}
@ -1850,7 +1912,8 @@ void Write(std::uint64_t address, std::span<const std::byte> source, std::size_t
auto* destination = reinterpret_cast<void*>(address);
CheckRange(destination, source.size(), alignment, true);
// Stamped like a GPU write: a collect memoized for this packet would not see the page fault.
StoreOwnBytes(address, source.size(), [&] { std::memcpy(destination, source.data(), source.size()); });
if (source.size() >= PageBytes) StoreOwnBytes(address, source);
else StoreOwnBytes(address, source.size(), [&] { std::memcpy(destination, source.data(), source.size()); });
}
}

68
core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp
View File

@ -1,12 +1,14 @@
#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp"
#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp"
#include "prx/libc/include/General.hpp"
#include "prx/libc/include/PreciseWait.hpp"
#include <algorithm>
#include <thread>
#include <chrono>
#include <cstdio>
#include <cstring>
#include <limits>
#include <memory>
#include <stdexcept>
#include <string>
@ -113,6 +115,7 @@ std::vector<std::byte> copySource(std::uint64_t source, std::size_t bytes, bool
}
constexpr std::uint32_t DmaSelectGds = 1;
constexpr std::uint32_t DmaSelectNowhere = 2;
bool gdsRange(std::uint64_t offset, std::size_t bytes) {
return offset <= GdsBytes && bytes <= GdsBytes - offset;
@ -186,7 +189,7 @@ std::string_view UnsupportedReason(std::uint32_t header) {
case 0x11: case 0x12: case 0x13: case 0x15: case 0x16: case 0x20: case 0x22: case 0x26: case 0x27:
case 0x2a: case 0x2d: case 0x2f: case 0x35: case 0x37: case 0x40: case 0x42: case 0x45: case 0x46: case 0x50:
case 0x58: case 0x63: case 0x64: case 0x69: case 0x76: case 0x79: case 0x7a:
case 0x81: case 0x83: case 0x9f: return {};
case 0x81: case 0x83: case 0x9f: case 0x28: return {};
case 0x24: case 0x25: case 0x2c: case 0x38:
if (IndirectDrawsDisabled()) return "graphics draw, shader stages and guest render-target materialization are not implemented";
return {};
@ -203,7 +206,7 @@ std::string_view UnsupportedReason(std::uint32_t header) {
case 0x43: case 0x47: case 0x48:
return "guest cache actions, GPU events and interrupt delivery are not implemented";
case 0x8e: return "GPU LOD statistics are not implemented; synthetic results are forbidden";
case 0x28: case 0x41: case 0x68: case 0x78:
case 0x41: case 0x68: case 0x78:
return "opcode is named but has no handler in the reference dispatch table";
default: return "opcode is not known in the reference";
}
@ -258,7 +261,7 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
// neither changes what the packet writes.
const auto flags = header & 0xffu;
const bool registerWrite = opcode == 0x69 || opcode == 0x76 || opcode == 0x79 || opcode == 0x7a;
auto allowedFlags = opcode == 0x11 ? 2u : registerWrite ? 6u : opcode == 0x3c || opcode == 0x93 ? 2u : 0u;
auto allowedFlags = opcode == 0x11 || opcode == 0x15 || opcode == 0x16 ? 2u : registerWrite ? 6u : opcode == 0x3c || opcode == 0x93 ? 2u : 0u;
if (opcode != 0x20 && opcode != 0x22) allowedFlags |= 1u;
if ((flags & ~allowedFlags) != 0) throw std::runtime_error("PM4 header flags 0x" + ToHex(flags) + " are not implemented");
switch (opcode) {
@ -320,10 +323,10 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
require((packet[8] & 3u) == 0 && packet[8] >= (opcode == 0x2c ? 16u : 20u), "invalid indirect draw stride");
require((packet[9] & ~0x20u) == (opcode == 0x2c ? 2u : 0u), "unsupported indirect draw initiator");
break;
case 0x15: size(5); if ((packet[4] & ~0xa020u) != 0x41u) throw std::runtime_error("dispatch modifiers 0x" + ToHex(packet[4]) + " are not implemented"); break;
case 0x15: size(5); if ((packet[4] & ~0xa024u) != 0x41u) throw std::runtime_error("dispatch modifiers 0x" + ToHex(packet[4]) + " are not implemented"); break;
case 0x16:
require(packet.size() == 3 || packet.size() == 4, "invalid indirect dispatch size");
require((packet.back() & ~0xa020u) == 0x41u, "indirect dispatch modifiers are not implemented");
require((packet.back() & ~0xa024u) == 0x41u, "indirect dispatch modifiers are not implemented");
break;
case 0x22:
size(5);
@ -333,6 +336,11 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
require((packet[4] & ~ConditionalWordsMask) == 0, "COND_EXEC reserved count bits are not implemented");
break;
case 0x42: size(2); require(packet[1] == 0, "unsupported PFP_SYNC_ME payload"); break;
case 0x28:
graphics();
size(3);
require((packet[1] & 0x7fffffffu) == 0 && (packet[2] & 0x7fffffffu) == 0, "CONTEXT_CONTROL register loading and shadowing are not implemented");
break;
case 0x46: {
require((packet[1] & ~0x73fu) == 0, "unsupported EVENT_WRITE flags or reserved bits");
const auto eventType = packet[1] & 0x3fu;
@ -348,6 +356,15 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
size(2);
require(eventIndex == 0 || eventIndex == 7, "invalid cache-flush event index");
break;
case 0x19: case 0x1a: case 0x26:
size(2);
require(eventIndex == 0, "invalid pipeline statistics or SQ event index");
break;
case 0x24:
graphics();
size(2);
require(eventIndex == 0, "invalid VGT_FLUSH event index");
break;
case 0x39:
graphics();
size(4);
@ -363,11 +380,11 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
const auto controlMask = packet.size() == 8 ? 0x86287fc3u : 0xfeecfffbu;
require((packet[1] & ~controlMask) == 0, "unsupported ACQUIRE_MEM control flags");
require(queue == 0 || (packet[1] & 0x06287fc3u) == 0, "graphics cache operation in compute queue");
require(packet[3] == 0 && packet[5] == 0, "ACQUIRE_MEM ranges above 40 bits are not implemented");
require(packet[3] <= 0xffu && packet[5] <= 0xffu, "invalid ACQUIRE_MEM range high bits");
require(packet[6] <= 0xffffu, "invalid ACQUIRE_MEM poll interval");
const auto base = static_cast<std::uint64_t>(packet[4]) << 8u;
const auto bytes = static_cast<std::uint64_t>(packet[2]) << 8u;
require(bytes <= (1ull << 40u) - base, "ACQUIRE_MEM range exceeds 40-bit address space");
const auto base = ((static_cast<std::uint64_t>(packet[5]) << 32u) | packet[4]) << 8u;
const auto bytes = ((static_cast<std::uint64_t>(packet[3]) << 32u) | packet[2]) << 8u;
require(bytes <= (1ull << 48u) - base, "ACQUIRE_MEM range exceeds 48-bit address space");
if (packet.size() == 8) {
require((packet[7] & ~0x3ffffu) == 0, "unsupported ACQUIRE_MEM GCR flags");
require((packet[7] & 0x2000u) == 0, "ACQUIRE_MEM cache discard is not implemented");
@ -377,13 +394,13 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
case 0x63: case 0x64: case 0x9f:
if (opcode != 0x63) graphics();
size(5);
require((packet[1] & 3u) == 0 && packet[3] == 0x80000000u && packet[4] <= 0x3fffu, "unsupported indirect-register address or control fields");
require((packet[1] & 3u) == 0 && packet[4] <= 0x3fffu && (packet[3] == 0x80000000u || ((packet[3] & ~0xffffu) == 0 && packet[4] <= 0x10000u - packet[3])), "unsupported indirect-register address or control fields");
break;
case 0x69: case 0x76: case 0x79: case 0x7a: {
if (IsTagMarker(packet)) break;
if (opcode != 0x76) graphics();
require(packet.size() >= 3, "register packet has no values");
if (opcode == 0x7a) require((packet[1] & 0xf0000000u) == 0 || (packet.size() == 3 && packet[1] == 0x20000243u), "indexed register bank selection is not implemented");
if (opcode == 0x7a) require((packet[1] & 0xf0000000u) == 0 || (packet.size() == 3 && (packet[1] == 0x20000243u || packet[1] == 0x10000242u)), "indexed register bank selection is not implemented");
const auto offset = registerOffset(packet[1]);
require(packet.size() - 2 <= 0x10000u - offset, "register range overflow");
break;
@ -427,7 +444,7 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
require(packet.size() >= 5, "WRITE_DATA has no data");
require((packet[1] & ~(0x40110f00u | WriteDataCachePolicy)) == 0, "WRITE_DATA engine or reserved fields are not implemented");
const auto destination = (packet[1] >> 8u) & 0xfu;
require(destination == 1 || destination == 2 || (queue != 0 && destination == 5), "WRITE_DATA register or GDS destination is not implemented");
require(destination == 1 || destination == 2 || destination == 5, "WRITE_DATA register or GDS destination is not implemented");
require((packet[2] & 3u) == 0, "misaligned WRITE_DATA destination");
break;
}
@ -444,7 +461,8 @@ void Validate(std::span<const std::uint32_t> packet, std::uint32_t queue) {
case 0x50:
size(7);
require((packet[1] & ~(0xe0300001u | DmaSourceCachePolicy | DmaDestinationCachePolicy)) == 0, "DMA_DATA reserved fields are not implemented");
require(memorySelector(dmaDestination(packet)) || dmaDestination(packet) == DmaSelectGds, "DMA_DATA register or prefetch destination is not implemented");
require(memorySelector(dmaDestination(packet)) || dmaDestination(packet) == DmaSelectGds || dmaDestination(packet) == DmaSelectNowhere, "DMA_DATA register destination is not implemented");
require(dmaDestination(packet) != DmaSelectNowhere || memorySelector(dmaSource(packet)), "DMA_DATA prefetch of a register, GDS or immediate source is not implemented");
require(memorySelector(dmaSource(packet)) || dmaSource(packet) == 2 || dmaSource(packet) == DmaSelectGds, "DMA_DATA register source is not implemented");
require(dmaSource(packet) != DmaSelectGds || (packet[3] == 0 && gdsRange(packet[2], packet[6] & 0x3ffffffu)), "DMA_DATA GDS source range exceeds the GDS");
require(dmaDestination(packet) != DmaSelectGds || (packet[5] == 0 && gdsRange(packet[4], packet[6] & 0x3ffffffu)), "DMA_DATA GDS destination range exceeds the GDS");
@ -570,6 +588,7 @@ std::optional<StoreWrite> ResolveStore(std::span<const std::uint32_t> packet, co
}
case 0x50: {
if (packet.size() < 7 || dmaDestination(packet) == DmaSelectGds) return std::nullopt;
if (dmaDestination(packet) == DmaSelectNowhere) return StoreWrite{address(packet[2], packet[3]), {}, {}};
const std::size_t bytes = packet[6] & 0x3ffffffu;
const auto destination = address(packet[4], packet[5]);
if (!fits(destination, bytes)) return std::nullopt;
@ -734,9 +753,20 @@ DrawParameters ResolveDraw(std::span<const std::uint32_t> packet, const QueueSta
std::vector<std::uint32_t> ReadIndirectRegisters(std::span<const std::uint32_t> packet) {
require(packet.size() == 5 && IndirectRegisterOpcode((packet[0] >> 8u) & 0xffu), "expected indirect register packet");
std::vector<std::uint32_t> pairs(static_cast<std::size_t>(packet[4]) * 2);
// Named for the [hooksync] attribution (the read goes through the flush hook).
const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Registers);
if (packet[3] != 0x80000000u) {
std::vector<std::uint32_t> values(packet[4]);
GuestMemory::Read(address(packet[1], packet[2]), std::as_writable_bytes(std::span(values)), 4);
std::vector<std::uint32_t> pairs;
pairs.reserve(values.size() * 2);
for (std::size_t i = 0; i < values.size(); ++i) {
pairs.push_back(packet[3] + static_cast<std::uint32_t>(i));
pairs.push_back(values[i]);
}
return pairs;
}
std::vector<std::uint32_t> pairs(static_cast<std::size_t>(packet[4]) * 2);
GuestMemory::Read(address(packet[1], packet[2]), std::as_writable_bytes(std::span(pairs)), 4);
return pairs;
}
@ -760,11 +790,10 @@ void Execute(std::span<const std::uint32_t> packet, QueueState& queue) {
case 0x10:
switch ((packet[0] >> 2u) & 0x3fu) {
case 0: return;
case 0x09: queue = QueueState{}; return;
case 0x09: queue = std::move(*std::make_unique<QueueState>()); return;
case 0x0b: queue.markers.emplace_back(reinterpret_cast<const char*>(packet.data() + 1)); return;
case 0x0c:
require(!queue.markers.empty(), "marker stack underflow");
queue.markers.pop_back();
if (!queue.markers.empty()) queue.markers.pop_back();
return;
case 0x1a:
if (TraceContextState()) std::fprintf(stderr, "[context] op %u: %zu registers, saved %d, cb0 %x info %x mask %x\n", packet[1], queue.context.size(), queue.savedContext.has_value() ? 1 : 0, queue.context.contains(0x318) ? queue.context.at(0x318) : 0u, queue.context.contains(0x31c) ? queue.context.at(0x31c) : 0u, queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u);
@ -811,7 +840,7 @@ void Execute(std::span<const std::uint32_t> packet, QueueState& queue) {
const auto start = std::chrono::steady_clock::now();
bool warned = false;
while (!WaitSatisfied(packet)) {
std::this_thread::sleep_for(std::chrono::microseconds(50));
PreciseSleepUs(50);
if (!warned && std::chrono::steady_clock::now() - start > std::chrono::seconds(5)) {
warned = true;
std::fprintf(stderr, "[gpu] WAIT_REG_MEM at 0x%llx still waiting after 5s (function %u ref 0x%x mask 0x%x)\n",
@ -868,9 +897,10 @@ void Execute(std::span<const std::uint32_t> packet, QueueState& queue) {
GuestMemory::Write(destination, data);
return;
}
case 0x28: return;
case 0x50: {
const std::size_t bytes = packet[6] & 0x3ffffffu;
if (bytes == 0) return;
if (bytes == 0 || dmaDestination(packet) == DmaSelectNowhere) return;
auto data = dmaSourceBytes(packet);
const auto destination = dmaDestination(packet) == DmaSelectGds ? GdsAddress() + packet[4] : address(packet[4], packet[5]);
GuestMemory::CheckRange(reinterpret_cast<void*>(destination), bytes, 1, true);

23
core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp
View File

@ -201,6 +201,28 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val
return true;
}
bool ShaderMemory::accessible(void* context, std::uint64_t address, std::uint64_t bytes) {
auto& self = *static_cast<ShaderMemory*>(context);
if (bytes == 0 || address > std::numeric_limits<std::uint64_t>::max() - bytes) return false;
const auto end = address + bytes;
while (address < end) {
const auto next = self.initial.upper_bound(address);
if (next != self.initial.begin()) {
const auto previous = std::prev(next);
if (address - previous->first < previous->second.size()) {
address = previous->first + previous->second.size();
continue;
}
}
const auto base = address & ~static_cast<std::uint64_t>(PageBytes - 1);
const auto pageEnd = std::min<std::uint64_t>(base + PageBytes, end);
const auto& page = self.page(base);
if (!page.wordwise && !page.valid.all() && !GuestMemory::Accessible(reinterpret_cast<const void*>(address), static_cast<std::size_t>(pageEnd - address))) return false;
address = pageEnd;
}
return true;
}
ShaderMemory::KnownValueCounts ShaderMemory::KnownValues() {
auto& totals = CaptureTotals();
return {totals.wordsKnown.load(std::memory_order_relaxed), totals.wordsKnownVerified.load(std::memory_order_relaxed), totals.wordsKnownMismatches.load(std::memory_order_relaxed)};
@ -233,6 +255,7 @@ std::shared_ptr<const ShaderRecompiler::ResourceCapture> ShaderMemory::capture(c
runtime.userContext = this;
runtime.readMemory = &read;
runtime.readSpecializationMemory = &read;
runtime.accessible = &accessible;
auto capture = invocation != nullptr ? invocation->Capture(runtime) : handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime);
if (profile) {
totals.captureNanoseconds += NanosecondsSince(started);

3
core/libs/prx/libSceAgcDriver/Execution/src/SubgroupClock.cpp
View File

@ -15,7 +15,8 @@ constexpr std::array<std::string_view, 19> NarrowClockFamilies{
}
bool NarrowSubgroupClock(VkDriverId driver, std::string_view deviceName) {
bool NarrowSubgroupClock(VkDriverId driver, std::uint32_t deviceId, std::string_view deviceName) {
if (driver == VK_DRIVER_ID_AMD_PROPRIETARY && (deviceId == 0x1114u || deviceId == 0x744cu || deviceId == 0x747eu)) return true;
if (driver != VK_DRIVER_ID_MESA_RADV) return false;
constexpr std::string_view prefix = "(RADV ";
const auto start = deviceName.rfind(prefix);

260
core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp
View File

@ -6,6 +6,7 @@
#include "prx/libSceAgcDriver/Execution/include/Driver.hpp"
#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp"
#include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp"
#include "prx/libSceAgcDriver/Execution/include/FragmentBarycentric.hpp"
#include "prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp"
#include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp"
#include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp"
@ -160,6 +161,20 @@ struct ComputePipelineObjects {
}
};
constexpr std::uint64_t WorkgroupMemoryGranule = 1u << 20u;
struct WorkgroupMemory {
VkDevice device = VK_NULL_HANDLE;
PFN_vkDestroyDescriptorPool destroyPool = nullptr;
std::unique_ptr<Graphics::DeviceBuffer> buffer;
VkDescriptorPool pool = VK_NULL_HANDLE;
VkDescriptorSet set = VK_NULL_HANDLE;
~WorkgroupMemory() {
buffer.reset();
if (pool != VK_NULL_HANDLE) destroyPool(device, pool, nullptr);
}
};
struct VulkanDevice::State {
void* library = nullptr;
PFN_vkGetInstanceProcAddr instanceProc = nullptr;
@ -201,16 +216,22 @@ struct VulkanDevice::State {
// VK_EXT_descriptor_indexing with non-uniform image array indexing (bindless image tables in
// graphics stages, and compute workgroups wider than a wave).
bool descriptorIndexing = false;
bool storageBufferUpdateAfterBind = false;
VkPhysicalDeviceDescriptorIndexingPropertiesEXT descriptorIndexingProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_PROPERTIES_EXT};
bool imageInt64Atomics = false;
bool bufferInt64Atomics = false;
bool primitiveListRestart = false;
bool depthClipControl = false;
bool graphicsPipelineLibrary = false;
bool imageViewMinLod = false;
bool pipelineExecutableInfo = false;
bool maintenance8 = false;
std::uint32_t srgbDecodeFormats = 0;
bool storageImageWithoutFormat = false;
bool depthClamp = false;
bool depthBounds = false;
bool depthBiasClamp = false;
bool dualSrcBlend = false;
bool occlusionQueryPrecise = false;
VkDeviceSize hostImportAlignment = 0;
bool dmaBufImport = false;
@ -219,8 +240,11 @@ struct VulkanDevice::State {
bool samplerAnisotropy = false;
bool textureCompressionBC = false;
bool samplerFilterMinmax = false;
bool nonSeamlessCubeMap = false;
bool fragmentShaderPixelInterlock = false;
bool conservativeRasterization = false;
bool provokingVertexLast = false;
bool provokingVertexModePerPipeline = false;
// VK_KHR_timeline_semaphore enabled: the recorder's unlocked waits are available.
bool timelineSemaphores = false;
bool computeWave32 = false;
@ -241,6 +265,8 @@ struct VulkanDevice::State {
// Device-local buffers of a repeated 16-byte fill pattern (FillBuffer), most recently used
// last; each is filled once by a doubling chain in device memory.
std::vector<std::pair<std::array<std::uint32_t, 4>, std::shared_ptr<Graphics::DeviceBuffer>>> patternBuffers;
VkDescriptorSetLayout workgroupMemoryLayout = VK_NULL_HANDLE;
std::shared_ptr<WorkgroupMemory> workgroupMemory;
// Compute pipeline objects by variant (and push-constant use), under their own mutex: the
// dispatch's find-or-insert and the verify switch's lookups touch the map, recipes hold weak
// references to its objects.
@ -332,10 +358,56 @@ struct VulkanDevice::State {
return function;
}
void CreateWorkgroupMemoryLayout() {
const VkDescriptorSetLayoutBinding binding{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr};
VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
info.bindingCount = 1;
info.pBindings = &binding;
check(DeviceFunction<PFN_vkCreateDescriptorSetLayout>("vkCreateDescriptorSetLayout")(device, &info, nullptr, &workgroupMemoryLayout), "vkCreateDescriptorSetLayout workgroup memory");
}
std::shared_ptr<WorkgroupMemory> WorkgroupMemoryFor(const Graphics::Context& context, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint32_t dwords) {
const std::uint64_t range = properties.limits.maxStorageBufferRange;
const auto groupBytes = static_cast<std::uint64_t>(dwords) * 4u;
std::uint64_t groups = 1;
for (const std::uint64_t count : {x, y, z}) {
if (count != 0u && groups > range / groupBytes / count) {
throw std::runtime_error("Vulkan dispatch: the LDS of " + std::to_string(x) + "x" + std::to_string(y) + "x" + std::to_string(z) + " workgroups of " + std::to_string(groupBytes) + " bytes each is over maxStorageBufferRange " + std::to_string(range));
}
groups *= count;
}
const auto bytes = std::max<std::uint64_t>(groups, 1u) * groupBytes;
if (workgroupMemory != nullptr && workgroupMemory->buffer->Size() >= bytes) return workgroupMemory;
const auto size = std::min<std::uint64_t>((bytes + WorkgroupMemoryGranule - 1u) / WorkgroupMemoryGranule * WorkgroupMemoryGranule, range);
auto memory = std::make_shared<WorkgroupMemory>();
memory->device = device;
memory->destroyPool = DeviceFunction<PFN_vkDestroyDescriptorPool>("vkDestroyDescriptorPool");
memory->buffer = std::make_unique<Graphics::DeviceBuffer>(context, static_cast<std::size_t>(size), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
const VkDescriptorPoolSize poolSize{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1};
VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
poolInfo.maxSets = 1;
poolInfo.poolSizeCount = 1;
poolInfo.pPoolSizes = &poolSize;
check(DeviceFunction<PFN_vkCreateDescriptorPool>("vkCreateDescriptorPool")(device, &poolInfo, nullptr, &memory->pool), "vkCreateDescriptorPool workgroup memory");
VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};
allocation.descriptorPool = memory->pool;
allocation.descriptorSetCount = 1;
allocation.pSetLayouts = &workgroupMemoryLayout;
check(DeviceFunction<PFN_vkAllocateDescriptorSets>("vkAllocateDescriptorSets")(device, &allocation, &memory->set), "vkAllocateDescriptorSets workgroup memory");
const VkDescriptorBufferInfo described{memory->buffer->Handle(), 0, size};
VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
write.dstSet = memory->set;
write.dstBinding = 0;
write.descriptorCount = 1;
write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
write.pBufferInfo = &described;
DeviceFunction<PFN_vkUpdateDescriptorSets>("vkUpdateDescriptorSets")(device, 1, &write, 0, nullptr);
workgroupMemory = std::move(memory);
return workgroupMemory;
}
void Upload(std::span<const std::byte> pixels) {
APS5_LOG_OUT("Upload pixels=%zu uploadSize=%llu buffer=%p memory=%p mapping=%p", pixels.size(), static_cast<unsigned long long>(uploadSize), reinterpret_cast<void*>(uploadBuffer), reinterpret_cast<void*>(uploadMemory), uploadMapping);
if (uploadSize < pixels.size()) {
APS5_LOG_OUT("Upload reallocating oldSize=%llu newSize=%zu", static_cast<unsigned long long>(uploadSize), pixels.size());
if (uploadMapping) DeviceFunction<PFN_vkUnmapMemory>("vkUnmapMemory")(device, uploadMemory);
uploadMapping = nullptr;
if (uploadBuffer) DeviceFunction<PFN_vkDestroyBuffer>("vkDestroyBuffer")(device, uploadBuffer, nullptr);
@ -359,7 +431,6 @@ struct VulkanDevice::State {
}
}
require(memoryType < memoryProperties.memoryTypeCount, "coherent host upload memory is unavailable");
APS5_LOG_OUT("Upload requirements size=%llu alignment=%llu typeBits=0x%x memoryType=%u", static_cast<unsigned long long>(requirements.size), static_cast<unsigned long long>(requirements.alignment), requirements.memoryTypeBits, memoryType);
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = memoryType;
@ -367,10 +438,8 @@ struct VulkanDevice::State {
check(DeviceFunction<PFN_vkBindBufferMemory>("vkBindBufferMemory")(device, uploadBuffer, uploadMemory, 0), "vkBindBufferMemory display upload");
check(DeviceFunction<PFN_vkMapMemory>("vkMapMemory")(device, uploadMemory, 0, pixels.size(), 0, &uploadMapping), "vkMapMemory display upload");
uploadSize = pixels.size();
APS5_LOG_OUT("Upload allocation complete buffer=%p memory=%p mapping=%p size=%llu", reinterpret_cast<void*>(uploadBuffer), reinterpret_cast<void*>(uploadMemory), uploadMapping, static_cast<unsigned long long>(uploadSize));
}
std::memcpy(uploadMapping, pixels.data(), pixels.size());
APS5_LOG_OUT("Upload memcpy complete bytes=%zu", pixels.size());
}
bool SharedSlotInFlight() {
@ -535,6 +604,8 @@ struct VulkanDevice::State {
Graphics::ClearImageMirrors(device);
Graphics::ClearHostImports(device);
patternBuffers.clear();
workgroupMemory.reset();
if (workgroupMemoryLayout != VK_NULL_HANDLE) reinterpret_cast<PFN_vkDestroyDescriptorSetLayout>(deviceProc(device, "vkDestroyDescriptorSetLayout"))(device, workgroupMemoryLayout, nullptr);
descriptorCache.reset();
emptyBuffer.reset();
samplerCache.reset();
@ -760,6 +831,12 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &barycentricFeatures};
state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &features);
state->fragmentShaderBarycentric = barycentricFeatures.fragmentShaderBarycentric == VK_TRUE;
if (state->fragmentShaderBarycentric && hasExtension(VK_KHR_DRIVER_PROPERTIES_EXTENSION_NAME)) {
VkPhysicalDeviceDriverProperties driver{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES};
VkPhysicalDeviceProperties2 driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &driver};
state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &driverProperties);
state->fragmentShaderBarycentric = FragmentShaderBarycentricUsable(driver.driverID, state->fragmentShaderBarycentric);
}
}
VkPhysicalDeviceFragmentShaderInterlockFeaturesEXT interlockFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT};
if (hasExtension(VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME)) {
@ -779,7 +856,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
VkPhysicalDeviceDriverProperties driver{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES};
VkPhysicalDeviceProperties2 driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &driver};
state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &driverProperties);
state->narrowSubgroupClock = NarrowSubgroupClock(driver.driverID, state->properties.deviceName);
state->narrowSubgroupClock = NarrowSubgroupClock(driver.driverID, state->properties.deviceID, state->properties.deviceName);
APS5_LOG_OUT("Shader clock driver=%d narrowSubgroupClock=%d", static_cast<int>(driver.driverID), state->narrowSubgroupClock ? 1 : 0);
}
std::vector<const char*> deviceExtensions;
@ -809,6 +886,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
atomicInt64Features = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES_KHR};
atomicInt64Features.shaderBufferInt64Atomics = VK_TRUE;
if (bufferInt64Atomics) deviceExtensions.push_back(VK_KHR_SHADER_ATOMIC_INT64_EXTENSION_NAME);
state->bufferInt64Atomics = bufferInt64Atomics;
VkPhysicalDeviceShaderImageAtomicInt64FeaturesEXT imageAtomicInt64Features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_IMAGE_ATOMIC_INT64_FEATURES_EXT};
if (hasExtension(VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME)) {
VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &imageAtomicInt64Features};
@ -826,6 +904,21 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
deviceExtensions.push_back(VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME);
state->capabilities.push_back(spv::CapabilitySignedZeroInfNanPreserve);
state->spirvExtensions.push_back("SPV_KHR_float_controls");
VkPhysicalDeviceShaderFloat16Int8FeaturesKHR float16Features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT16_INT8_FEATURES_KHR};
if (hasExtension(VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME)) {
VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &float16Features};
state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &features);
}
const bool float16Rte = float16Features.shaderFloat16 == VK_TRUE && floatControls.shaderRoundingModeRTEFloat16 == VK_TRUE &&
floatControls.shaderDenormPreserveFloat16 == VK_TRUE && floatControls.shaderSignedZeroInfNanPreserveFloat16 == VK_TRUE;
float16Features = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT16_INT8_FEATURES_KHR};
float16Features.shaderFloat16 = VK_TRUE;
if (float16Rte) {
deviceExtensions.push_back(VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME);
state->capabilities.push_back(spv::CapabilityFloat16);
state->capabilities.push_back(spv::CapabilityRoundingModeRTE);
state->capabilities.push_back(spv::CapabilityDenormPreserve);
}
deviceExtensions.push_back(VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME);
deviceExtensions.push_back(VK_KHR_8BIT_STORAGE_EXTENSION_NAME);
state->capabilities.push_back(4448);
@ -891,6 +984,21 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
}
listRestartFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PRIMITIVE_TOPOLOGY_LIST_RESTART_FEATURES_EXT};
listRestartFeatures.primitiveTopologyListRestart = state->primitiveListRestart ? VK_TRUE : VK_FALSE;
VkPhysicalDeviceProvokingVertexFeaturesEXT provokingFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROVOKING_VERTEX_FEATURES_EXT};
if (hasExtension(VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME)) {
VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &provokingFeatures};
state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &features);
state->provokingVertexLast = provokingFeatures.provokingVertexLast == VK_TRUE;
if (state->provokingVertexLast) {
VkPhysicalDeviceProvokingVertexPropertiesEXT provokingProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROVOKING_VERTEX_PROPERTIES_EXT};
VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &provokingProperties};
state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &properties);
state->provokingVertexModePerPipeline = provokingProperties.provokingVertexModePerPipeline == VK_TRUE;
deviceExtensions.push_back(VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME);
}
}
provokingFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROVOKING_VERTEX_FEATURES_EXT};
provokingFeatures.provokingVertexLast = state->provokingVertexLast ? VK_TRUE : VK_FALSE;
VkPhysicalDeviceImageViewMinLodFeaturesEXT minLodFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_VIEW_MIN_LOD_FEATURES_EXT};
if (hasExtension(VK_EXT_IMAGE_VIEW_MIN_LOD_EXTENSION_NAME)) {
VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &minLodFeatures};
@ -916,6 +1024,33 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
state->depthClipControl = depthClipFeatures.depthClipControl == VK_TRUE;
if (state->depthClipControl) deviceExtensions.push_back(VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME);
}
VkPhysicalDeviceNonSeamlessCubeMapFeaturesEXT nonSeamlessCubeFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NON_SEAMLESS_CUBE_MAP_FEATURES_EXT};
if (hasExtension(VK_EXT_NON_SEAMLESS_CUBE_MAP_EXTENSION_NAME)) {
VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &nonSeamlessCubeFeatures};
state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &features);
state->nonSeamlessCubeMap = nonSeamlessCubeFeatures.nonSeamlessCubeMap == VK_TRUE;
if (state->nonSeamlessCubeMap) deviceExtensions.push_back(VK_EXT_NON_SEAMLESS_CUBE_MAP_EXTENSION_NAME);
}
nonSeamlessCubeFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NON_SEAMLESS_CUBE_MAP_FEATURES_EXT};
nonSeamlessCubeFeatures.nonSeamlessCubeMap = VK_TRUE;
VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT libraryFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GRAPHICS_PIPELINE_LIBRARY_FEATURES_EXT};
VkPhysicalDeviceExtendedDynamicStateFeaturesEXT dynamicStateFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_FEATURES_EXT};
VkPhysicalDeviceDynamicRenderingFeaturesKHR renderingFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DYNAMIC_RENDERING_FEATURES_KHR};
constexpr std::array<const char*, 6> libraryExtensions{VK_KHR_PIPELINE_LIBRARY_EXTENSION_NAME, VK_EXT_GRAPHICS_PIPELINE_LIBRARY_EXTENSION_NAME, VK_EXT_EXTENDED_DYNAMIC_STATE_EXTENSION_NAME, VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME, VK_KHR_DEPTH_STENCIL_RESOLVE_EXTENSION_NAME, VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME};
if (std::getenv("APS5_NO_GPL") == nullptr && std::all_of(libraryExtensions.begin(), libraryExtensions.end(), hasExtension)) {
dynamicStateFeatures.pNext = &renderingFeatures;
libraryFeatures.pNext = &dynamicStateFeatures;
VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &libraryFeatures};
state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &features);
VkPhysicalDeviceGraphicsPipelineLibraryPropertiesEXT libraryProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GRAPHICS_PIPELINE_LIBRARY_PROPERTIES_EXT};
VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &libraryProperties};
state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &properties);
state->graphicsPipelineLibrary = libraryFeatures.graphicsPipelineLibrary == VK_TRUE && dynamicStateFeatures.extendedDynamicState == VK_TRUE && renderingFeatures.dynamicRendering == VK_TRUE && libraryProperties.graphicsPipelineLibraryFastLinking == VK_TRUE && properties.properties.limits.maxPushConstantsSize >= Graphics::PipelinePushConstantBytes;
if (state->graphicsPipelineLibrary) deviceExtensions.insert(deviceExtensions.end(), libraryExtensions.begin(), libraryExtensions.end());
}
renderingFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DYNAMIC_RENDERING_FEATURES_KHR, nullptr, VK_TRUE};
libraryFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GRAPHICS_PIPELINE_LIBRARY_FEATURES_EXT, &dynamicStateFeatures, VK_TRUE};
dynamicStateFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_FEATURES_EXT, &renderingFeatures, VK_TRUE};
if (state->meshShader) {
deviceExtensions.insert(deviceExtensions.end(), meshExtensions.begin(), meshExtensions.end());
state->capabilities.push_back(5283);
@ -941,7 +1076,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
enabled.tessellationShader = available.tessellationShader;
state->tessellationShader = enabled.tessellationShader == VK_TRUE;
if (state->tessellationShader) state->capabilities.push_back(3);
APS5_LOG_OUT("Vulkan features tessellationAvailable=%u mesh=%u depthClip=%u depthRangeUnrestricted=%u", static_cast<unsigned>(state->tessellationShader), static_cast<unsigned>(state->meshShader), static_cast<unsigned>(state->depthClipControl), static_cast<unsigned>(state->depthRangeUnrestricted));
APS5_LOG_OUT("Vulkan features tessellationAvailable=%u mesh=%u depthClip=%u depthRangeUnrestricted=%u pipelineLibrary=%u", static_cast<unsigned>(state->tessellationShader), static_cast<unsigned>(state->meshShader), static_cast<unsigned>(state->depthClipControl), static_cast<unsigned>(state->depthRangeUnrestricted), static_cast<unsigned>(state->graphicsPipelineLibrary));
require(available.samplerAnisotropy && available.textureCompressionBC, "device lacks sampler anisotropy or BC texture compression support required for texture sampling");
enabled.samplerAnisotropy = VK_TRUE;
enabled.textureCompressionBC = VK_TRUE;
@ -960,6 +1095,8 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
state->depthBounds = enabled.depthBounds == VK_TRUE;
enabled.depthBiasClamp = available.depthBiasClamp;
state->depthBiasClamp = enabled.depthBiasClamp == VK_TRUE;
enabled.dualSrcBlend = available.dualSrcBlend;
state->dualSrcBlend = enabled.dualSrcBlend == VK_TRUE;
enabled.occlusionQueryPrecise = available.occlusionQueryPrecise;
state->occlusionQueryPrecise = enabled.occlusionQueryPrecise == VK_TRUE;
// Recompiled storage-image access declares no format (the guest descriptor decides it).
@ -981,6 +1118,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
enabled.shaderClipDistance = available.shaderClipDistance;
if (enabled.shaderStorageImageWriteWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageWriteWithoutFormat);
if (enabled.shaderStorageImageReadWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageReadWithoutFormat);
state->storageImageWithoutFormat = enabled.shaderStorageImageWriteWithoutFormat == VK_TRUE && enabled.shaderStorageImageReadWithoutFormat == VK_TRUE;
// Bindless image tables index an image array with a wave-uniform runtime slot.
enabled.shaderSampledImageArrayDynamicIndexing = available.shaderSampledImageArrayDynamicIndexing;
enabled.shaderStorageImageArrayDynamicIndexing = available.shaderStorageImageArrayDynamicIndexing;
@ -991,17 +1129,23 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &descriptorIndexingFeatures};
state->InstanceFunction<PFN_vkGetPhysicalDeviceFeatures2>("vkGetPhysicalDeviceFeatures2")(selected, &features);
state->descriptorIndexing = descriptorIndexingFeatures.shaderSampledImageArrayNonUniformIndexing == VK_TRUE && descriptorIndexingFeatures.shaderStorageImageArrayNonUniformIndexing == VK_TRUE;
state->storageBufferUpdateAfterBind = descriptorIndexingFeatures.descriptorBindingStorageBufferUpdateAfterBind == VK_TRUE;
if (state->storageBufferUpdateAfterBind) {
VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &state->descriptorIndexingProperties};
state->InstanceFunction<PFN_vkGetPhysicalDeviceProperties2>("vkGetPhysicalDeviceProperties2")(selected, &properties);
}
}
descriptorIndexingFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES_EXT};
if (state->descriptorIndexing || state->storageBufferUpdateAfterBind) deviceExtensions.push_back(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME);
if (state->descriptorIndexing) {
descriptorIndexingFeatures.shaderSampledImageArrayNonUniformIndexing = VK_TRUE;
descriptorIndexingFeatures.shaderStorageImageArrayNonUniformIndexing = VK_TRUE;
deviceExtensions.push_back(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME);
state->capabilities.push_back(spv::CapabilityShaderNonUniform);
state->capabilities.push_back(spv::CapabilitySampledImageArrayNonUniformIndexing);
state->capabilities.push_back(spv::CapabilityStorageImageArrayNonUniformIndexing);
state->spirvExtensions.push_back("SPV_EXT_descriptor_indexing");
}
descriptorIndexingFeatures.descriptorBindingStorageBufferUpdateAfterBind = state->storageBufferUpdateAfterBind ? VK_TRUE : VK_FALSE;
state->samplerAnisotropy = true;
state->textureCompressionBC = true;
deviceInfo.pEnabledFeatures = &enabled;
@ -1015,10 +1159,18 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
depthClipFeatures.pNext = const_cast<void*>(deviceInfo.pNext);
deviceInfo.pNext = &depthClipFeatures;
}
if (state->graphicsPipelineLibrary) {
renderingFeatures.pNext = const_cast<void*>(deviceInfo.pNext);
deviceInfo.pNext = &libraryFeatures;
}
if (state->primitiveListRestart) {
listRestartFeatures.pNext = const_cast<void*>(deviceInfo.pNext);
deviceInfo.pNext = &listRestartFeatures;
}
if (state->provokingVertexLast) {
provokingFeatures.pNext = const_cast<void*>(deviceInfo.pNext);
deviceInfo.pNext = &provokingFeatures;
}
if (state->imageViewMinLod) {
minLodFeatures.pNext = const_cast<void*>(deviceInfo.pNext);
deviceInfo.pNext = &minLodFeatures;
@ -1027,6 +1179,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
maintenance8Features.pNext = const_cast<void*>(deviceInfo.pNext);
deviceInfo.pNext = &maintenance8Features;
}
if (state->nonSeamlessCubeMap) {
nonSeamlessCubeFeatures.pNext = const_cast<void*>(deviceInfo.pNext);
deviceInfo.pNext = &nonSeamlessCubeFeatures;
}
byteFeatures.pNext = const_cast<void*>(deviceInfo.pNext);
if (state->fragmentShaderBarycentric) {
barycentricFeatures.pNext = byteFeatures.pNext;
@ -1040,6 +1196,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
clockFeatures.pNext = byteFeatures.pNext;
byteFeatures.pNext = &clockFeatures;
}
if (float16Rte) {
float16Features.pNext = byteFeatures.pNext;
byteFeatures.pNext = &float16Features;
}
if (bufferInt64Atomics) {
atomicInt64Features.pNext = byteFeatures.pNext;
byteFeatures.pNext = &atomicInt64Features;
@ -1053,7 +1213,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
imageRobustnessFeatures.pNext = byteFeatures.pNext;
byteFeatures.pNext = &imageRobustnessFeatures;
}
if (state->descriptorIndexing) {
if (state->descriptorIndexing || state->storageBufferUpdateAfterBind) {
descriptorIndexingFeatures.pNext = byteFeatures.pNext;
byteFeatures.pNext = &descriptorIndexingFeatures;
}
@ -1149,6 +1309,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u
state->colorTransfer = std::make_unique<Graphics::GpuColorTransfer>(graphicsContext());
state->descriptorCache = std::make_unique<Graphics::DescriptorCache>(graphicsContext());
state->samplerCache = std::make_unique<Graphics::SamplerCache>();
state->CreateWorkgroupMemoryLayout();
state->recorder = std::make_unique<Graphics::Recorder>(graphicsContext(), state->timelineSemaphores);
state->recorder->Activate();
state->context = buildContext();
@ -1297,6 +1458,13 @@ bool VulkanDevice::OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) c
return state->recorder && state->recorder->OpenWriteOverlaps(address, bytes);
}
bool VulkanDevice::RecordedWritesSettled(std::uint64_t address, std::size_t bytes) const {
if (!state->recorder) return true;
const auto& recorder = *state->recorder;
if (!recorder.PendingWriteSettled(address, bytes)) return false;
return Graphics::Recorder::PendingCompletionLabels() == 0 || !recorder.CompletionLabelIn(address, bytes);
}
int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::byte> bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) {
if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4;
auto& recorder = *state->recorder;
@ -1312,13 +1480,7 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span<const std::by
static const bool drain = std::getenv("APS5_DRAIN_COMPLETION_LABELS") != nullptr;
const auto context = graphicsContext();
const auto* import = Graphics::HostImportFor(context, address, bytes.size());
if (import == nullptr) {
// Memory the GPU has no view of: the store is a completion action of the batch (it runs after
// the recorded work, in order, like a label behind write-backs) instead of a device drain.
if (drain) return 3;
recorder.AfterCompletions(address, bytes, stamp, queue, false);
return 6;
}
if (import == nullptr) return 3;
// A unit shadow's results in the label's unit reach the import before the store lands over
// part of it (the store's stamp then makes the unit stale); the deferred label paths call no
// FlushPending of their own.
@ -1382,9 +1544,9 @@ bool VulkanDevice::DumpSamplesOnGpu(std::uint64_t address) {
const auto* import = Graphics::HostImportFor(context, address, bytes);
if (import == nullptr || import->address == 0) return false;
if (Graphics::AnyShadowedOverlaps(address, bytes)) Graphics::PublishShadow(address, bytes, Graphics::PublishScope::PartialUnits, Graphics::PublishReason::Label);
recorder.FlushStoresOverlapping(address, bytes);
if (!recorder.QueuesSampleDumps()) recorder.FlushStoresOverlapping(address, bytes);
recorder.FlushKeyStoresOverlapping(address, bytes);
if (!recorder.DumpSamples(import->address + (address - import->base))) return false;
if (!recorder.DumpSamples(import->address + (address - import->base), address)) return false;
recorder.NotePendingWrite(address, bytes);
GuestMemory::MarkWritten(address, bytes);
return true;
@ -2033,6 +2195,8 @@ FrameDumps& Dumps() {
}
bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) {
static const bool storeAtFlip = Graphics::StorageTexture::StoreAtFlipRequested(std::getenv("APS5_STORE_AT_FLIP"));
if (storeAtFlip) Graphics::StorageTexture::FlushAllPending("flip");
if (buffer.tilingMode == 1) {
const auto pixels = ReadDisplayBuffer(buffer);
return present(buffer.width, buffer.height, true, pixels);
@ -2461,6 +2625,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const {
target.fragmentShaderBarycentricEnabled = state->fragmentShaderBarycentric;
target.nonConstantImageOffsets = state->maintenance8;
target.narrowSubgroupClock = state->narrowSubgroupClock;
target.fixedPushSlots = state->graphicsPipelineLibrary;
target.srgbDecodeFormats = state->srgbDecodeFormats;
if (state->meshShader) {
const auto& mesh = state->meshLimits;
@ -2470,10 +2635,20 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const {
return target;
}
std::optional<ShaderRecompiler::GeometryStageLimits> VulkanDevice::GeometryLimits() const {
if (!state->geometryShader) return std::nullopt;
const auto& limits = state->properties.limits;
return ShaderRecompiler::GeometryStageLimits{limits.maxGeometryInputComponents, limits.maxGeometryOutputComponents, limits.maxGeometryOutputVertices, limits.maxGeometryTotalOutputComponents, limits.maxFragmentInputComponents};
}
bool VulkanDevice::PrimitiveListRestart() const {
return state->primitiveListRestart;
}
bool VulkanDevice::DualSrcBlend() const {
return state->dualSrcBlend;
}
bool VulkanDevice::SamplerFilterMinmax() const {
return state->samplerFilterMinmax;
}
@ -2482,6 +2657,18 @@ bool VulkanDevice::ConservativeRasterization() const {
return state->conservativeRasterization;
}
VkShaderStageFlags VulkanDevice::SubgroupStages() const {
return state->subgroup.supportedStages;
}
bool VulkanDevice::ProvokingVertexLast() const {
return state->provokingVertexLast;
}
bool VulkanDevice::GraphicsPipelineLibraries() const {
return state->graphicsPipelineLibrary;
}
Graphics::Context VulkanDevice::graphicsContext() const {
static const bool noCache = std::getenv("APS5_NO_CONTEXT_CACHE") != nullptr;
if (state->contextReady && !noCache) return state->context;
@ -2523,25 +2710,33 @@ Graphics::Context VulkanDevice::buildContext() const {
context.samplerCache = state->samplerCache.get();
context.drawIndirectFirstInstance = state->drawIndirectFirstInstance;
context.multiDrawIndirect = state->multiDrawIndirect;
context.dualSrcBlend = state->dualSrcBlend;
context.depthBounds = state->depthBounds;
context.depthBiasClamp = state->depthBiasClamp;
context.samplerFilterMinmax = state->samplerFilterMinmax;
context.nonSeamlessCubeMap = state->nonSeamlessCubeMap;
context.conservativeRasterization = state->conservativeRasterization;
context.provokingVertexLast = state->provokingVertexLast;
context.graphicsPipelineLibrary = state->graphicsPipelineLibrary;
context.provokingVertexModePerPipeline = state->provokingVertexModePerPipeline;
context.drawIndirectCount = state->drawIndirectCount;
context.occlusionQueryPrecise = state->occlusionQueryPrecise;
context.emptyBuffer = state->emptyBuffer ? state->emptyBuffer->Handle() : VK_NULL_HANDLE;
context.copiedWriters = state->copiedWriters.get();
context.functions = state->functionsReady ? &state->deviceFunctions : nullptr;
context.descriptorIndexing = state->descriptorIndexing;
context.descriptorIndexingLimits = state->descriptorIndexingProperties;
context.imageInt64Atomics = state->imageInt64Atomics;
context.geometryShader = state->geometryShader;
context.sampleRateShading = state->sampleRateShading;
context.nullDescriptors = state->shaderProfile != nullptr && state->shaderProfile->NullDescriptors();
context.bufferInt64Atomics = state->bufferInt64Atomics;
context.primitiveListRestart = state->primitiveListRestart;
context.imageViewMinLod = state->imageViewMinLod;
context.pipelineExecutableInfo = state->pipelineExecutableInfo;
context.srgbDecodeFormats = state->srgbDecodeFormats;
context.memoryProperties2 = state->memoryBudget ? state->InstanceFunction<PFN_vkGetPhysicalDeviceMemoryProperties2>("vkGetPhysicalDeviceMemoryProperties2") : nullptr;
context.singlePassStorage = state->storageImageWithoutFormat;
return context;
}
@ -2628,6 +2823,7 @@ struct PreparedDispatch {
// APS5_PROFILE_DRAW: stage A's time, added to the [dispatch] phase totals by the dispatch, and
// the prepare's parts for the driver's 'prepare:' rows (PreparePhase order).
double prepareMs = 0;
std::uint64_t dispatchThreads = 0;
enum PreparePhase : std::size_t { PrepareKey, PrepareFind, PreparePrecollect, PreparePresync, PrepareStageA, PreparePhaseCount };
std::array<double, PreparePhaseCount> phaseMs{};
// The stage-A pre-sync (see PrepareDispatch): the newest recorder serial waited for without the
@ -3071,7 +3267,7 @@ std::uint64_t VulkanDevice::presync(std::span<const std::pair<std::uint64_t, std
return serial;
}
std::shared_ptr<PreparedDispatch> VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span<const Graphics::GuestMemorySnapshot> snapshots) {
std::shared_ptr<PreparedDispatch> VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::uint64_t dispatchThreads) {
ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion);
// APS5_LOCKED_BUILD=1: the whole build under the mutex, as before the split.
static const bool lockedBuild = std::getenv("APS5_LOCKED_BUILD") != nullptr;
@ -3081,6 +3277,7 @@ std::shared_ptr<PreparedDispatch> VulkanDevice::PrepareDispatch(const ShaderReco
const Graphics::CompiledShader compute{ShaderRecompiler::ShaderStage::Compute, &shader, 0};
const auto context = graphicsContext();
auto prepared = std::make_shared<PreparedDispatch>();
prepared->dispatchThreads = dispatchThreads;
// A cached build (the map locks itself) is revalidated under the mutex by the dispatch, which
// looks it up again by the key made here; the object stays local, only its surfaces matter
// below. Anything inserted between here and the dispatch is simply replaced by this build.
@ -3100,7 +3297,7 @@ std::shared_ptr<PreparedDispatch> VulkanDevice::PrepareDispatch(const ShaderReco
phase(PreparedDispatch::PrepareFind);
}
if (cached == nullptr) {
prepared->resources = std::make_shared<Graphics::ShaderResources>(context, compute, snapshots, true);
prepared->resources = std::make_shared<Graphics::ShaderResources>(context, compute, snapshots, true, dispatchThreads);
phase(PreparedDispatch::PrepareStageA);
}
if (cached != nullptr && CachedPrecollect()) {
@ -3208,6 +3405,8 @@ void VulkanDevice::decideIndirect(RecordedDispatch& record, IndirectOutcome& out
outcome.cpuReason = 1;
Graphics::Recorder::CountSync(2);
recorder.Sync();
} else if (record.shader->program->workgroupMemoryDwords != 0) {
outcome.cpuReason = 8;
} else if (labelPending() || std::any_of(state->copiedWriters->begin(), state->copiedWriters->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); })
|| std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); })) {
outcome.cpuReason = 2;
@ -3331,8 +3530,13 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) {
}
context.Resolved(&Graphics::DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->pipeline);
resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout);
if (const auto dwords = record.shader->program->workgroupMemoryDwords; dwords != 0) {
auto memory = state->WorkgroupMemoryFor(context, record.x, record.y, record.z, dwords);
context.Resolved(&Graphics::DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, ShaderRecompiler::WorkgroupMemoryDescriptorSet, 1, &memory->set, 0, nullptr);
recorder.Keep(std::move(memory));
}
if (record.pushStages != 0) {
context.Resolved(&Graphics::DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, record.objects->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes, record.pushBytes->data());
context.Resolved(&Graphics::DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, record.objects->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::ComputePushConstantBytes, record.pushBytes->data());
}
const auto gpuTiming = recorder.BeginGpuTiming(record.programAddress != 0 ? record.programAddress : record.shader->program->variantId);
if (argumentImport != nullptr) context.Resolved(&Graphics::DeviceFunctions::cmdDispatchIndirect, "vkCmdDispatchIndirect")(commands, argumentImport->buffer, arguments - argumentImport->base);
@ -3378,7 +3582,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
}
const std::array<Graphics::CompiledShader, 1> shaders{{{ShaderRecompiler::ShaderStage::Compute, &shader, 0}}};
const auto pushStages = Graphics::PushConstantStages(shaders);
if (pushStages != 0 && state->properties.limits.maxPushConstantsSize < Graphics::PipelinePushConstantBytes) {
if (pushStages != 0 && state->properties.limits.maxPushConstantsSize < Graphics::ComputePushConstantBytes) {
throw std::runtime_error("Vulkan dispatch: compute push constant range exceeds device limit");
}
auto pushBytes = Graphics::AssemblePushConstants(shaders);
@ -3504,7 +3708,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
}
if (resources == nullptr) {
const auto buildStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{};
resources = std::make_shared<Graphics::ShaderResources>(context, shaders[0], snapshots);
resources = std::make_shared<Graphics::ShaderResources>(context, shaders[0], snapshots, false, prepared != nullptr ? prepared->dispatchThreads : 0);
if (profile) timer.add(PhaseResourcesFullBuild, std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - buildStart).count());
if (cacheable) {
++d.cacheMisses;
@ -3544,11 +3748,11 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
moduleInfo.pCode = shader.spirv.data();
check(state->DeviceFunction<PFN_vkCreateShaderModule>("vkCreateShaderModule")(state->device, &moduleInfo, nullptr, &objects->module), "vkCreateShaderModule");
timing.Mark("validate_shader_module");
const auto setLayout = resources->Layout();
const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes};
const std::array<VkDescriptorSetLayout, 2> setLayouts{resources->Layout(), shader.workgroupMemoryDwords != 0 ? state->workgroupMemoryLayout : VK_NULL_HANDLE};
const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::ComputePushConstantBytes};
VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
layoutInfo.setLayoutCount = 1;
layoutInfo.pSetLayouts = &setLayout;
layoutInfo.setLayoutCount = shader.workgroupMemoryDwords != 0 ? ShaderRecompiler::WorkgroupMemoryDescriptorSet + 1 : 1;
layoutInfo.pSetLayouts = setLayouts.data();
layoutInfo.pushConstantRangeCount = pushStages != 0 ? 1 : 0;
layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr;
check(state->DeviceFunction<PFN_vkCreatePipelineLayout>("vkCreatePipelineLayout")(state->device, &layoutInfo, nullptr, &objects->layout), "vkCreatePipelineLayout");
@ -3566,6 +3770,8 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec
pipelineInfo.layout = objects->layout;
if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] creating a pipeline for %zu SPIR-V words (program 0x%llx)\n", shader.spirv.size(), static_cast<unsigned long long>(programAddress));
check(state->DeviceFunction<PFN_vkCreateComputePipelines>("vkCreateComputePipelines")(state->device, context.pipelineCache, 1, &pipelineInfo, nullptr, &objects->pipeline), "vkCreateComputePipelines");
objects->destroyModule(objects->device, objects->module, nullptr);
objects->module = VK_NULL_HANDLE;
Graphics::LogPipelineStatistics_nid_no_patch(context, objects->pipeline);
timing.Mark("pipeline_create");
if (pipelineKey != 0) {

47
core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp
View File

@ -22,6 +22,14 @@ struct BufferAllocation {
std::size_t bytes;
VkBufferUsageFlags usage;
VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
VkDeviceSize offset = 0;
bool slab = false;
};
struct SlabSlot {
VkDeviceMemory memory;
VkDeviceSize offset;
void* mapping;
};
// Released buffer allocations kept for reuse, since creating, binding and mapping one costs tens of
@ -43,23 +51,39 @@ struct BufferAllocation {
// the 256-byte copied-region buffers to fit, and those then missed on every build (81869 misses
// and 80850 evictions per run, each a Vulkan create or destroy under the pool mutex, 3.5 s per
// run). APS5_BUFFER_POOL_SHARED=1 keeps one tier as before.
//
// Device-local allocations (the detiler's scratch buffers and the staging shadows of written guest
// buffers, see GuestBufferMemory) are retained in a third tier with a budget of their own, video
// memory instead of pinned host memory: APS5_STAGING_POOL_MIB (default 512); 0 keeps them in the
// two host tiers as before.
class BufferPool {
public:
explicit BufferPool(const Context& context);
static VkDeviceSize DeviceBudget(const VkPhysicalDeviceMemoryProperties& memory);
~BufferPool();
BufferPool(const BufferPool&) = delete;
BufferPool& operator=(const BufferPool&) = delete;
// The size a buffer for `bytes` is created with: its size class, or `bytes` itself when large.
static std::size_t Capacity(std::size_t bytes);
static std::size_t Capacity(std::size_t bytes, VkMemoryPropertyFlags properties);
static VkBufferUsageFlags Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties);
std::optional<BufferAllocation> Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
void Put(const BufferAllocation& allocation) noexcept;
static bool SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom);
static VkDeviceSize SlabBlockBytes(std::size_t capacity);
SlabSlot TakeSlot(const Context& context, std::uint32_t memoryType, std::size_t capacity, bool addressable);
void PutSlot(VkDeviceMemory memory, VkDeviceSize offset) noexcept;
std::size_t SlabBlocks();
private:
struct SlabBlock {
VkDeviceMemory memory = VK_NULL_HANDLE;
std::byte* mapping = nullptr;
VkDeviceSize slotBytes = 0;
std::vector<std::uint32_t> free;
std::uint32_t used = 0;
std::uint64_t slab = 0;
};
struct Slab {
std::vector<SlabBlock*> available;
std::size_t emptyBlocks = 0;
};
static std::uint64_t SlabKey(std::uint32_t memoryType, std::size_t capacity, bool addressable);
void freeBlock(SlabBlock& block) noexcept;
struct Slot {
BufferAllocation allocation;
std::uint64_t lastUse;
@ -90,8 +114,9 @@ private:
// The tier a buffer of `capacity` and `properties` is retained in (the large one for everything
// when shared; the device tier for device-local memory while it has a budget).
Tier& tierFor(std::size_t capacity, VkMemoryPropertyFlags properties);
// The device tier's budget (APS5_STAGING_POOL_MIB), read once.
static VkDeviceSize DeviceBudget();
static bool DeviceTierEnabled();
static bool DeviceTiered(VkMemoryPropertyFlags properties);
static std::size_t NextCapacity(std::size_t capacity);
void destroy(const BufferAllocation& allocation) noexcept;
// Moves the tier's least recently used slot (the oldest front of its lists) to `evicted`; the
// caller destroys those after releasing the mutex, so builds taking buffers on other threads
@ -104,18 +129,24 @@ private:
PFN_vkUnmapMemory unmap;
PFN_vkDestroyBuffer destroyBuffer;
PFN_vkFreeMemory freeMemory;
PFN_vkAllocateMemory allocateMemory;
PFN_vkMapMemory mapMemory;
std::mutex mutex;
// Not `small`/`large`: <rpcndr.h> (via <windows.h>) defines `small` as a macro.
Tier smallTier;
Tier largeTier;
Tier deviceTier;
std::uint64_t clock = 0;
std::mutex slabMutex;
std::unordered_map<std::uint64_t, Slab> slabs;
std::unordered_map<VkDeviceMemory, std::unique_ptr<SlabBlock>> slabBlocks;
static constexpr VkDeviceSize budget = 512ull * 1024 * 1024;
// The small tier's own budget (slots of at most half a MiB each): pinned host memory the large
// tier's budget does not count.
static constexpr VkDeviceSize smallBudget = 64ull * 1024 * 1024;
// Requests of this size and more keep their exact size and go to the large tier.
static constexpr std::size_t classLimit = std::size_t{1} << 20u;
static constexpr unsigned deviceClassBits = 3;
static constexpr std::size_t defaultSlots = 4096;
};

34
core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp
View File

@ -13,21 +13,31 @@ enum class ColorTileMode : std::uint32_t {
Linear = 0,
Standard4KB = 5,
Standard64KB = 9,
S4KBX = 0x15,
D4KBX = 0x16,
RenderTarget = 0x1b
};
constexpr bool ColorTileModeIsXor(ColorTileMode mode) {
return mode == ColorTileMode::S4KBX || mode == ColorTileMode::D4KBX || mode == ColorTileMode::RenderTarget;
}
constexpr std::uint32_t ColorTileModeBlockBytes(ColorTileMode mode) {
return mode == ColorTileMode::Linear ? 256u : mode == ColorTileMode::Standard4KB || mode == ColorTileMode::S4KBX || mode == ColorTileMode::D4KBX ? 4096u : 65536u;
}
constexpr TextureTileMode ColorTextureTileMode(ColorTileMode mode) {
return mode == ColorTileMode::Linear ? TextureTileMode::kLinear : mode == ColorTileMode::Standard4KB ? TextureTileMode::kStandard4KB : mode == ColorTileMode::Standard64KB ? TextureTileMode::kStandard64KB : TextureTileMode::kR64KBX;
return mode == ColorTileMode::Linear ? TextureTileMode::kLinear : mode == ColorTileMode::Standard4KB ? TextureTileMode::kStandard4KB : mode == ColorTileMode::Standard64KB ? TextureTileMode::kStandard64KB : mode == ColorTileMode::S4KBX ? TextureTileMode::kS4KBX : mode == ColorTileMode::D4KBX ? TextureTileMode::kD4KBX : TextureTileMode::kR64KBX;
}
ColorTileMode DecodeColorTileMode(std::uint32_t attrib3);
class ColorTargetLayout {
public:
ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement = 4);
ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement = 4, std::uint32_t pipeBankXor = 0);
std::size_t Bytes() const { return bytes; }
std::size_t LinearBytes() const { return static_cast<std::size_t>(width) * height * elementBytes; }
std::size_t Alignment() const { return mode == ColorTileMode::Linear ? 256u : mode == ColorTileMode::Standard4KB ? 4096u : 65536u; }
std::size_t Alignment() const { return ColorTileModeBlockBytes(mode); }
std::uint32_t BlocksPerRow() const { return pitch / blockWidth; }
std::size_t Offset(std::uint32_t x, std::uint32_t y) const;
void Detile(std::span<const std::byte> source, std::span<std::byte> destination) const;
@ -43,11 +53,29 @@ private:
std::uint32_t elementBytes;
std::uint32_t blockWidth = 1;
std::uint32_t blockHeight = 1;
std::uint32_t pipeBankXor = 0;
// The XOR swizzle is linear over GF(2), so a block offset is xOffsets[x] ^ yOffsets[y].
const std::uint32_t* xOffsets = nullptr;
const std::uint32_t* yOffsets = nullptr;
};
class CmaskLayout {
public:
static constexpr std::size_t Alignment = 4096;
CmaskLayout(std::uint32_t width, std::uint32_t height);
std::size_t Bytes() const { return bytes; }
std::uint32_t BlocksPerRow() const { return blocksPerRow; }
std::uint32_t TilesX() const { return (width + 7u) / 8u; }
std::uint32_t TilesY() const { return (height + 7u) / 8u; }
std::size_t Nibble(std::uint32_t tileX, std::uint32_t tileY) const;
private:
std::uint32_t width;
std::uint32_t height;
std::uint32_t blocksPerRow;
std::size_t bytes;
};
}
#endif

26
core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp
View File

@ -52,10 +52,23 @@ struct DeviceFunctions {
PFN_vkCmdDispatchIndirect cmdDispatchIndirect = nullptr;
PFN_vkCmdBeginRenderPass cmdBeginRenderPass = nullptr;
PFN_vkCmdEndRenderPass cmdEndRenderPass = nullptr;
PFN_vkCmdBeginRenderingKHR cmdBeginRendering = nullptr;
PFN_vkCmdEndRenderingKHR cmdEndRendering = nullptr;
PFN_vkCmdSetViewport cmdSetViewport = nullptr;
PFN_vkCmdSetScissor cmdSetScissor = nullptr;
PFN_vkCmdSetDepthBounds cmdSetDepthBounds = nullptr;
PFN_vkCmdSetDepthBias cmdSetDepthBias = nullptr;
PFN_vkCmdSetCullModeEXT cmdSetCullMode = nullptr;
PFN_vkCmdSetFrontFaceEXT cmdSetFrontFace = nullptr;
PFN_vkCmdSetDepthTestEnableEXT cmdSetDepthTestEnable = nullptr;
PFN_vkCmdSetDepthWriteEnableEXT cmdSetDepthWriteEnable = nullptr;
PFN_vkCmdSetDepthCompareOpEXT cmdSetDepthCompareOp = nullptr;
PFN_vkCmdSetDepthBoundsTestEnableEXT cmdSetDepthBoundsTestEnable = nullptr;
PFN_vkCmdSetStencilTestEnableEXT cmdSetStencilTestEnable = nullptr;
PFN_vkCmdSetStencilOpEXT cmdSetStencilOp = nullptr;
PFN_vkCmdSetStencilCompareMask cmdSetStencilCompareMask = nullptr;
PFN_vkCmdSetStencilWriteMask cmdSetStencilWriteMask = nullptr;
PFN_vkCmdSetStencilReference cmdSetStencilReference = nullptr;
PFN_vkCmdBindVertexBuffers cmdBindVertexBuffers = nullptr;
PFN_vkCmdBindIndexBuffer cmdBindIndexBuffer = nullptr;
PFN_vkCmdDraw cmdDraw = nullptr;
@ -125,7 +138,9 @@ struct Context {
bool occlusionQueryPrecise = false;
bool depthBounds = false;
bool depthBiasClamp = false;
bool dualSrcBlend = false;
bool samplerFilterMinmax = false;
bool nonSeamlessCubeMap = false;
bool conservativeRasterization = false;
VkBuffer emptyBuffer = VK_NULL_HANDLE;
// The device's list of recorded dispatches whose copied written buffers await a CPU write-back
@ -137,14 +152,20 @@ struct Context {
// VK_EXT_descriptor_indexing with non-uniform sampled/storage image array indexing enabled
// (bindless image tables in graphics stages).
bool descriptorIndexing = false;
VkPhysicalDeviceDescriptorIndexingPropertiesEXT descriptorIndexingLimits{};
bool imageInt64Atomics = false;
bool geometryShader = false;
bool sampleRateShading = false;
bool nullDescriptors = false;
bool bufferInt64Atomics = false;
bool primitiveListRestart = false;
bool imageViewMinLod = false;
bool pipelineExecutableInfo = false;
bool graphicsPipelineLibrary = false;
std::uint32_t srgbDecodeFormats = 0;
bool provokingVertexLast = false;
bool provokingVertexModePerPipeline = false;
bool singlePassStorage = false;
template<typename TFunction>
TFunction Function(const char* name) const {
@ -170,6 +191,11 @@ struct Context {
}
};
inline void EndRenderPass(const Context& context, VkCommandBuffer commands) {
if (context.graphicsPipelineLibrary) context.Resolved(&DeviceFunctions::cmdEndRendering, "vkCmdEndRenderingKHR")(commands);
else context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(commands);
}
}
#endif

9
core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp
View File

@ -83,8 +83,17 @@ struct DccKeyProofCounts {
std::uint64_t proved;
std::uint64_t scanned;
std::uint64_t unstable;
std::uint64_t rangeProved;
std::uint64_t rangeScanned;
};
DccKeyProofCounts KeyProofCounts();
struct DccRangeProof {
std::uint64_t address = 0;
std::uint64_t count = 0;
DccKeys keys = DccKeys::Uncompressed;
std::uint64_t generation = 0;
};
DccKeys ProvedCurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes, DccRangeProof& proof);
// A surface's texels as a read sees them: the guest bytes, or the clear value of fast-cleared keys.
void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span<std::byte> bytes);
void NoteKeysFillOnGpu(std::uint64_t begin, std::size_t count, DccKeys keys);

3
core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp
View File

@ -3,6 +3,7 @@
#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
#include "prx/libSceAgcDriver/Graphics/include/State.hpp"
#include <array>
#include <cstdint>
#include <memory>
#include <span>
@ -15,6 +16,8 @@ VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target);
std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel);
void ClearDepthSurfaces(VkDevice device);
bool DepthSurfaceAt(std::uint64_t address);
std::uint64_t HtileDepthClearAddress(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const std::array<std::uint32_t, 3>& numThreads);
void NoteHtileDepthClear(std::uint64_t htileAddress);
std::shared_ptr<Texture> DepthSurfaceTexture(const Context& context, std::span<const std::uint32_t> words, const GuestTextureResource& resource, VkComponentMapping components);
}

5
core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp
View File

@ -12,11 +12,14 @@
namespace AgcDriver::Graphics {
struct HostImport;
// `recipe`, when given, receives the DrawRecipe a recorded, cacheable, reusable, non-indirect draw
// built for its draw-cache entry (design_cpu_final M8; null otherwise, and always under
// APS5_NO_DRAW_RECIPE=1).
void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span<const CompiledShader> shaders, std::span<const GuestMemorySnapshot> snapshots = {}, std::shared_ptr<const DrawRecipe>* recipe = nullptr);
std::optional<std::string> KnownValidationFailure(const Context& context, std::span<const CompiledShader> shaders, const State& state);
std::uint64_t DrawRenderPassKey(const Context& context, const State& state, std::span<const VkImageView> targetViews);
struct DrawInputCopy {
std::shared_ptr<Buffer> buffer;
@ -26,7 +29,9 @@ struct DrawInputCopy {
std::uint64_t registryGeneration = 0;
};
DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uint64_t address, std::size_t bytes, std::size_t alignment, Recorder::SnapshotUse use);
DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes);
void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCopy& copy, Recorder::SnapshotUse use, std::uint32_t derived);
const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment);
std::array<std::uint32_t, 4> MeshIndexBufferDescriptor(const Pm4::DrawParameters& draw);

14
core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp
View File

@ -27,11 +27,12 @@ struct HostImport {
VkDeviceMemory memory;
VkDeviceAddress address;
void* alias = nullptr;
std::weak_ptr<const GuestAllocations::Range> range {};
std::shared_ptr<const GuestAllocations::Range> range {};
// Identity for the life of this import (see HostImportSerial); 0 until first asked for.
std::uint64_t serial = 0;
bool unwatched = false;
bool dmaBuf = false;
std::shared_ptr<void> chunk;
};
enum class ImportWatch : std::uint8_t { Watch, Unwatch };
@ -150,6 +151,8 @@ struct MirrorStats {
std::uint64_t rebuilds = 0;
std::uint64_t blocksCopied = 0;
std::uint64_t heapRefills = 0;
std::uint64_t sweeps = 0;
std::uint64_t heapChecks = 0;
};
MirrorStats MirrorCounters();
void ClearImageMirrors(VkDevice device);
@ -180,13 +183,14 @@ public:
// (DescriptorBinding::bufferWritten): the CPU never has to wait for it. `atomic` marks an
// element the shader updates atomically (DescriptorBinding::bufferAtomic), staged in device
// memory whatever its size (see AllowDeviceStaging).
void AddWritable(std::uint64_t address, std::size_t bytes, bool atomic = false);
void AddWritable(std::uint64_t address, std::size_t bytes, bool atomic = false, bool swept = false);
void AddReadable(std::uint64_t address, std::size_t bytes);
// Device-local staging of written and atomic elements inside host imports (see
// GuestBufferMemory.cpp): allowed only for a build whose every use records its work and then
// calls RecordCopyBacks (a dispatch), since a staged region's results reach guest memory by
// that copy alone. Call before Upload.
void AllowDeviceStaging() { stagingAllowed = true; }
void AllowAdjustedRegions() { adjustedRegions = true; }
// Records the copy-in of every staged region anew for another use of this upload (a resource
// cache hit, from ShaderResources::Revalidate, under GuestMemory::GpuMutex, once the imports
// were confirmed unchanged): the previous use's copy-back left the shadow behind, and the next
@ -248,6 +252,7 @@ public:
// the GPU copies out of an import (gpuCopy) notes its read itself when the copy is recorded.
// For the recorder's read tracking (ShaderResources::MarkGpuWrites); nothing once committed.
std::vector<std::pair<std::uint64_t, std::uint64_t>> InPlaceReads() const;
std::vector<std::pair<std::uint64_t, std::uint64_t>> DeviceReads() const;
private:
struct Region {
@ -290,6 +295,7 @@ private:
bool copiedBack = false;
// An element the shader updates atomically lies inside (AddWritable's `atomic`).
bool atomic = false;
bool swept = false;
// The gpuCopy buffer is a device-local staging shadow (see stagingEligible): no host
// mapping, so nothing is ever stored from it by the CPU, and the region is taken even when
// the import could bind it in place.
@ -323,7 +329,7 @@ private:
// The region of a descriptor-bound range (AddWritable/AddReadable), committed pages only. A
// range inside a base region of the space adds nothing: the region serves it, as today's merge
// of the two did.
void addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic);
void addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic, bool swept);
// Gives a region a buffer of its own with its bytes (guest memory for host-backed and writable
// ranges, plus the write-back's reference copy for a range a descriptor writes; else its snapshot).
void copyRegion(Region& region, bool addressable);
@ -336,12 +342,14 @@ private:
// and an atomic element or a size within the written-shadow window. Independent of the import,
// so UploadPrepare and UploadFinish decide alike.
bool stagingEligible(const Region& region, bool addressable) const;
bool bindableInPlace(std::uint64_t offset, bool addressable) const;
// Records the import-to-buffer copies of the given gpuCopy regions into the open batch, with
// the barriers that order them after earlier recorded writes and before the shaders reading them.
void recordGpuCopies(std::span<Region* const> copies, bool addressable);
void takeHeapReferences();
Context context;
bool stagingAllowed = false;
bool adjustedRegions = false;
GuestAllocations::Lease lease;
// The cached address space this build maps through (its lease pins the ranges); `regions` then
// holds only the regions outside it (V#s, snapshots, ranges copied per build).

7
core/libs/prx/libSceAgcDriver/Graphics/include/GuestSamplerResource.hpp
View File

@ -6,6 +6,8 @@
#endif
#include <vulkan/vulkan.h>
#include <cstdint>
#include <array>
#include <optional>
#include <span>
namespace AgcDriver::Graphics {
@ -24,12 +26,15 @@ struct GuestSamplerResource {
float lodBias;
VkBorderColor borderColor;
VkSamplerReductionMode reductionMode = VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT;
bool forceDegamma = false;
bool compareEnable = false;
VkCompareOp compareOp = VK_COMPARE_OP_NEVER;
bool unnormalizedCoordinates = false;
bool nonSeamlessCube = false;
};
GuestSamplerResource DecodeSamplerResource(std::span<const std::uint32_t> words, bool unnormalizedProven = false);
GuestSamplerResource DecodeSamplerResource(std::span<const std::uint32_t> words, bool unnormalizedProven = false, bool forceDegammaPaired = false);
std::optional<std::array<std::uint32_t, 4>> SingleLevelSamplerWords(std::span<const std::uint32_t, 4> words, bool singleLevelImage, bool mipmappedImage);
}

3
core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp
View File

@ -76,6 +76,8 @@ struct GuestTextureResource {
std::uint8_t dstSelY;
std::uint8_t dstSelZ;
std::uint8_t dstSelW;
std::uint32_t pipeBankXor = 0;
std::uint8_t bcSwizzle = 0;
// Last mip level the view exposes; the surface itself holds mipCount levels.
std::uint32_t lastLevel = 0;
// DCC metadata of a compressed surface, or 0 (see DccMetadata.hpp).
@ -89,6 +91,7 @@ struct GuestTextureResource {
float EffectiveMinLod(const GuestTextureResource& resource);
GuestTextureResource DecodeTextureResource(std::span<const std::uint32_t> words);
bool DescriptorSingleLevel(std::span<const std::uint32_t> words);
bool MatchesGuestDimension(ShaderRecompiler::DescriptorImageShape shape, TextureDimension dimension);
}

22
core/libs/prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp Normal file
View File

@ -0,0 +1,22 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PARALLELCOMPARE_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PARALLELCOMPARE_HPP
#include <cstddef>
#include <cstdint>
#include <span>
namespace AgcDriver::Graphics {
struct CompareSpan {
const void* first;
const void* second;
std::size_t bytes;
};
void CompareSpans(std::span<const CompareSpan> spans, std::span<std::uint8_t> equal);
void CompareSpansFrom(std::span<const CompareSpan> spans, std::span<std::uint8_t> equal, std::size_t parallelBytes);
unsigned CompareHelpers();
}
#endif

67
core/libs/prx/libSceAgcDriver/Graphics/include/PassHazards.hpp Normal file
View File

@ -0,0 +1,67 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PASSHAZARDS_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PASSHAZARDS_HPP
#ifndef VK_NO_PROTOTYPES
#define VK_NO_PROTOTYPES
#endif
#include <vulkan/vulkan.h>
#include <cstddef>
#include <cstdint>
#include <span>
#include <utility>
#include <vector>
namespace AgcDriver::Graphics {
using GuestRangeList = std::span<const std::pair<std::uint64_t, std::uint64_t>>;
struct PassAccess {
GuestRangeList reads;
GuestRangeList writes;
std::span<const std::pair<VkImage, bool>> images;
std::span<const std::pair<VkImage, bool>> attachments;
bool anyReads = false;
bool anyWrites = false;
};
enum class PassHazard : std::uint8_t { None, ReadAfterWrite, WriteAfterWrite, WriteAfterRead, Image, UnknownRead, UnknownWrite };
class GuestRangeSet {
public:
void Insert(std::uint64_t begin, std::uint64_t end);
bool Overlaps(std::uint64_t begin, std::uint64_t end) const;
bool Empty() const { return ranges.empty(); }
std::size_t Size() const { return ranges.size(); }
void Clear() { ranges.clear(); }
private:
std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges;
};
class PassHazards {
public:
PassHazard Check(const PassAccess& access, bool samePass) const;
void Add(const PassAccess& access);
void BeginPass() { ++pass; }
void Clear();
bool Empty() const;
private:
struct Attachment {
VkImage image;
bool written;
std::uint64_t pass;
};
bool overlapsAny(const GuestRangeSet& set, GuestRangeList ranges) const;
GuestRangeSet reads;
GuestRangeSet writes;
std::vector<std::pair<VkImage, bool>> images;
std::vector<Attachment> attachments;
std::uint64_t pass = 0;
bool anyReads = false;
bool anyWrites = false;
};
}
#endif

18
core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp
View File

@ -2,6 +2,7 @@
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINE_HPP
#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp"
#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
#include <set>
namespace AgcDriver::Graphics {
@ -19,12 +20,14 @@ public:
Framebuffer(const Framebuffer&) = delete;
Framebuffer& operator=(const Framebuffer&) = delete;
VkFramebuffer Handle() const { return framebuffer; }
std::span<const VkImageView> Views() const { return views; }
// Forgets the handle without destroying it (the device it belongs to is already gone).
void Abandon() noexcept;
private:
Context context;
VkFramebuffer framebuffer = VK_NULL_HANDLE;
std::vector<VkImageView> views;
};
// The shader modules, layout, render pass and VkPipeline of one draw configuration. Viewport,
@ -48,6 +51,9 @@ public:
// The same inside a render pass another pipeline of the same attachments began (compatible by
// construction: the attachment formats alone decide).
void Continue(VkCommandBuffer commands, const State& state) const;
bool SplitsFaces() const { return backFaces != VK_NULL_HANDLE; }
bool Optimized() const { return optimized != nullptr && optimized->handle.load() != VK_NULL_HANDLE; }
void ContinueBackFaces(VkCommandBuffer commands, const State& state) const;
void PushConstants(VkCommandBuffer commands, VkShaderStageFlags stages, std::span<const std::byte, PipelinePushConstantBytes> bytes) const;
// Forgets the Vulkan objects without destroying them: for entries of a device that is already gone.
void Abandon() noexcept;
@ -60,15 +66,23 @@ private:
std::shared_ptr<Framebuffer> framebuffer;
};
void release() noexcept;
void setLibraryState(VkCommandBuffer commands, const State& state) const;
Context context;
std::vector<VkShaderModule> _modules;
VkPipelineLayout layout = VK_NULL_HANDLE;
VkRenderPass renderPass = VK_NULL_HANDLE;
VkPipeline pipeline = VK_NULL_HANDLE;
VkPipeline backFaces = VK_NULL_HANDLE;
std::size_t attachments = 0;
std::size_t colorAttachments = 0;
bool depthBounds = false;
bool depthBias = false;
bool libraries = false;
std::shared_ptr<OptimizedPipeline> optimized;
bool dynamicRendering = false;
VkFormat depthFormat = VK_FORMAT_UNDEFINED;
VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_UNDEFINED;
std::vector<std::uint32_t> colorSlots;
std::vector<CachedFramebuffer> framebuffers;
};
@ -83,13 +97,15 @@ std::shared_ptr<Pipeline> CachedPipeline(const Context& context, const State& st
// entries of a gone device are recognised by their buffer pool (made and reset with the device, so
// it tells device instances apart when the loader reuses a VkDevice handle) and forgotten unused.
void ClearCachedPipelines(VkDevice device);
bool HasStencil(VkFormat format);
// Device limit checks of the viewport, which is dynamic state and so no longer checked by Pipeline.
void ValidateViewport(const Context& context, const VkViewport& viewport);
void ValidateDepthBounds(const Context& context, const State& state);
void ValidateProvokingVertex(const Context& context, const State& state, std::span<const CompiledShader> shaders = {});
void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment);
// Returns the color attachment locations the pixel shader writes.
std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false);
std::set<std::uint32_t> ValidateShaders(std::span<const CompiledShader> shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false, bool bufferInt64Atomics = false);
}

50
core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp Normal file
View File

@ -0,0 +1,50 @@
#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINELIBRARY_HPP
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINELIBRARY_HPP
#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
#include <array>
#include <atomic>
#include <memory>
#include <cstddef>
#include <cstdint>
#include <span>
#include <vector>
namespace AgcDriver::Graphics {
struct PipelineLibraryKeys {
std::vector<std::byte> vertexInput;
std::vector<std::byte> preRasterization;
std::vector<std::byte> fragmentShader;
std::vector<std::byte> fragmentOutput;
std::vector<std::byte> renderPass;
std::vector<std::byte> layout;
};
std::span<const VkDynamicState> PipelineLibraryDynamicStates();
struct OptimizedPipeline {
VkDevice device = VK_NULL_HANDLE;
PFN_vkDestroyPipeline destroy = nullptr;
std::atomic<VkPipeline> handle{VK_NULL_HANDLE};
std::atomic<bool> released{false};
void Release() noexcept;
};
VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr<OptimizedPipeline>* optimized = nullptr);
void WaitForOptimizedPipelines(VkDevice device);
void ClearPipelineLibraries(VkDevice device);
struct PipelineLibraryCounters {
std::array<std::uint64_t, 4> built{};
std::uint64_t linked = 0;
std::uint64_t optimized = 0;
};
PipelineLibraryCounters PipelineLibraryCountersOf(VkDevice device);
}
#endif

78
core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp
View File

@ -2,6 +2,7 @@
#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RECORDER_HPP
#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
#include "prx/libSceAgcDriver/Graphics/include/PassHazards.hpp"
#include <atomic>
#include <chrono>
#include <cstddef>
@ -58,16 +59,13 @@ public:
// destination access mask.
void MarkCovered(VkAccessFlags access);
void MarkShaderReadsCovered() { MarkCovered(VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); }
// A recorded draw's render pass (Draw.cpp) is left open after the draw: the next draw of the
// same attachments (`key`: the views and the extent) continues it when nothing was recorded in
// between and the earlier draw allowed it (`continuable`: it wrote nothing but its
// attachments, so no barrier is owed inside the pass), and anything else recorded first ends
// it (vkCmdEndRenderPass, one trailing barrier for the pass, the end of the draw class range
// `timing`): Commands(), RecordStore and Submit end it; CommandsInRenderPass hands a
// continuing draw the command buffer without ending it.
bool ContinuesRenderPass(std::uint64_t key) const;
VkCommandBuffer CommandsInRenderPass();
void LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool continuable, std::function<void(VkCommandBuffer)> afterPass = {});
struct DrawPassStart {
VkCommandBuffer commands = VK_NULL_HANDLE;
bool continued = false;
bool barrier = true;
};
DrawPassStart StartDrawPass(std::uint64_t key, bool forced, bool fenced, const PassAccess& access);
void LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool fenced, const PassAccess& access, std::function<void(VkCommandBuffer)> afterPass = {});
// DCC "uncompressed" key stores (DccMetadata.cpp StoreUncompressedOnGpu): queued on the open
// batch and recorded as one run (one barrier pair for every queued fill) at Submit, before a
// label store (RecordStore), or before a command that writes or reads a queued range (the
@ -90,12 +88,12 @@ public:
static constexpr std::size_t KeptBytesBudget = std::size_t{512} << 20u;
void BoundKeptBytes();
std::size_t InFlightKeptBytes() const { return inFlightKeptBytes; }
enum class SnapshotUse : std::uint8_t { Storage, Vertex, Index16, Index32 };
static constexpr std::size_t DrawSnapshotBudget = std::size_t{256} << 20u;
enum class SnapshotUse : std::uint8_t { Storage, Vertex, Index16, Index32, Index16Restart, Index32Restart };
static constexpr std::size_t DrawSnapshotBudget = std::size_t{1024} << 20u;
static constexpr std::size_t DrawSnapshotEntries = 1024;
static constexpr std::size_t DrawInputBudget = std::size_t{1024} << 20u;
static constexpr std::size_t DrawInputEntries = 16384;
std::shared_ptr<Buffer> ReusableDrawSnapshot(std::uint64_t address, std::size_t bytes, SnapshotUse use = SnapshotUse::Storage, std::uint32_t* derived = nullptr);
std::shared_ptr<Buffer> ReusableDrawSnapshot(std::uint64_t address, std::size_t bytes, SnapshotUse use = SnapshotUse::Storage, std::uint32_t* derived = nullptr, std::uint64_t generation = 0);
void KeepDrawSnapshot(std::uint64_t address, std::size_t bytes, std::uint64_t generation, std::uint64_t registryGeneration, std::shared_ptr<Buffer> buffer, SnapshotUse use = SnapshotUse::Storage, std::uint32_t derived = 0);
void OnComplete(std::function<void()> action);
void NotePendingWrite(std::uint64_t address, std::size_t bytes);
@ -124,7 +122,7 @@ public:
// on the open batch, no snapshot: the only reader holds the mutex. `kind` names the reader for
// the [recorder] hit counters. APS5_COPY_READ_TRACKING=0 notes nothing (ReadTracking() is then
// false and the CPU copy falls back to Idle()).
enum class ReadKind : std::uint8_t { DispatchElement = 0, GpuCopy, AddressBased, Indirect, StorageUpload, CopySource, Count };
enum class ReadKind : std::uint8_t { DispatchElement = 0, GpuCopy, AddressBased, Indirect, StorageUpload, CopySource, DrawInput, Count };
static bool ReadTracking();
void NotePendingRead(std::uint64_t address, std::size_t bytes, ReadKind kind);
void NotePendingReads(std::span<const std::pair<std::uint64_t, std::uint64_t>> ranges, ReadKind kind);
@ -184,8 +182,25 @@ public:
void CountSamples();
bool RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress record, VkDeviceAddress arguments, std::span<const std::uint32_t, 7> rules);
std::uint64_t SamplesTotal();
bool DumpSamples(VkDeviceAddress target);
void NoteSampledDraw();
bool DumpSamples(VkDeviceAddress target, std::uint64_t address = 0);
void NoteSampledDraw(VkCommandBuffer commands);
bool QueuesSampleDumps() const;
void PrepareSampleSlot();
void EndPassSamples();
struct SampleDumpStatistics {
std::uint64_t queued;
std::uint64_t queuedInPass;
std::uint64_t recordedAtOnce;
std::uint64_t segments;
std::uint64_t folds;
};
static SampleDumpStatistics SampleDumpCounts();
struct QueuedWrite {
std::uint64_t begin;
std::uint64_t end;
bool computed;
};
static std::vector<std::size_t> QueuedWriteGroups(std::span<const QueuedWrite> writes);
// Waits only for the batches up to the newest one that writes the range (submitting the open
// batch when it is that one); later batches stay in flight. Fences of one queue signal in
// submission order, so completions still run in order. Debug aid: APS5_NO_SYNC_THROUGH=1 syncs all.
@ -519,6 +534,12 @@ private:
std::shared_ptr<void> samplePool;
bool sampleActive = false;
bool samplesDrawn = false;
VkQueryPool segments = VK_NULL_HANDLE;
std::shared_ptr<void> segmentPool;
std::uint32_t segmentNext = 0;
std::uint32_t segmentSlot = 0;
bool segmentActive = false;
std::uint32_t queuedDumps = 0;
std::vector<std::uint64_t> timedKeys;
std::vector<std::uint64_t> timedBytes;
// The whole-batch timed range (BatchTimingKey) and its stamps once read (see Completed).
@ -560,6 +581,9 @@ private:
VkDeviceSize offset;
std::vector<std::byte> bytes;
std::uint64_t address;
VkDeviceAddress dumpTarget = 0;
std::uint32_t dumpSegments = 0;
std::uint64_t End() const;
};
std::vector<Queued> queued;
} run;
@ -581,14 +605,14 @@ private:
};
std::vector<Image> images;
} tracker;
// The render pass a recorded draw left open (see ContinuesRenderPass).
struct RenderPass {
bool open = false;
bool continuable = false;
bool fenced = false;
std::uint64_t key = 0;
std::uint32_t timing = NoTiming;
std::function<void(VkCommandBuffer)> afterPass;
} renderPass;
PassHazards passHazards;
// A pass ended in this batch: Submit records the host-read barrier its draws left out.
bool hostReadOwed = false;
// Queued DCC key stores (see QueueKeyStore).
@ -622,9 +646,21 @@ private:
bool gpuSampleCounter();
void endSamples(Batch& batch);
void foldSamples(Batch& batch, VkDeviceAddress target);
bool segmentMode() const;
bool segmentSlotReady(const Batch& batch) const;
void takeSegmentPool(Batch& batch);
void beginSegment(Batch& batch);
void endSegment(Batch& batch);
void copySegments(VkCommandBuffer commands, std::uint32_t count);
void retireSegments(std::uint32_t count);
void recordQueuedDumps(VkCommandBuffer commands, std::span<const Batch::StoreRun::Queued> writes);
void dispatchDumps(VkCommandBuffer commands, std::uint32_t firstDump, std::uint32_t dumps, std::uint32_t firstSegment, std::uint32_t lastSegment);
void foldSegments();
void foldQueuedDumps(VkCommandBuffer commands, std::span<const Batch::StoreRun::Queued> writes, std::size_t first, std::size_t end, std::uint32_t& dumpIndex, std::uint32_t& consumed);
struct SampleSegment {
std::shared_ptr<void> pool;
VkQueryPool handle;
std::uint32_t slot = 0;
};
std::vector<SampleSegment> pendingSamples;
bool countingSamples = false;
@ -636,6 +672,9 @@ private:
VkPipelineLayout sampleLayout = VK_NULL_HANDLE;
VkPipeline samplePipeline = VK_NULL_HANDLE;
std::shared_ptr<void> samplePools;
VkPipelineLayout dumpLayout = VK_NULL_HANDLE;
VkPipeline dumpPipeline = VK_NULL_HANDLE;
std::shared_ptr<void> segmentPools;
// BeginGpuTiming on the open batch without Commands() (RecordStore times its own run, which
// Commands() would close).
std::uint32_t beginTiming(std::uint64_t key);
@ -673,7 +712,7 @@ private:
// form, toward the host only); returns whether the batch's host-read barrier was included.
bool closeStoreRun(bool atSubmit = false);
// Ends the render pass a draw left open (vkCmdEndRenderPass, the pass's trailing barrier).
void endOpenRenderPass();
void endOpenRenderPass(bool barrier = true);
// Records the queued key stores as one run (`forWriter`: before a command writing, reading or
// labelling over one, not at Submit).
void recordKeyStores(bool forWriter);
@ -763,6 +802,7 @@ private:
std::map<DrawSnapshotKey, DrawSnapshot> drawSnapshots;
std::array<DrawSnapshotPool, 2> drawSnapshotPools;
void eraseDrawSnapshot(std::map<DrawSnapshotKey, DrawSnapshot>::iterator entry);
bool refreshDrawSnapshot(DrawSnapshot& entry, std::uint64_t address, std::size_t bytes);
};
}

2
core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp
View File

@ -35,6 +35,8 @@ private:
// the handles a failed construction left.
bool ready = false;
VkDeviceSize allocationBytes = 0;
VkDeviceSize offset = 0;
bool slab = false;
VkBufferUsageFlags usage;
VkMemoryPropertyFlags properties;
std::shared_ptr<BufferPool> cache;

11
core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp
View File

@ -19,15 +19,21 @@ public:
Sampler(const Sampler&) = delete;
Sampler& operator=(const Sampler&) = delete;
bool ForcesDegamma() const;
VkSampler Handle() const;
bool RequiresFilterMinmax() const;
bool ReadsOpaqueBlackBorder() const;
bool RequiresNonSeamlessCube() const;
private:
void release() noexcept;
Context context;
bool forcesDegamma = false;
VkSampler sampler = VK_NULL_HANDLE;
bool requiresFilterMinmax = false;
bool opaqueBlackBorder = false;
bool requiresNonSeamlessCube = false;
};
// One VkSampler per distinct S# (its 4 words plus the shader's depth-compare use and the
@ -41,7 +47,7 @@ public:
explicit SamplerCache(std::size_t capacity = 1024);
SamplerCache(const SamplerCache&) = delete;
SamplerCache& operator=(const SamplerCache&) = delete;
std::shared_ptr<Sampler> Get(const Context& context, std::span<const std::uint32_t> words, bool compareEnable, bool unnormalizedProven = false);
std::shared_ptr<Sampler> Get(const Context& context, std::span<const std::uint32_t> words, bool compareEnable, bool unnormalizedProven = false, bool forceDegammaPaired = false);
// APS5_PROFILE_DRAW counters: lookups served by an existing sampler, and samplers created.
std::uint64_t Hits() const { return hits; }
std::uint64_t Misses() const { return misses; }
@ -60,6 +66,9 @@ private:
};
void RequireFilterMinmax(const Context& context, VkFormat format, std::uint32_t samplerMask, std::span<const std::shared_ptr<Sampler>> samplers);
void RequireDegammaFormat(std::uint32_t guestFormat, std::uint32_t samplerMask, std::span<const std::shared_ptr<Sampler>> samplers);
void RequireBorderSwizzle(std::uint32_t bcSwizzle, std::uint32_t samplerMask, std::span<const std::shared_ptr<Sampler>> samplers);
void RequireNonSeamlessCube(bool cube, std::uint32_t samplerMask, std::span<const std::shared_ptr<Sampler>> samplers);
}

43
core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp
View File

@ -37,6 +37,7 @@ struct TextureCacheUse {
TextureCacheUse TextureCacheUsage();
std::uint64_t SampledTextureCacheBudget(const Context& context);
std::shared_ptr<Texture> CachedSampledTexture(const Context& context, std::span<const std::uint32_t> words);
bool SampledTexturesShareEntry(std::span<const std::uint32_t> first, std::span<const std::uint32_t> second);
// The cached storage image of a surface (render targets use it as their resident image); brought up
// to date with guest memory before it is returned.
@ -68,12 +69,12 @@ public:
// The layout for `key` (binding, type, count, stage flags per binding, as ShaderResources builds
// it from `bindings`), created on first use.
VkDescriptorSetLayout Layout(std::span<const std::uint32_t> key, std::span<const VkDescriptorSetLayoutBinding> bindings);
VkDescriptorSetLayout Layout(std::span<const std::uint32_t> key, std::span<const VkDescriptorSetLayoutBinding> bindings, bool updateAfterBind = false);
struct SetAllocation {
VkDescriptorSet set = VK_NULL_HANDLE;
VkDescriptorPool pool = VK_NULL_HANDLE;
};
SetAllocation Allocate(VkDescriptorSetLayout layout, std::span<const VkDescriptorPoolSize> sizes);
SetAllocation Allocate(VkDescriptorSetLayout layout, std::span<const VkDescriptorPoolSize> sizes, bool updateAfterBind = false);
void Free(const SetAllocation& allocation) noexcept;
// APS5_PROFILE_DRAW counters: layouts served from the map / created, sets allocated, pools opened.
struct Stats {
@ -93,6 +94,7 @@ private:
mutable std::mutex mutex;
std::map<std::vector<std::uint32_t>, VkDescriptorSetLayout> layouts;
std::vector<VkDescriptorPool> pools;
std::vector<VkDescriptorPool> updateAfterBindPools;
std::vector<VkDescriptorPool> dedicated;
Stats stats;
};
@ -100,14 +102,14 @@ private:
class ShaderResources {
public:
ShaderResources(const Context& context, const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes);
ShaderResources(const Context& context, std::span<const CompiledShader> shaders, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes, std::span<const GuestMemorySnapshot> snapshots = {});
ShaderResources(const Context& context, std::span<const CompiledShader> shaders, const ColorTarget& target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::size_t indexBytes, std::span<const GuestMemorySnapshot> snapshots = {});
ShaderResources(const Context& context, const CompiledShader& compute, std::span<const GuestMemorySnapshot> snapshots = {});
// Two-stage build for dispatches (see build): with `deferred` the constructor runs stage A only,
// which needs no device lock, and Complete() runs stage B under GuestMemory::GpuMutex; until
// then no other member may be used. `compute` and `snapshots` must outlive Complete(). An
// address-based shader (BDA tables) builds entirely in Complete(): its lease acquisition
// reconciles imports and refreshes mirrors, which needs the lock.
ShaderResources(const Context& context, const CompiledShader& compute, std::span<const GuestMemorySnapshot> snapshots, bool deferred);
ShaderResources(const Context& context, const CompiledShader& compute, std::span<const GuestMemorySnapshot> snapshots, bool deferred, std::uint64_t dispatchThreads = 0);
void Complete();
bool Completed() const { return completed; }
~ShaderResources();
@ -155,11 +157,7 @@ public:
const std::vector<std::pair<std::uint64_t, std::uint64_t>>& GpuWrites() const { return guestMemory.Writes(); }
std::vector<std::pair<std::uint64_t, std::uint64_t>> InPlaceReads() const { return guestMemory.InPlaceReads(); }
std::vector<std::pair<VkImage, bool>> StorageImages() const;
// Whether a use writes guest memory beyond a draw's attachments (storage images, written or
// copied buffers, an address-based build's unknown writes), or reads `image` (a view of it
// sampled, or the image itself bound): a recorded draw's render pass may only be continued by
// a draw for which neither holds (Draw.cpp).
bool WritesMemory() const;
std::vector<std::pair<std::uint64_t, std::uint64_t>> DeviceReads() const { return guestMemory.DeviceReads(); }
bool ReadsImage(const StorageTexture* image) const;
const std::vector<std::uint32_t>& LayoutKey() const { return layoutKey; }
// Debug aid: each bound guest resource with the fraction of sampled bytes that are nonzero.
@ -186,6 +184,8 @@ public:
// ranges are fixed by the build), so a Revalidate's collects on the same worker are memo hits.
void PrecollectSurfaces() const;
bool Reusable() const { return reusable; }
static bool NeverReusable(std::span<const CompiledShader> shaders);
const std::vector<std::shared_ptr<Texture>>& SampledTextures() const { return textures; }
// `shaders` are the stages the object was built from, in build order (a recorded draw's vertex
// and fragment stages, or one compute stage): their bindings are walked like the build did.
// How a Revalidate proved (or refused) the object, for the [recipe] line: the proof path taken
@ -294,15 +294,15 @@ private:
// GuestMemory::GpuMutex): the texture and storage image lookups (they refresh, upload and flush
// through the recorder), the rest of the upload, the BDA objects, the descriptor writes and the
// reusability record. build runs both.
void build(std::span<const CompiledShader> shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes);
void buildPrepare(std::span<const CompiledShader> shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes);
void build(std::span<const CompiledShader> shaders, const ColorTarget* target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::size_t indexBytes);
void buildPrepare(std::span<const CompiledShader> shaders, const ColorTarget* target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::size_t indexBytes);
void buildComplete();
// APS5_PROFILE_DRAW: closes the current sub-phase of the build into the [resources] totals.
double phase(BuildPhase which);
// `written` is the element's DescriptorBinding::bufferWritten: a read-only element binds the
// same way but is left out of the write set (no write-back, no pending-write note). `atomic`
// is its bufferAtomic (see GuestBufferMemory::AddWritable).
std::size_t addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic);
std::size_t addGuestBuffer(std::span<const std::uint32_t> words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read);
std::size_t addDataBuffer(std::span<const std::uint32_t> words);
// Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache
// locks itself); stage B looks the sampled textures and storage images up (resolveImageBinding).
@ -335,12 +335,19 @@ private:
std::uint64_t entryGeneration = 0;
};
std::vector<ImageRecord> imageRecords;
struct PreviousSampled {
std::array<std::uint32_t, 8> words{};
bool depthCompare = false;
std::shared_ptr<Texture> texture;
};
PreviousSampled previousSampled;
// The next record resolveImageBinding consumes (records follow the deferredImages order).
std::size_t nextImageRecord = 0;
// Stage B: the record's texture when the fastRevalidate predicate proves it current under the
// lock and the cache still holds it; null sends the element to cachedTexture.
std::shared_ptr<Texture> fastTexture(const ImageRecord& record);
void resolveImageBinding(const ShaderRecompiler::DescriptorBinding& binding, Binding& item, std::span<const std::shared_ptr<Sampler>> shaderSamplers);
void applyAnisoOverride();
void forgetDeferredInputs();
void release() noexcept;
void prepareAddressBindings(std::span<const CompiledShader> shaders, std::span<const GuestMemorySnapshot> snapshots);
@ -414,9 +421,12 @@ private:
// The cache pool the set was allocated from, freed back to it on release.
VkDescriptorPool cachePool = VK_NULL_HANDLE;
std::vector<Allocation> allocations;
std::vector<VkDescriptorBufferInfo> allocationDescriptors;
// Guest buffer elements bound read-only: each use of this object skips that many pending-write
// notes (counted in MarkGpuWrites for the [buffers] line).
std::size_t readOnlyBuffers = 0;
bool drawBuild = false;
std::uint64_t dispatchThreads = 0;
std::vector<std::shared_ptr<Texture>> textures;
std::vector<bool> textureFirstLayer;
std::vector<std::shared_ptr<StorageTexture>> storageTextures;
@ -428,6 +438,14 @@ private:
std::vector<bool> storageAtomic64;
std::vector<std::shared_ptr<Sampler>> samplers;
std::shared_ptr<Sampler> paddingSampler;
struct SamplerSource {
std::array<std::uint32_t, 4> words{};
bool compareEnable = false;
bool unnormalized = false;
bool singleLevelImage = false;
bool mipmappedImage = false;
};
std::vector<SamplerSource> samplerSources;
bool reusable = false;
std::vector<DirectRegion> directRegions;
std::vector<ValidatedSurface> validatedTextures;
@ -457,6 +475,7 @@ private:
};
std::vector<DeferredImages> deferredImages;
std::uint64_t storageBuffers = 0;
bool updateAfterBind = false;
std::uint32_t plannedSampledImages = 0;
std::uint32_t plannedStorageImages = 0;
// The compute constructor's shader and captured regions: the caller's objects, valid only until

27
core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp
View File

@ -9,7 +9,29 @@
namespace AgcDriver::Graphics {
inline constexpr std::uint32_t PipelinePushConstantBytes = 128;
inline constexpr std::uint32_t PipelinePushConstantBytes = 256;
inline constexpr std::uint32_t PipelinePushSlotBytes = 128;
inline constexpr std::uint32_t ComputePushConstantBytes = 128;
inline std::uint32_t PushBlockBytes(bool libraries) {
return libraries ? PipelinePushConstantBytes : PipelinePushSlotBytes;
}
inline VkShaderStageFlags PreRasterizationPushStages(bool meshShader, bool tessellation, bool geometry) {
VkShaderStageFlags stages = VK_SHADER_STAGE_VERTEX_BIT;
if (tessellation) stages |= VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT;
if (geometry) stages |= VK_SHADER_STAGE_GEOMETRY_BIT;
if (meshShader) stages |= VK_SHADER_STAGE_MESH_BIT_EXT;
return stages;
}
inline std::uint32_t FixedPushOffset(ShaderRecompiler::ShaderStage stage) {
return stage == ShaderRecompiler::ShaderStage::Fragment ? PipelinePushSlotBytes : 0u;
}
inline std::uint32_t StagePushOffset(std::uint32_t cursor, ShaderRecompiler::ShaderStage stage, bool fixedSlots) {
return fixedSlots && stage == ShaderRecompiler::ShaderStage::Fragment ? PipelinePushSlotBytes : cursor;
}
struct CompiledShader {
ShaderRecompiler::ShaderStage stage;
@ -23,6 +45,7 @@ inline VkShaderStageFlagBits VulkanStage(ShaderRecompiler::ShaderStage stage) {
case ShaderRecompiler::ShaderStage::Local: return VK_SHADER_STAGE_VERTEX_BIT;
case ShaderRecompiler::ShaderStage::TessellationControl: return VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT;
case ShaderRecompiler::ShaderStage::TessellationEvaluation: return VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT;
case ShaderRecompiler::ShaderStage::Geometry: return VK_SHADER_STAGE_GEOMETRY_BIT;
case ShaderRecompiler::ShaderStage::Mesh: return VK_SHADER_STAGE_MESH_BIT_EXT;
case ShaderRecompiler::ShaderStage::Fragment: return VK_SHADER_STAGE_FRAGMENT_BIT;
case ShaderRecompiler::ShaderStage::Compute: return VK_SHADER_STAGE_COMPUTE_BIT;
@ -37,6 +60,7 @@ inline VkPipelineStageFlags PipelineStages(std::span<const CompiledShader> shade
case VK_SHADER_STAGE_VERTEX_BIT: result |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT; break;
case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT: result |= VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT; break;
case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT: result |= VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT; break;
case VK_SHADER_STAGE_GEOMETRY_BIT: result |= VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT; break;
case VK_SHADER_STAGE_MESH_BIT_EXT: result |= VK_PIPELINE_STAGE_MESH_SHADER_BIT_EXT; break;
case VK_SHADER_STAGE_FRAGMENT_BIT: result |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; break;
default: throw std::runtime_error("AGC graphics: invalid graphics pipeline stage");
@ -63,6 +87,7 @@ inline std::array<std::byte, PipelinePushConstantBytes> AssemblePushConstants(st
if (bytes.empty()) continue;
Require(bytes.size() % 4 == 0 && shader.pushConstantOffset % 4 == 0, "shader push constant range is not DWORD aligned");
Require(shader.pushConstantOffset < PipelinePushConstantBytes && bytes.size() <= PipelinePushConstantBytes - shader.pushConstantOffset, "shader push constant range lies outside the pipeline push constant block");
Require(bytes.size() <= PipelinePushSlotBytes - shader.pushConstantOffset % PipelinePushSlotBytes, "shader push constant range crosses a push constant slot");
for (std::size_t i = 0; i < bytes.size(); ++i) {
const auto position = shader.pushConstantOffset + i;
Require(!occupied[position], "shader push constant ranges of different stages overlap");

16
core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp
View File

@ -38,6 +38,7 @@ struct ColorTarget {
std::uint8_t componentMapping;
ColorTileMode tileMode = ColorTileMode::Linear;
std::uint32_t elementBytes = 4;
ShaderRecompiler::ColorExportPacking packing = ShaderRecompiler::ColorExportPacking::None;
// DCC metadata of a compressed target (CB_COLOR_INFO DCC_ENABLE), or 0 (see DccMetadata.hpp).
std::uint64_t dccAddress = 0;
bool dccAlphaOnMsb = false;
@ -54,6 +55,8 @@ struct ColorTarget {
std::uint32_t depth = 1;
std::uint32_t depthSlice = 0;
std::uint32_t exportIndex = 0;
bool uintExport = false;
std::uint32_t pipeBankXor = 0;
};
struct DepthTarget {
@ -63,6 +66,7 @@ struct DepthTarget {
VkFormat format;
float clearDepth;
std::uint8_t clearStencil;
std::uint64_t htileAddress = 0;
};
struct State {
@ -78,6 +82,9 @@ struct State {
float depthBiasConstant = 0.0f;
float depthBiasSlope = 0.0f;
float depthBiasClamp = 0.0f;
bool depthBiasPerFace = false;
float backDepthBiasConstant = 0.0f;
float backDepthBiasSlope = 0.0f;
bool stencilTest = false;
VkStencilOpState stencilFront{};
VkStencilOpState stencilBack{};
@ -86,6 +93,7 @@ struct State {
std::vector<VkPipelineColorBlendAttachmentState> blends;
bool hasColorTarget;
bool rectList = false;
bool dualSourceBlend = false;
VkExtent2D renderExtent;
VkPrimitiveTopology topology;
bool primitiveRestart = false;
@ -96,6 +104,7 @@ struct State {
VkRect2D scissor;
VkCullModeFlags cullMode;
VkFrontFace frontFace;
VkProvokingVertexModeEXT provokingVertexMode = VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT;
VkPipelineColorBlendAttachmentState blend;
std::array<float, 4> blendConstants;
};
@ -103,6 +112,7 @@ struct State {
ShaderStages DecodeShaderStages(const QueueState& queue);
State DecodeState(const QueueState& queue);
std::array<std::uint8_t, 8> ExportMappings(const State& state);
std::array<ShaderRecompiler::ColorExportPacking, 8> ExportPackings(const State& state);
ColorTarget DecodeColorBuffer(const Registers& context, std::uint32_t slot);
std::size_t CmaskBytes(std::uint32_t width, std::uint32_t height);
std::uint32_t ColorWriteMask(const Registers& context);
@ -151,8 +161,8 @@ struct DrawKeyRange {
std::uint32_t first;
std::uint32_t count;
};
inline constexpr std::array<DrawKeyRange, 45> DrawKeyRegisters{{
{RegisterBank::Context, 0x000, 1}, {RegisterBank::Context, 0x002, 1}, {RegisterBank::Context, 0x007, 7}, {RegisterBank::Context, 0x010, 6}, {RegisterBank::Context, 0x01a, 4},
inline constexpr std::array<DrawKeyRange, 47> DrawKeyRegisters{{
{RegisterBank::Context, 0x000, 1}, {RegisterBank::Context, 0x002, 1}, {RegisterBank::Context, 0x005, 1}, {RegisterBank::Context, 0x007, 7}, {RegisterBank::Context, 0x010, 6}, {RegisterBank::Context, 0x01a, 5},
{RegisterBank::Context, 0x080, 4}, {RegisterBank::Context, 0x08c, 4}, {RegisterBank::Context, 0x090, 2}, {RegisterBank::Context, 0x094, 2}, {RegisterBank::Context, 0x0b4, 2}, {RegisterBank::Context, 0x105, 4}, {RegisterBank::Context, 0x10b, 3}, {RegisterBank::Context, 0x10f, 6},
// SPI_PS_INPUT_CNTL_0..31, SPI_PS_INPUT_ENA/ADDR, SPI_PS_IN_CONTROL, SPI_SHADER_POS/Z/COL_FORMAT,
// CB_BLEND0..7_CONTROL, GE_MAX_OUTPUT_PER_SUBGROUP.
@ -161,7 +171,7 @@ inline constexpr std::array<DrawKeyRange, 45> DrawKeyRegisters{{
// PA_SU_VTX_CNTL, the sample masks, PA_SC_CONSERVATIVE_RASTERIZATION_CNTL.
{RegisterBank::Context, 0x200, 8}, {RegisterBank::Context, 0x292, 2}, {RegisterBank::Context, 0x29b, 1}, {RegisterBank::Context, 0x2ab, 1}, {RegisterBank::Context, 0x2ce, 1}, {RegisterBank::Context, 0x2d5, 2}, {RegisterBank::Context, 0x2db, 2}, {RegisterBank::Context, 0x2de, 6}, {RegisterBank::Context, 0x2f8, 2}, {RegisterBank::Context, 0x30e, 2}, {RegisterBank::Context, 0x313, 1},
// CB_COLOR0..7_BASE .. DCC_BASE (15 words a slot), CB_COLOR0..7_BASE_EXT, DCC_BASE_EXT, ATTRIB2, ATTRIB3.
{RegisterBank::Context, 0x318, 0x78}, {RegisterBank::Context, 0x390, 8}, {RegisterBank::Context, 0x3a8, 0x18},
{RegisterBank::Context, 0x318, 0x78}, {RegisterBank::Context, 0x390, 8}, {RegisterBank::Context, 0x398, 8}, {RegisterBank::Context, 0x3a8, 0x18},
// The pixel program address, RSRC2 and user words; the geometry-back user pointer and program
// address; the vertex/geometry-front RSRC1/RSRC2 and user words; the vertex program address;
// the hull user pointer, program address, RSRC2 and user words; the local program address.

39
core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp
View File

@ -35,6 +35,7 @@ bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat);
// StorageTexture::upload); false for integer formats and non-clear keys.
bool StorageClearAvailable(const Context& context, std::uint32_t guestFormat, DccKeys keys);
std::uint64_t SampledTextureMemory();
bool ClearKeepsDenormals(const Context& context, VkFormat format);
// A sampled texture's own VkImage with its memory, shared with the recorder while a recorded upload
// still writes it (see the snapshot constructor), so the texture may go before the batch completes.
@ -129,6 +130,7 @@ public:
VkImageView View(std::uint32_t mip);
VkImageView FirstLayerView(std::uint32_t mip);
VkImageView StorageView(std::uint32_t mip, bool firstLayer);
VkImageView ElementView();
VkImageView AtomicView(std::uint32_t mip, bool firstLayer);
VkImageView Atomic64View(std::uint32_t mip, bool firstLayer);
// Render targets live in the same images: draws attach mip 0 through a view of the color
@ -151,6 +153,7 @@ public:
// overlapping pending image, as before).
static bool FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except = nullptr, const char* reason = "memory access", PublishScope scope = PublishScope::Whole, bool* published = nullptr);
static void FlushAllPending(const char* reason);
static bool StoreAtFlipRequested(const char* value);
// See PendingSerial: a change of a surface's source outside the registry (a unit shadow
// retile) moves it too.
static void BumpPendingSerial();
@ -228,8 +231,9 @@ public:
// `image` and `layer` naming it, whatever other images lie over the range (`others`, of which
// `inside` wholly inside it: stale images of earlier uses of the memory, which the title's
// transient allocator hands out again); otherwise part of one image (the fill inside the
// surface, the surface inside the fill, or straddling), several, or none but exactly one
// surface's DCC metadata. Images the cache let go (see Flush) do not count.
// surface, the surface inside the fill, or straddling), several, or a fill of one surface's DCC
// keys (Keys: at its dccAddress and at least one key long, and while other images overlap it no
// longer than its key extent; see keysFillMatches). Images the cache let go (see Flush) do not count.
enum class FillCover { None, Exact, Inside, Around, Straddle, Several, Keys, Layer };
struct FillCoverage {
FillCover cover = FillCover::None;
@ -285,12 +289,15 @@ public:
DccKeys UploadedKeys() const { return uploadedKeys; }
DccKeys FilledKeys() const { return filledKeys; }
DccKeyProof& KeyProof() const { return keyProof; }
DccRangeProof& TargetKeyProof() const { return targetKeyProof; }
DccKeys ProvedKeys() const;
bool ServesKeysAt(std::uint64_t dccAddress) const;
// Brings the image up to date with guest memory before another use; returns whether its content
// was still current (nothing uploaded).
// Keeps the image current with guest memory (see GuestMemory::CollectWrites).
bool Refresh();
static std::uint64_t SinglePassMoves();
static std::uint64_t RefreshesProved();
std::uint64_t GuestBytes() const;
VkDeviceSize AllocationBytes() const { return memoryBytes; }
@ -309,6 +316,7 @@ private:
// APS5_BLOCK_WRITEBACK_EACH=1 stores the touched units only).
void writeBack(std::uint64_t address, std::size_t bytes);
void writeBackLayers(const std::vector<bool>& layers);
bool unchangedSinceBaseline(std::uint64_t from, std::uint64_t to) const;
// Tracked units as 64 KiB write-stamp blocks (`blockUnits`: a thin tiled surface at a 64 KiB
// aligned base; APS5_NO_BLOCK_TRACKING=1 tracks array layers as above instead): a fill of one
// layer, a CPU write or another image's store then costs the blocks it touched, moved through
@ -368,6 +376,9 @@ private:
std::uint64_t layerBegin(std::uint32_t layer) const { return descriptor.baseAddress + static_cast<std::uint64_t>(layer) * trackedLayerBytes; }
bool anyLayerPending() const;
void refreshGeneration();
bool refreshProved();
void takeRefreshProof(bool aliased);
bool otherPendingOverlaps() const;
// Marks `count` tracked layers from `first` pending and registers the image (MarkDirty's
// registration; APS5_EAGER_WRITEBACK=1 stores at once instead).
void markLayersPending(std::uint32_t first, std::uint32_t count);
@ -417,8 +428,18 @@ private:
void forgetBorrowed(std::uint32_t first, std::uint32_t count);
bool clearByKeysFill(DccKeys keys, std::uint8_t key);
bool overlaps(std::uint64_t address, std::size_t bytes) const;
// Whether the image is live (not released) and overlaps the fill: the test that gives ClassifyFill
// its overlapping images, and that NoteKeysFill and ClearByKeysFill use to bound the key match.
bool overlapsLive(std::uint64_t address, std::size_t bytes) const;
// Whether a fill of [address, address + bytes) is a fill of this image's DCC keys, `overlapped`
// saying whether any live image overlaps the fill (see Texture.cpp).
bool keysFillMatches(std::uint64_t address, std::size_t bytes, bool overlapped) const;
bool pendingUnitInside(std::uint64_t address, std::size_t bytes) const;
VkImageView createView(std::uint32_t mip, bool firstLayer, VkFormat format) const;
bool singlePass();
VkImageView elementLayerView(std::uint32_t level, std::uint32_t layer);
void recordDirectUploadBarrier(VkCommandBuffer commands, bool discard);
void recordDirectUploadDone(VkCommandBuffer commands);
void release() noexcept;
Context context;
@ -431,10 +452,19 @@ private:
SurfaceGeometry geometry;
std::vector<std::byte> original;
mutable std::array<std::uint64_t, 4> comparedGuestBytes{};
std::vector<std::byte> generationBaseline;
// DCC keys the image content was uploaded under: a fast-cleared surface starts as its clear value.
DccKeys uploadedKeys = DccKeys::Uncompressed;
mutable DccKeys filledKeys = DccKeys::Uncompressed;
mutable DccKeyProof keyProof;
mutable DccRangeProof targetKeyProof;
struct RefreshProof {
std::uint64_t generation = 0;
std::uint64_t pendingSerial = 0;
std::uint64_t keyGeneration = 0;
DccKeys keys = DccKeys::Uncompressed;
};
RefreshProof refreshProof;
struct ForeignKeyProof {
std::uint64_t dccAddress = 0;
DccKeyProof proof;
@ -470,6 +500,9 @@ private:
std::map<std::uint32_t, VkImageView> firstLayerViews;
std::map<std::pair<std::uint32_t, bool>, VkImageView> atomicViews;
std::map<std::pair<std::uint32_t, bool>, VkImageView> uintViews;
VkImageView elementView = VK_NULL_HANDLE;
std::map<std::uint32_t, VkImageView> elementLayerViews;
std::int8_t singlePassState = -1;
bool attachable = false;
std::map<std::tuple<VkFormat, std::uint32_t, std::uint32_t>, VkImageView> attachmentViews;
VkImage proxyImage = VK_NULL_HANDLE;
@ -509,7 +542,7 @@ private:
// element (a fast miss is followed by a full lookup); the refresh, upload, DCC scan and pending
// flush rows lie inside the storage and sampled rows.
struct LookupOutcomes {
enum Kind : std::size_t { SampledFast, SampledFastMiss, SampledHitView, SampledHitClearedView, SampledHitSnapshot, SampledMadeView, SampledMadeSnapshot, StorageHit, StorageMade, RefreshUnchanged, RefreshCompared, UploadDirect, UploadCpu, UploadClear, DccScan, PendingFlush, Count };
enum Kind : std::size_t { SampledFast, SampledFastMiss, SampledHitView, SampledHitClearedView, SampledHitSnapshot, SampledMadeView, SampledMadeSnapshot, StorageHit, StorageMade, RefreshUnchanged, RefreshCompared, RefreshProved, UploadDirect, UploadCpu, UploadClear, DccScan, PendingFlush, Count };
static const char* Name(Kind kind);
static bool Profiled();
// Charges the time since `start` to `kind` on this thread and returns now.

25
core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp
View File

@ -5,12 +5,16 @@
#include "prx/libSceAgcDriver/Graphics/include/Context.hpp"
#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp"
#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp"
#include <cstddef>
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
namespace AgcDriver::Graphics {
class Buffer;
// The part of a mip one Dispatch moves: only elements whose tiled offset (relative to the mip's
// tiled base; the linear offset of a linear surface) lies in [rangeBegin, rangeEnd), with the
// tiled buffer holding the mip from tiledBase and the linear buffer holding `linearBytes` of
@ -29,6 +33,7 @@ namespace AgcDriver::Graphics {
std::uint32_t columnEnd = 0;
std::uint32_t rowBegin = 0;
std::uint32_t rowEnd = 0;
std::uint32_t pipeBankXor = 0;
};
class TextureDetiler {
@ -43,13 +48,18 @@ namespace AgcDriver::Graphics {
// restricts the move to part of the mip (a partial upload or write-back, StorageTexture),
// leaving the other elements of both buffers alone.
void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, bool thick = false, const DetileWindow& window = {});
void DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, const DetileWindow& window = {});
static VkFormat ImageElementFormat(std::uint32_t elementBytes);
// Recycles the descriptor sets of the previous batch; call before recording a new command batch.
void BeginBatch();
void DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array<std::uint32_t, 2>& clearWords, bool write);
std::uint32_t CmaskErrors();
private:
VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick);
VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image = false);
void release() noexcept;
VkDescriptorSet allocateSet();
VkDescriptorSet allocateSet(bool image = false);
void checkWindow(const TileMipLayout& layout, const DetileWindow& window, std::uint32_t& columnEnd, std::uint32_t& rowEnd) const;
const Context context;
VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE;
@ -58,6 +68,17 @@ namespace AgcDriver::Graphics {
std::vector<std::pair<std::uint32_t, VkPipeline>> pipelines;
std::vector<VkDescriptorPool> descriptorPools;
std::size_t allocatedSets = 0;
VkDescriptorSetLayout cmaskDescriptorLayout = VK_NULL_HANDLE;
VkPipelineLayout cmaskPipelineLayout = VK_NULL_HANDLE;
VkPipeline cmaskPipeline = VK_NULL_HANDLE;
std::vector<VkDescriptorPool> cmaskDescriptorPools;
std::size_t allocatedCmaskSets = 0;
std::unique_ptr<Buffer> cmaskErrors;
VkDescriptorSetLayout imageDescriptorLayout = VK_NULL_HANDLE;
VkPipelineLayout imagePipelineLayout = VK_NULL_HANDLE;
VkShaderModule imageModule = VK_NULL_HANDLE;
std::vector<VkDescriptorPool> imageDescriptorPools;
std::size_t allocatedImageSets = 0;
};
}

1
core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp
View File

@ -15,6 +15,7 @@ struct Context;
VkFormat ResolveTextureFormat(std::uint32_t guestFormat);
std::uint32_t SrgbDecodeFormats(PFN_vkGetPhysicalDeviceFormatProperties formatProperties, VkPhysicalDevice physical);
VkFormat SampledTextureFormat(const Context& context, std::uint32_t guestFormat);
bool IsSrgbTextureFormat(std::uint32_t guestFormat);
std::optional<std::uint32_t> FindGuestTextureFormat(VkFormat format, std::uint32_t elementBytes);
std::optional<std::uint32_t> FindGuestColorTargetFormat(VkFormat format, std::uint32_t elementBytes);
std::uint32_t BytesPerElement(std::uint32_t guestFormat);

5
core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp
View File

@ -89,6 +89,11 @@ inline constexpr TextureSwizzleEquation kTextureSwizzleEquations[] = {
{0x119u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x20004020u, 0x10020004u, 0x04010010u, 0x08008008u, 0x00000008u, 0x08000000u, 0x00010000u, 0x00000010u}},
{0x119u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x20002020u, 0x10010004u, 0x04008010u, 0x08004008u, 0x00000008u, 0x08000000u, 0x00008000u, 0x00000010u}},
{0x119u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00001000u, 0x00000001u, 0x02000000u, 0x20002010u, 0x10010002u, 0x04008008u, 0x08004004u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000008u}},
{0x11au, 1u, {0x00000001u, 0x00000002u, 0x01000000u, 0x00001000u, 0x00002000u, 0x02000000u, 0x00000004u, 0x04000000u, 0x00008008u, 0x00010010u, 0x00024040u, 0x00040020u, 0x08000000u, 0x00008000u, 0x00000010u, 0x10000000u}},
{0x11au, 2u, {0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x00002000u, 0x04000000u, 0x00008008u, 0x00010010u, 0x08020040u, 0x00044020u, 0x00000004u, 0x10000000u, 0x00008000u, 0x00000010u}},
{0x11au, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x01000000u, 0x00002000u, 0x02000000u, 0x00008008u, 0x00010010u, 0x04020040u, 0x00044020u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000010u}},
{0x11au, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00000002u, 0x02000000u, 0x00008008u, 0x00010010u, 0x04020040u, 0x00042020u, 0x00000004u, 0x08000000u, 0x00004000u, 0x00000008u}},
{0x11au, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00008008u, 0x00010012u, 0x04020040u, 0x00042020u, 0x00000004u, 0x08000000u, 0x00004000u, 0x00000008u}},
};
inline const TextureSwizzleEquation* FindTextureSwizzleEquation(std::uint32_t swizzleMode, std::uint32_t elementBytes) {

2
core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp
View File

@ -141,7 +141,7 @@ void MarkShadowed(const HostImport& import, std::span<const ShadowedRange> range
// The import retires because its registration vanished or changed size, so the rest of its memory
// belongs to the title again and is dropped (counted `dropped on retire`); `lost on retire` counts
// the units nothing could record.
void RetireShadow(const Context& context, const HostImport& import, const std::function<bool(std::uint64_t, std::uint64_t)>& registered);
void RetireShadow(VkDevice device, const HostImport& import, const std::function<bool(std::uint64_t, std::uint64_t)>& registered);
// Teardown: every shadow of the context's device is published; DestroyShadows then drops them.
void PublishAllShadows(const Context& context, PublishReason reason);
void DestroyShadows(VkDevice device);

70
core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp
View File

@ -3,7 +3,9 @@
#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp"
#include <algorithm>
#include <cstring>
#include <limits>
#include <optional>
#include <set>
#include <span>
#include <utility>
@ -18,8 +20,17 @@ struct VertexFormat {
const char* scalar;
};
inline bool NullVertexDescriptor(const ShaderRecompiler::VertexAttribute& attribute) {
const auto& fields = attribute.resource.fields;
return fields[0] == 0 && fields[1] == 0 && fields[2] == 0 && fields[3] == 0;
}
inline VertexFormat DecodeVertexFormat(const ShaderRecompiler::VertexAttribute& attribute) {
Require(attribute.components >= 1 && attribute.components <= 4, "invalid vertex attribute component count");
if (NullVertexDescriptor(attribute)) {
const std::array formats{VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32G32_SFLOAT, VK_FORMAT_R32G32B32_SFLOAT, VK_FORMAT_R32G32B32A32_SFLOAT};
return {formats[attribute.components - 1], attribute.components * 4u, 4u, "f32"};
}
const auto components = attribute.formatComponents == 0u ? attribute.components : attribute.formatComponents;
Require(components >= 1u && components <= 4u, "invalid vertex format component count");
const auto format = (attribute.resource.fields[3] >> 12u) & 0x7fu;
@ -108,7 +119,7 @@ inline VertexInputLayout BuildVertexInputLayout(const Context& context, std::spa
Require((fields[1] & 0x80000000u) == 0 && (fields[3] & 0x00800000u) == 0 && (fields[3] >> 30u) == 0, "unsupported vertex buffer descriptor flags");
const auto stride = (fields[1] >> 16u) & 0x3fffu;
const auto address = fields[0] | (static_cast<std::uint64_t>(fields[1] & 0xffffu) << 32u);
Require(address != 0 && address % format.alignment == 0 && stride % format.alignment == 0, "unaligned vertex buffer");
Require(NullVertexDescriptor(attribute) || (address != 0 && address % format.alignment == 0 && stride % format.alignment == 0), "unaligned vertex buffer");
Require(stride <= context.limits.maxVertexInputBindingStride, "vertex stride exceeds device limits");
Require(context.formatProperties != nullptr, "missing vertex format property query");
VkFormatProperties properties{};
@ -123,11 +134,28 @@ inline VertexInputLayout BuildVertexInputLayout(const Context& context, std::spa
// The whole byte range a vertex buffer descriptor covers (records * stride, or records when the
// stride is 0): what an indirect draw, whose counts only the GPU knows, copies for the fetch.
inline std::uint32_t VertexBufferOutOfBoundsSelect(const ShaderRecompiler::VertexAttribute& attribute) {
return (attribute.resource.fields[3] >> 28u) & 3u;
}
inline bool VertexFetchOutOfRange(const ShaderRecompiler::VertexAttribute& attribute) {
const auto stride = (attribute.resource.fields[1] >> 16u) & 0x3fffu;
const auto select = VertexBufferOutOfBoundsSelect(attribute);
return stride == 0 && (select == 2u || select == 3u) && attribute.resource.fields[2] == 0;
}
inline std::uint64_t ZeroStrideAvailableBytes(const ShaderRecompiler::VertexAttribute& attribute) {
const auto records = attribute.resource.fields[2];
if (VertexBufferOutOfBoundsSelect(attribute) == 2u && records != 0) return std::max<std::uint64_t>(records, DecodeVertexFormat(attribute).bytes);
return records;
}
inline std::size_t VertexBufferExtent(const ShaderRecompiler::VertexAttribute& attribute) {
if (NullVertexDescriptor(attribute)) return DecodeVertexFormat(attribute).bytes;
const auto stride = (attribute.resource.fields[1] >> 16u) & 0x3fffu;
const auto records = attribute.resource.fields[2];
Require(attribute.fetchIndex <= 1, "unsupported vertex fetch index");
const auto bytes = stride == 0 ? static_cast<std::uint64_t>(records) : static_cast<std::uint64_t>(records) * stride;
const auto bytes = stride == 0 ? ZeroStrideAvailableBytes(attribute) : static_cast<std::uint64_t>(records) * stride;
Require(bytes != 0 && bytes <= std::numeric_limits<std::size_t>::max(), "empty or oversized vertex buffer descriptor");
const auto address = attribute.resource.fields[0] | (static_cast<std::uint64_t>(attribute.resource.fields[1] & 0xffffu) << 32u);
Require(address != 0 && bytes <= std::numeric_limits<std::uint64_t>::max() - address, "invalid vertex buffer address range");
@ -137,15 +165,16 @@ inline std::size_t VertexBufferExtent(const ShaderRecompiler::VertexAttribute& a
inline std::size_t VertexBufferReadSize(const ShaderRecompiler::VertexAttribute& attribute, std::uint32_t maxIndex, std::uint32_t instances, std::uint32_t firstInstance = 0) {
Require(instances != 0, "vertex input requires nonzero instance count");
Require(firstInstance <= std::numeric_limits<std::uint32_t>::max() - (instances - 1u), "vertex input instance range overflow");
if (NullVertexDescriptor(attribute)) return DecodeVertexFormat(attribute).bytes;
const auto stride = (attribute.resource.fields[1] >> 16u) & 0x3fffu;
const auto records = attribute.resource.fields[2];
Require(attribute.fetchIndex <= 1, "unsupported vertex fetch index");
const auto index = attribute.fetchIndex == 0 ? maxIndex : firstInstance + instances - 1u;
const auto bytes = DecodeVertexFormat(attribute).bytes;
Require(stride == 0 || index < records, "vertex fetch exceeds descriptor record count");
const auto available = stride == 0 ? static_cast<std::uint64_t>(records) : static_cast<std::uint64_t>(records) * stride;
Require(stride == 0 || VertexBufferOutOfBoundsSelect(attribute) <= 1u || index < records, "vertex fetch exceeds descriptor record count");
const auto required = static_cast<std::uint64_t>(stride) * index + bytes;
Require(required <= available && required <= std::numeric_limits<std::size_t>::max(), "vertex fetch exceeds descriptor byte range");
Require(stride != 0 || required <= ZeroStrideAvailableBytes(attribute), "vertex fetch exceeds descriptor byte range");
Require(required <= std::numeric_limits<std::size_t>::max(), "vertex fetch exceeds addressable byte range");
const auto address = attribute.resource.fields[0] | (static_cast<std::uint64_t>(attribute.resource.fields[1] & 0xffffu) << 32u);
Require(address != 0 && required <= std::numeric_limits<std::uint64_t>::max() - address, "invalid vertex buffer address range");
return static_cast<std::size_t>(required);
@ -163,6 +192,7 @@ struct VertexCopyPlan {
std::vector<std::pair<std::uint64_t, std::uint64_t>> copies;
std::vector<std::size_t> copyOf;
std::vector<std::uint64_t> offsets;
std::vector<std::uint32_t> alignments;
};
inline VertexCopyPlan PlanVertexCopies(std::span<const VertexFetch> fetches) {
@ -186,7 +216,9 @@ inline VertexCopyPlan PlanVertexCopies(std::span<const VertexFetch> fetches) {
if (!joins) {
lead = i;
plan.copies.emplace_back(fetch.begin, fetch.end);
plan.alignments.push_back(fetch.alignment);
}
plan.alignments.back() = std::max(plan.alignments.back(), fetch.alignment);
auto& copy = plan.copies.back();
copy.second = std::max(copy.second, fetch.end);
plan.copyOf[i] = plan.copies.size() - 1;
@ -195,6 +227,34 @@ inline VertexCopyPlan PlanVertexCopies(std::span<const VertexFetch> fetches) {
return plan;
}
inline std::optional<std::uint32_t> HighestDrawIndex(std::span<const std::byte> indices, std::uint32_t indexSize, bool skipRestart) {
Require(indexSize == 2 || indexSize == 4, "unsupported index size");
const auto restartIndex = indexSize == 2 ? 0xffffu : 0xffffffffu;
std::optional<std::uint32_t> highest;
for (std::size_t offset = 0; offset + indexSize <= indices.size(); offset += indexSize) {
std::uint32_t index = 0;
if (indexSize == 2) {
std::uint16_t value = 0;
std::memcpy(&value, indices.data() + offset, sizeof(value));
index = value;
} else {
std::memcpy(&index, indices.data() + offset, sizeof(index));
}
if (skipRestart && index == restartIndex) continue;
highest = std::max(highest.value_or(0u), index);
}
return highest;
}
inline std::vector<std::size_t> SoloZeroPaddedFetchIndices(const std::vector<VertexFetch>& fetches, const std::vector<std::size_t>& fetchValid) {
Require(fetchValid.size() == fetches.size(), "fetch validity does not match the fetch count");
std::vector<std::size_t> solo;
for (std::size_t i = 0; i < fetches.size(); ++i) {
if (fetchValid[i] < fetches[i].end - fetches[i].begin) solo.push_back(i);
}
return solo;
}
}
#endif

38
core/libs/prx/libSceAgcDriver/Graphics/shaders/CmaskClear.comp Normal file
View File

@ -0,0 +1,38 @@
#version 450
layout(local_size_x = 8, local_size_y = 8) in;
layout(set = 0, binding = 0, std430) readonly buffer Cmask { uint words[]; } cmask;
layout(set = 0, binding = 1, std430) buffer Errors { uint codes; } errors;
layout(set = 0, binding = 2) writeonly uniform uimage2D target;
layout(push_constant) uniform Push {
uint base;
uint width;
uint height;
uint blocksPerRow;
uint clear0;
uint clear1;
uint write;
} push;
const uint xMasks[13] = uint[](0x008u, 0x000u, 0x040u, 0x000u, 0x080u, 0x000u, 0x100u, 0x000u, 0x200u, 0x008u, 0x010u, 0x040u, 0x020u);
const uint yMasks[13] = uint[](0x000u, 0x010u, 0x000u, 0x040u, 0x000u, 0x080u, 0x000u, 0x100u, 0x000u, 0x008u, 0x010u, 0x020u, 0x040u);
void main() {
uvec2 pixel = gl_GlobalInvocationID.xy;
if (pixel.x >= push.width || pixel.y >= push.height) return;
uint x = pixel.x & ~7u;
uint y = pixel.y & ~7u;
uint offset = 0u;
for (uint bit = 0u; bit < 13u; ++bit) offset |= (uint(bitCount((x & xMasks[bit]) ^ (y & yMasks[bit]))) & 1u) << bit;
uint nibble = ((y / 512u) * push.blocksPerRow + x / 1024u) * 8192u + offset;
uint byteAddress = push.base + nibble / 2u;
uint code = (cmask.words[byteAddress / 4u] >> ((byteAddress % 4u) * 8u + (nibble % 2u) * 4u)) & 0xfu;
if (code == 0xfu) return;
if (code == 0u && push.write != 0u) {
imageStore(target, ivec2(pixel), uvec4(push.clear0, push.clear1, 0u, 0u));
return;
}
if ((pixel.x & 7u) == 0u && (pixel.y & 7u) == 0u) atomicOr(errors.codes, 1u << code);
}

40
core/libs/prx/libSceAgcDriver/Graphics/shaders/SampleDumps.comp Normal file
View File

@ -0,0 +1,40 @@
#version 450
#extension GL_EXT_buffer_reference : require
#extension GL_EXT_buffer_reference_uvec2 : require
layout(local_size_x = 1) in;
layout(buffer_reference, std430, buffer_reference_align = 8) buffer Words { uint words[]; };
layout(push_constant) uniform Parameters {
Words counter;
Words dumps;
uint firstDump;
uint dumpCount;
uint firstSegment;
uint lastSegment;
} parameters;
void main() {
uint low = parameters.counter.words[0];
uint high = parameters.counter.words[1];
uint segment = parameters.firstSegment;
for (uint dump = parameters.firstDump; dump < parameters.firstDump + parameters.dumpCount; ++dump) {
uint end = parameters.dumps.words[dump * 4u + 2u];
for (; segment < end; ++segment) {
uint carry;
low = uaddCarry(low, parameters.counter.words[4u + 2u * segment], carry);
high = high + parameters.counter.words[5u + 2u * segment] + carry;
}
Words target = Words(uvec2(parameters.dumps.words[dump * 4u], parameters.dumps.words[dump * 4u + 1u]));
for (uint db = 0u; db < 16u; ++db) {
target.words[db * 4u] = db == 0u ? low : 0u;
target.words[db * 4u + 1u] = (db == 0u ? high : 0u) | 0x80000000u;
}
}
for (; segment < parameters.lastSegment; ++segment) {
uint carry;
low = uaddCarry(low, parameters.counter.words[4u + 2u * segment], carry);
high = high + parameters.counter.words[5u + 2u * segment] + carry;
}
parameters.counter.words[0] = low;
parameters.counter.words[1] = high;
}

65
core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp
View File

@ -1,4 +1,7 @@
#version 450
#ifdef LINEAR_IMAGE
#extension GL_EXT_shader_image_load_formatted : require
#endif
layout(constant_id = 0) const uint ELEMENT_BYTES = 4u;
layout(constant_id = 1) const uint BLOCK_BYTES = 256u;
@ -29,8 +32,13 @@ layout(constant_id = 21) const uint BLOCK_HEIGHT = 0u;
layout(local_size_x = 8, local_size_y = 8) in;
#ifdef LINEAR_IMAGE
layout(set = 0, binding = 0, std430) buffer Tiled { uint data[]; } tiledBuffer;
layout(set = 0, binding = 1) uniform uimage2D linearImage;
#else
layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } inputBuffer;
layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } outputBuffer;
#endif
layout(push_constant) uniform Push {
uint srcBase;
@ -55,6 +63,7 @@ layout(push_constant) uniform Push {
// whole mip).
uint columnBegin;
uint rowBegin;
uint pipeBankXor;
} params;
uint equationBit(uint mask, uvec2 p) {
@ -117,7 +126,7 @@ uint standard64Extra(uint x, uint y) {
uint blockOffset(uvec2 p) {
if (TILE_FAMILY == 2u) {
return equationOffset(p);
return equationOffset(p) ^ params.pipeBankXor;
}
uint offset = standardOffset(p.x, p.y);
if (BLOCK_BYTES > 4096u) {
@ -141,6 +150,43 @@ uvec2 blockExtent() {
return uvec2(width, BLOCK_BYTES / (width * ELEMENT_BYTES));
}
#ifdef LINEAR_IMAGE
void storeElement(uint src, uvec2 p) {
uvec4 value = uvec4(0u);
uint word = src >> 2;
if (ELEMENT_BYTES == 1u) {
value.x = (tiledBuffer.data[word] >> ((src & 3u) * 8u)) & 0xffu;
} else if (ELEMENT_BYTES == 2u) {
value.x = (tiledBuffer.data[word] >> ((src & 3u) * 8u)) & 0xffffu;
} else {
value.x = tiledBuffer.data[word];
if (ELEMENT_BYTES >= 8u) value.y = tiledBuffer.data[word + 1u];
if (ELEMENT_BYTES >= 16u) {
value.z = tiledBuffer.data[word + 2u];
value.w = tiledBuffer.data[word + 3u];
}
}
imageStore(linearImage, ivec2(p), value);
}
void loadElement(uvec2 p, uint dst) {
uvec4 value = imageLoad(linearImage, ivec2(p));
uint word = dst >> 2;
if (ELEMENT_BYTES >= 4u) {
tiledBuffer.data[word] = value.x;
if (ELEMENT_BYTES >= 8u) tiledBuffer.data[word + 1u] = value.y;
if (ELEMENT_BYTES >= 16u) {
tiledBuffer.data[word + 2u] = value.z;
tiledBuffer.data[word + 3u] = value.w;
}
} else {
uint mask = ELEMENT_BYTES == 1u ? 0xffu : 0xffffu;
uint shift = (dst & 3u) * 8u;
atomicAnd(tiledBuffer.data[word], ~(mask << shift));
atomicOr(tiledBuffer.data[word], (value.x & mask) << shift);
}
}
#else
void copyElement(uint src, uint dst) {
if (ELEMENT_BYTES >= 4u) {
for (uint i = 0u; i < ELEMENT_BYTES; i += 4u) {
@ -154,6 +200,7 @@ void copyElement(uint src, uint dst) {
atomicOr(outputBuffer.data[dst >> 2], value << shift);
}
}
#endif
void main() {
uvec2 p = gl_GlobalInvocationID.xy + uvec2(params.columnBegin, params.rowBegin);
@ -169,11 +216,19 @@ void main() {
if (linearOffset < params.rangeBegin || linearOffset >= params.rangeEnd) {
return;
}
#ifdef LINEAR_IMAGE
if (RETILE != 0u) {
loadElement(p, params.dstBase + linearOffset - params.tiledBase);
} else {
storeElement(params.srcBase + linearOffset - params.tiledBase, p);
}
#else
if (RETILE != 0u) {
copyElement(params.srcBase + linearOffset - params.linearBase, params.dstBase + linearOffset - params.tiledBase);
} else {
copyElement(params.srcBase + linearOffset - params.tiledBase, params.dstBase + linearOffset - params.linearBase);
}
#endif
return;
}
@ -193,10 +248,18 @@ void main() {
tiledOffset -= params.tiledBase;
linearOffset -= params.linearBase;
#ifdef LINEAR_IMAGE
if (RETILE != 0u) {
loadElement(p, params.dstBase + tiledOffset);
} else {
storeElement(params.srcBase + tiledOffset, p);
}
#else
if (RETILE != 0u) {
copyElement(params.srcBase + linearOffset, params.dstBase + tiledOffset);
} else {
copyElement(params.srcBase + tiledOffset, params.dstBase + linearOffset);
}
#endif
}

157
core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp
View File

@ -5,6 +5,7 @@
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <limits>
namespace AgcDriver::Graphics {
@ -16,12 +17,19 @@ bool sharedTiers() {
return shared;
}
bool exactSizes() {
static const bool exact = std::getenv("APS5_NO_BUFFER_CLASSES") != nullptr;
return exact;
}
BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function<PFN_vkUnmapMemory>("vkUnmapMemory")), destroyBuffer(context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")), freeMemory(context.Function<PFN_vkFreeMemory>("vkFreeMemory")), maxSlots(std::max<std::size_t>(1, std::min<std::size_t>(MaxSlots(), context.limits.maxMemoryAllocationCount / 6u))) {
constexpr VkBufferUsageFlags deviceUsage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
}
BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function<PFN_vkUnmapMemory>("vkUnmapMemory")), destroyBuffer(context.Function<PFN_vkDestroyBuffer>("vkDestroyBuffer")), freeMemory(context.Function<PFN_vkFreeMemory>("vkFreeMemory")), allocateMemory(context.Function<PFN_vkAllocateMemory>("vkAllocateMemory")), mapMemory(context.Function<PFN_vkMapMemory>("vkMapMemory")), maxSlots(std::max<std::size_t>(1, std::min<std::size_t>(MaxSlots(), context.limits.maxMemoryAllocationCount / 6u))) {
smallTier.budget = smallBudget;
largeTier.budget = budget;
deviceTier.budget = DeviceBudget();
deviceTier.budget = DeviceTierEnabled() ? DeviceBudget(context.memory) : 0;
}
BufferPool::~BufferPool() {
@ -30,38 +38,159 @@ BufferPool::~BufferPool() {
for (const auto& slot : slots) destroy(slot.allocation);
}
}
for (auto& [memory, block] : slabBlocks) freeBlock(*block);
}
VkDeviceSize BufferPool::DeviceBudget() {
static const VkDeviceSize deviceBudget = [] {
bool BufferPool::SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom) {
if (capacity >= classLimit || size > capacity || !std::has_single_bit(capacity)) return false;
const auto alignmentOk = alignment == 0 || (std::has_single_bit(alignment) && alignment <= capacity);
const auto atomOk = atom <= 1 || (std::has_single_bit(atom) && atom <= capacity);
return alignmentOk && atomOk;
}
VkDeviceSize BufferPool::SlabBlockBytes(std::size_t capacity) {
return std::max<VkDeviceSize>(VkDeviceSize{2} << 20u, static_cast<VkDeviceSize>(capacity) * 8u);
}
std::uint64_t BufferPool::SlabKey(std::uint32_t memoryType, std::size_t capacity, bool addressable) {
return (static_cast<std::uint64_t>(memoryType) << 40u) | (static_cast<std::uint64_t>(addressable ? 1u : 0u) << 39u) | static_cast<std::uint64_t>(capacity);
}
SlabSlot BufferPool::TakeSlot(const Context& context, std::uint32_t memoryType, std::size_t capacity, bool addressable) {
const auto key = SlabKey(memoryType, capacity, addressable);
const auto takeFrom = [&](SlabBlock& block, Slab& slab) {
const auto index = block.free.back();
block.free.pop_back();
if (block.used++ == 0) --slab.emptyBlocks;
if (block.free.empty()) slab.available.erase(std::find(slab.available.begin(), slab.available.end(), &block));
const auto offset = static_cast<VkDeviceSize>(index) * block.slotBytes;
return SlabSlot{block.memory, offset, block.mapping != nullptr ? block.mapping + offset : nullptr};
};
{
std::lock_guard lock(slabMutex);
auto& slab = slabs[key];
if (!slab.available.empty()) return takeFrom(*slab.available.back(), slab);
}
const auto blockBytes = SlabBlockBytes(capacity);
auto block = std::make_unique<SlabBlock>();
block->slotBytes = capacity;
block->slab = key;
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
const VkMemoryAllocateFlagsInfo flags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, nullptr, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0};
if (addressable) allocation.pNext = &flags;
allocation.allocationSize = blockBytes;
allocation.memoryTypeIndex = memoryType;
Check(allocateMemory(device, &allocation, nullptr, &block->memory), "vkAllocateMemory buffer slab");
if ((context.memory.memoryTypes[memoryType].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) {
void* mapping = nullptr;
const auto mapped = mapMemory(device, block->memory, 0, VK_WHOLE_SIZE, 0, &mapping);
if (mapped != VK_SUCCESS) freeMemory(device, block->memory, nullptr);
Check(mapped, "vkMapMemory buffer slab");
block->mapping = static_cast<std::byte*>(mapping);
}
const auto slots = static_cast<std::uint32_t>(blockBytes / capacity);
block->free.reserve(slots);
for (std::uint32_t index = slots; index-- != 0;) block->free.push_back(index);
std::lock_guard lock(slabMutex);
auto& slab = slabs[key];
auto& inserted = *block;
slabBlocks.emplace(inserted.memory, std::move(block));
slab.available.push_back(&inserted);
++slab.emptyBlocks;
return takeFrom(*slab.available.back(), slab);
}
void BufferPool::freeBlock(SlabBlock& block) noexcept {
if (block.mapping != nullptr) unmap(device, block.memory);
freeMemory(device, block.memory, nullptr);
}
void BufferPool::PutSlot(VkDeviceMemory memory, VkDeviceSize offset) noexcept {
std::unique_ptr<SlabBlock> released;
{
std::lock_guard lock(slabMutex);
const auto found = slabBlocks.find(memory);
if (found == slabBlocks.end()) return;
auto& block = *found->second;
auto& slab = slabs[block.slab];
if (block.free.empty()) slab.available.push_back(&block);
block.free.push_back(static_cast<std::uint32_t>(offset / block.slotBytes));
if (--block.used != 0) return;
if (slab.emptyBlocks == 0) {
++slab.emptyBlocks;
return;
}
slab.available.erase(std::find(slab.available.begin(), slab.available.end(), &block));
released = std::move(found->second);
slabBlocks.erase(found);
}
freeBlock(*released);
}
std::size_t BufferPool::SlabBlocks() {
std::lock_guard lock(slabMutex);
return slabBlocks.size();
}
bool BufferPool::DeviceTierEnabled() {
static const bool enabled = [] {
const char* value = std::getenv("APS5_STAGING_POOL_MIB");
return (value != nullptr ? std::strtoull(value, nullptr, 10) : 512ull) << 20u;
return value == nullptr || std::strtoull(value, nullptr, 10) != 0;
}();
return deviceBudget;
return enabled;
}
VkDeviceSize BufferPool::DeviceBudget(const VkPhysicalDeviceMemoryProperties& memory) {
static const char* value = std::getenv("APS5_STAGING_POOL_MIB");
if (value != nullptr) return static_cast<VkDeviceSize>(std::strtoull(value, nullptr, 10)) << 20u;
VkDeviceSize largest = 0;
for (std::uint32_t heap = 0; heap < memory.memoryHeapCount && heap < VK_MAX_MEMORY_HEAPS; ++heap) {
if ((memory.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) largest = std::max(largest, memory.memoryHeaps[heap].size);
}
return std::max<VkDeviceSize>(VkDeviceSize{512} << 20u, largest / 8u);
}
void BufferPool::destroy(const BufferAllocation& allocation) noexcept {
if (allocation.slab) {
destroyBuffer(device, allocation.buffer, nullptr);
PutSlot(allocation.memory, allocation.offset);
return;
}
// Device-local allocations (see DeviceBuffer) are never mapped.
if (allocation.mapping != nullptr) unmap(device, allocation.memory);
destroyBuffer(device, allocation.buffer, nullptr);
freeMemory(device, allocation.memory, nullptr);
}
std::size_t BufferPool::Capacity(std::size_t bytes) {
static const bool exact = std::getenv("APS5_NO_BUFFER_CLASSES") != nullptr;
bool BufferPool::DeviceTiered(VkMemoryPropertyFlags properties) {
return (properties & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0 && (properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0 && DeviceTierEnabled();
}
std::size_t BufferPool::Capacity(std::size_t bytes, VkMemoryPropertyFlags properties) {
constexpr std::size_t smallest = 256;
if (exact || bytes >= classLimit) return bytes;
return std::max(smallest, std::bit_ceil(bytes));
if (exactSizes()) return bytes;
if (bytes < classLimit) return std::max(smallest, std::bit_ceil(bytes));
if (!DeviceTiered(properties)) return bytes;
const auto step = std::bit_floor(bytes) >> deviceClassBits;
return (bytes + step - 1) / step * step;
}
std::size_t BufferPool::NextCapacity(std::size_t capacity) {
return capacity < classLimit ? capacity * 2 : capacity + (std::bit_floor(capacity) >> deviceClassBits);
}
VkBufferUsageFlags BufferPool::Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) {
return DeviceTiered(properties) ? usage | deviceUsage : usage;
}
BufferPool::Tier& BufferPool::tierFor(std::size_t capacity, VkMemoryPropertyFlags properties) {
if ((properties & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0 && (properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0 && DeviceBudget() != 0) return deviceTier;
if (DeviceTiered(properties)) return deviceTier;
return !sharedTiers() && capacity < classLimit ? smallTier : largeTier;
}
std::optional<BufferAllocation> BufferPool::Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) {
static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
const auto capacity = Capacity(bytes);
const auto capacity = Capacity(bytes, properties);
std::lock_guard lock(mutex);
++clock;
if (profile) {
@ -73,7 +202,9 @@ std::optional<BufferAllocation> BufferPool::Take(std::size_t bytes, VkBufferUsag
}
}
auto& tier = tierFor(capacity, properties);
const auto found = tier.free.find({capacity, usage, properties});
const auto largest = &tier == &deviceTier && !exactSizes() && bytes <= std::numeric_limits<std::size_t>::max() / 2 ? std::max(capacity, bytes * 2) : capacity;
auto found = tier.free.end();
for (auto size = capacity; size <= largest && found == tier.free.end(); size = NextCapacity(size)) found = tier.free.find({size, usage, properties});
if (found == tier.free.end() || found->second.empty()) {
++tier.misses;
return std::nullopt;

56
core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp
View File

@ -1,10 +1,12 @@
#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp"
#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp"
#include <array>
#include <bit>
#include <cstring>
#include <limits>
#include <mutex>
#include <stdexcept>
#include <utility>
namespace AgcDriver::Graphics {
namespace {
@ -41,11 +43,12 @@ std::uint32_t standard64Extra(std::uint32_t x, std::uint32_t y, std::uint32_t el
ColorTileMode DecodeColorTileMode(std::uint32_t attrib3) {
const auto resourceType = (attrib3 >> 24u) & 3u;
require((attrib3 & 0x80002000u) == 0 && ((resourceType == 1 && (attrib3 & 0x1fffu) == 0) || resourceType == 2) && ((attrib3 >> 27u) & 7u) == 1, "AGC graphics: unsupported color depth, dimension, resource level or metadata mode");
require((attrib3 & 0x80002000u) == 0 && ((resourceType == 0 && (attrib3 & 0x1fffu) == 0) || resourceType == 1 || resourceType == 2) && ((attrib3 >> 27u) & 7u) == 1, "AGC graphics: unsupported color depth, dimension, resource level or metadata mode");
const auto mode = (attrib3 >> 14u) & 0x1fu;
const auto fmaskMode = (attrib3 >> 19u) & 0x1fu;
require(fmaskMode == 0 || fmaskMode == 0x18, "AGC graphics: unsupported color FMASK swizzle mode");
require(mode == 0 || mode == 5 || mode == 9 || mode == 0x1b, "AGC graphics: unsupported color tile mode");
require(mode == 0 || mode == 5 || mode == 9 || mode == 0x15 || mode == 0x16 || mode == 0x1b, "AGC graphics: unsupported color tile mode");
require(fmaskMode == 0 || fmaskMode == 0x18 || fmaskMode == (mode & ~3u), "AGC graphics: unsupported color FMASK swizzle mode");
require(resourceType != 0 || mode == 0 || mode == 0x1b, "AGC graphics: 1D color targets with a standard swizzle mode are invalid");
return static_cast<ColorTileMode>(mode);
}
@ -56,14 +59,15 @@ struct SwizzleTables {
std::vector<std::uint32_t> y;
};
const SwizzleTables& renderTargetTables(std::uint32_t bytesPerElement, std::uint32_t blockWidth, std::uint32_t blockHeight) {
static std::once_flag once[5];
static SwizzleTables tables[5];
const SwizzleTables& equationTables(ColorTileMode mode, std::uint32_t bytesPerElement, std::uint32_t blockWidth, std::uint32_t blockHeight) {
static std::once_flag once[3][5];
static SwizzleTables tables[3][5];
const auto modeIndex = mode == ColorTileMode::S4KBX ? 0u : mode == ColorTileMode::D4KBX ? 1u : 2u;
const auto index = static_cast<std::size_t>(std::countr_zero(bytesPerElement));
std::call_once(once[index], [&] {
const auto* equation = FindTextureSwizzleEquation(27u, bytesPerElement);
require(equation != nullptr, "AGC graphics: no SW_64KB_R_X equation for the color element size");
auto& table = tables[index];
std::call_once(once[modeIndex][index], [&] {
const auto* equation = FindTextureSwizzleEquation(static_cast<std::uint32_t>(mode), bytesPerElement);
require(equation != nullptr, "AGC graphics: no XOR swizzle equation for the color tile mode and element size");
auto& table = tables[modeIndex][index];
table.x.resize(blockWidth);
table.y.resize(blockHeight);
for (std::uint32_t x = 0; x < blockWidth; ++x) {
@ -77,7 +81,7 @@ const SwizzleTables& renderTargetTables(std::uint32_t bytesPerElement, std::uint
table.y[y] = offset;
}
});
return tables[index];
return tables[modeIndex][index];
}
const SwizzleTables& standardTables(std::uint32_t bytesPerElement, std::uint32_t blockWidth, std::uint32_t blockHeight, bool block64KB) {
@ -96,9 +100,10 @@ const SwizzleTables& standardTables(std::uint32_t bytesPerElement, std::uint32_t
}
ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement) : width(width), height(height), pitch(width), mode(mode), bytes(0), elementBytes(bytesPerElement) {
ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement, std::uint32_t pipeBankXor) : width(width), height(height), pitch(width), mode(mode), bytes(0), elementBytes(bytesPerElement), pipeBankXor(pipeBankXor) {
require(width != 0 && height != 0 && width <= 16384 && height <= 16384, "AGC graphics: invalid color surface extent");
require(std::has_single_bit(bytesPerElement) && bytesPerElement <= 16u, "AGC graphics: unsupported color element size");
require(pipeBankXor == 0 || (ColorTileModeIsXor(mode) && pipeBankXor < ColorTileModeBlockBytes(mode) && pipeBankXor % 256u == 0), "AGC graphics: a color pipe/bank XOR applies only to whole 256-byte units of XOR swizzle blocks");
std::uint32_t paddedHeight = height;
switch (mode) {
case ColorTileMode::Linear: {
@ -106,14 +111,16 @@ ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height,
pitch = (width + pitchAlignment - 1u) / pitchAlignment * pitchAlignment;
break;
}
case ColorTileMode::S4KBX:
case ColorTileMode::D4KBX:
case ColorTileMode::RenderTarget: {
// SW_64KB_R_X: 64 KiB blocks of 2^(16 - log2(bpe)) elements, wider than tall for odd powers.
const auto log2Elements = 16u - static_cast<std::uint32_t>(std::countr_zero(bytesPerElement));
const auto log2Elements = (mode == ColorTileMode::RenderTarget ? 16u : 12u) - static_cast<std::uint32_t>(std::countr_zero(bytesPerElement));
blockWidth = 1u << ((log2Elements + 1u) / 2u);
blockHeight = 1u << (log2Elements / 2u);
pitch = (width + blockWidth - 1u) / blockWidth * blockWidth;
paddedHeight = (height + blockHeight - 1u) / blockHeight * blockHeight;
const auto& tables = renderTargetTables(bytesPerElement, blockWidth, blockHeight);
const auto& tables = equationTables(mode, bytesPerElement, blockWidth, blockHeight);
xOffsets = tables.x.data();
yOffsets = tables.y.data();
break;
@ -142,7 +149,7 @@ ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height,
std::size_t ColorTargetLayout::offset(std::uint32_t x, std::uint32_t y) const {
if (mode == ColorTileMode::Linear) return (static_cast<std::size_t>(y) * pitch + x) * elementBytes;
const auto block = static_cast<std::size_t>(y / blockHeight) * (pitch / blockWidth) + x / blockWidth;
return block * Alignment() + (xOffsets[x % blockWidth] ^ yOffsets[y % blockHeight]);
return block * Alignment() + (xOffsets[x % blockWidth] ^ yOffsets[y % blockHeight] ^ pipeBankXor);
}
std::size_t ColorTargetLayout::Offset(std::uint32_t x, std::uint32_t y) const {
@ -166,4 +173,23 @@ void ColorTargetLayout::Tile(std::span<const std::byte> source, std::span<std::b
}
}
CmaskLayout::CmaskLayout(std::uint32_t width, std::uint32_t height) : width(width), height(height), blocksPerRow((width + 1023u) / 1024u), bytes(0) {
require(width != 0 && height != 0 && width <= 16384 && height <= 16384, "AGC graphics: invalid CMASK surface extent");
bytes = static_cast<std::size_t>(blocksPerRow) * ((height + 511u) / 512u) * Alignment;
}
std::size_t CmaskLayout::Nibble(std::uint32_t tileX, std::uint32_t tileY) const {
require(tileX < TilesX() && tileY < TilesY(), "AGC graphics: CMASK tile out of range");
static constexpr std::array<std::pair<std::uint32_t, std::uint32_t>, 13> equation{{
{0x008u, 0x000u}, {0x000u, 0x010u}, {0x040u, 0x000u}, {0x000u, 0x040u}, {0x080u, 0x000u}, {0x000u, 0x080u}, {0x100u, 0x000u},
{0x000u, 0x100u}, {0x200u, 0x000u}, {0x008u, 0x008u}, {0x010u, 0x010u}, {0x040u, 0x020u}, {0x020u, 0x040u},
}};
const auto x = tileX * 8u;
const auto y = tileY * 8u;
std::size_t offset = 0;
for (std::size_t bit = 0; bit < equation.size(); ++bit) offset |= static_cast<std::size_t>(parity((x & equation[bit].first) ^ (y & equation[bit].second))) << bit;
const auto block = static_cast<std::size_t>(y / 512u) * blocksPerRow + x / 1024u;
return block * Alignment * 2u + offset;
}
}

4
core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp
View File

@ -7,7 +7,7 @@ namespace AgcDriver::Graphics {
void ReadColorTarget(const ColorTarget& target, std::span<std::byte> destination) {
PerformanceTimer timing("ColorTarget.Read");
const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes);
const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes, target.pipeBankXor);
Require(target.bytes == layout.Bytes() && destination.size() == layout.LinearBytes(), "color target transfer size mismatch");
if (target.tileMode == ColorTileMode::Linear && layout.Bytes() == layout.LinearBytes()) {
GuestMemory::Read(target.address, destination, layout.Alignment());
@ -23,7 +23,7 @@ void ReadColorTarget(const ColorTarget& target, std::span<std::byte> destination
void WriteColorTarget(const ColorTarget& target, std::span<const std::byte> source) {
PerformanceTimer timing("ColorTarget.Write");
const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes);
const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes, target.pipeBankXor);
Require(target.bytes == layout.Bytes() && source.size() == layout.LinearBytes(), "color target transfer size mismatch");
if (target.tileMode == ColorTileMode::Linear && layout.Bytes() == layout.LinearBytes()) {
GuestMemory::Write(target.address, source, layout.Alignment());

42
core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp
View File

@ -82,6 +82,8 @@ struct ProofCounters {
std::atomic<std::uint64_t> proved{0};
std::atomic<std::uint64_t> scanned{0};
std::atomic<std::uint64_t> unstable{0};
std::atomic<std::uint64_t> rangeProved{0};
std::atomic<std::uint64_t> rangeScanned{0};
};
ProofCounters& Proofs() {
@ -458,15 +460,18 @@ DccKeys textureClearKeys(const GuestTextureResource& resource, std::uint64_t gue
return keys;
}
DccKeys currentDccKeys(std::uint64_t metaAddress, std::size_t count) {
DccKeys currentDccKeys(std::uint64_t metaAddress, std::size_t count, bool& memoized) {
memoized = false;
if (metaAddress != 0 && count != 0 && GuestMemory::GpuMutex().HeldByThisThread()) {
if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->PendingWriteOverlaps(metaAddress, count)) {
if (const auto keys = PendingStoreKeys(*recorder, metaAddress, count)) return *keys;
if (const auto keys = PendingStoreKeys(*recorder, metaAddress, count)) {
memoized = true;
return *keys;
}
Recorder::CountSync(2);
recorder->SyncThrough(metaAddress, count);
}
}
bool memoized = false;
return readDccKeys(metaAddress, count, memoized);
}
@ -489,12 +494,14 @@ DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) {
}
DccKeys CurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) {
return currentDccKeys(metaAddress, DccKeyBytes(surfaceBytes));
bool memoized = false;
return currentDccKeys(metaAddress, DccKeyBytes(surfaceBytes), memoized);
}
DccKeys CurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes, std::size_t keyCount) {
Require(keyCount >= DccKeyBytes(surfaceBytes), "a DCC key range covers fewer keys than one per 256 surface bytes");
return currentDccKeys(metaAddress, keyCount);
bool memoized = false;
return currentDccKeys(metaAddress, keyCount, memoized);
}
bool IsDccClear(DccKeys keys) {
@ -618,7 +625,30 @@ DccKeys ProvedClearKeys(const GuestTextureResource& resource, std::uint64_t gues
DccKeyProofCounts KeyProofCounts() {
const auto& counters = Proofs();
return {counters.proved.load(std::memory_order_relaxed), counters.scanned.load(std::memory_order_relaxed), counters.unstable.load(std::memory_order_relaxed)};
return {counters.proved.load(std::memory_order_relaxed), counters.scanned.load(std::memory_order_relaxed), counters.unstable.load(std::memory_order_relaxed), counters.rangeProved.load(std::memory_order_relaxed), counters.rangeScanned.load(std::memory_order_relaxed)};
}
DccKeys ProvedCurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes, DccRangeProof& proof) {
const auto count = DccKeyBytes(surfaceBytes);
if (metaAddress == 0 || count == 0 || !KeyFastPath() || !GuestMemory::GpuMutex().HeldByThisThread()) return CurrentDccKeys(metaAddress, surfaceBytes);
auto& counters = Proofs();
const auto collected = GuestMemory::CollectWrites(metaAddress, count);
if (proof.generation != 0 && proof.address == metaAddress && proof.count == count && collected != 0 && GuestMemory::UnchangedSince(metaAddress, count, proof.generation)) {
counters.rangeProved.fetch_add(1, std::memory_order_relaxed);
return proof.keys;
}
bool memoized = false;
const auto keys = currentDccKeys(metaAddress, count, memoized);
bool stable = collected != 0 && !memoized;
if (stable) {
if (auto* recorder = Recorder::Active(); recorder != nullptr) {
const auto info = recorder->DescribePendingWrite(metaAddress, count);
stable = !info.has_value() || info->signaled;
}
}
proof = stable ? DccRangeProof{metaAddress, count, keys, collected} : DccRangeProof{};
counters.rangeScanned.fetch_add(1, std::memory_order_relaxed);
return keys;
}
void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span<std::byte> bytes) {

49
core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp
View File

@ -10,6 +10,7 @@
#include <map>
#include <memory>
#include <mutex>
#include <set>
#include <stdexcept>
#include <vector>
@ -76,6 +77,20 @@ public:
throw;
}
}
void ClearDepth(float value) {
auto* recorder = Recorder::Active();
std::unique_ptr<CommandBatch> batch;
if (recorder == nullptr) batch = std::make_unique<CommandBatch>(context);
const auto commands = recorder != nullptr ? recorder->Commands() : batch->Handle();
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
const VkClearDepthStencilValue clear{value, 0};
const VkImageSubresourceRange range{VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, 1};
context.Function<PFN_vkCmdClearDepthStencilImage>("vkCmdClearDepthStencilImage")(commands, image, VK_IMAGE_LAYOUT_GENERAL, &clear, 1, &range);
RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT);
if (batch) batch->SubmitAndWait();
else Recorder::CountBarriers(Recorder::CommandClass::Draw, 2);
}
~DepthSurface() { release(); }
DepthSurface(const DepthSurface&) = delete;
DepthSurface& operator=(const DepthSurface&) = delete;
@ -139,6 +154,11 @@ std::vector<std::unique_ptr<DepthSurface>>& surfaces() {
return *list;
}
std::set<std::uint64_t>& clearedHtiles() {
static auto* cleared = new std::set<std::uint64_t>();
return *cleared;
}
}
std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel) {
@ -151,13 +171,29 @@ std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel) {
VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target) {
std::lock_guard lock(surfacesMutex());
const bool cleared = target.htileAddress != 0 && clearedHtiles().erase(target.htileAddress) != 0;
for (const auto& surface : surfaces()) {
if (surface->context.device == context.device && sameSurface(surface->target, target)) return surface->view;
if (surface->context.device != context.device || !sameSurface(surface->target, target)) continue;
if (cleared) surface->ClearDepth(target.clearDepth);
return surface->view;
}
surfaces().push_back(std::make_unique<DepthSurface>(context, target));
return surfaces().back()->view;
}
std::uint64_t HtileDepthClearAddress(std::span<const std::uint32_t> code, std::span<const std::uint32_t> userData, const std::array<std::uint32_t, 3>& numThreads) {
static constexpr std::array<std::uint32_t, 17> htileMaskKernel{0xd7460000u, 0x04010c06u, 0x34000084u, 0xdc388000u, 0x04020000u, 0xbf8c3f70u, 0xd7710008u, 0x00120805u, 0xd7710009u, 0x00120a05u, 0xd771000au, 0x00120c05u, 0xd771000bu, 0x00120e05u, 0xdc788000u, 0x00020800u, 0xbf810000u};
if (userData.size() < 6 || numThreads[0] != 64 || numThreads[1] != 1 || numThreads[2] != 1) return 0;
if (code.size() < htileMaskKernel.size() || !std::equal(htileMaskKernel.begin(), htileMaskKernel.end(), code.begin())) return 0;
if ((userData[5] & 0xfu) != 0 || (userData[4] & 0xfu) != 0) return 0;
return userData[2] | (static_cast<std::uint64_t>(userData[3]) << 32u);
}
void NoteHtileDepthClear(std::uint64_t htileAddress) {
std::lock_guard lock(surfacesMutex());
clearedHtiles().insert(htileAddress);
}
void ClearDepthSurfaces(VkDevice device) {
std::lock_guard lock(surfacesMutex());
std::erase_if(surfaces(), [&](const auto& surface) { return surface->context.device == device; });
@ -169,7 +205,16 @@ std::shared_ptr<Texture> DepthSurfaceTexture(const Context& context, std::span<c
const auto found = std::find_if(list.rbegin(), list.rend(), [&](const auto& surface) {
return surface->context.device == context.device && (surface->target.address == resource.baseAddress || (surface->target.stencilAddress != 0 && surface->target.stencilAddress == resource.baseAddress));
});
return found == list.rend() ? nullptr : (*found)->Sampled(words, resource, components);
if (found == list.rend()) return nullptr;
const auto& target = (*found)->target;
if (resource.width != target.extent.width || resource.height != target.extent.height) return nullptr;
const bool stencil = target.stencilAddress != 0 && resource.baseAddress == target.stencilAddress;
if (!stencil) {
const bool d16 = target.format == VK_FORMAT_D16_UNORM || target.format == VK_FORMAT_D16_UNORM_S8_UINT;
const bool depthBits = words.size() >= 4 && ShaderRecompiler::DepthBitsTextureWidth(words[1], words[3]) == (d16 ? 16u : 32u);
if (ResolveTextureFormat(resource.format) != (d16 ? VK_FORMAT_R16_UNORM : VK_FORMAT_R32_SFLOAT) && !depthBits) return nullptr;
}
return (*found)->Sampled(words, resource, components);
}
bool DepthSurfaceAt(std::uint64_t address) {

Some files were not shown because too many files have changed in this diff Show More