diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f05a2e7bb..8ac64417d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,13 +4,20 @@ on: push: branches: [main] pull_request: + branches: [main] + types: [opened, synchronize, reopened] workflow_dispatch: + inputs: + pr: + description: Pull request number to build + required: true + type: string permissions: contents: read concurrency: - group: build-${{ github.ref }} + group: build-${{ github.event_name }}-${{ github.event.pull_request.number || inputs.pr || github.ref }} cancel-in-progress: true env: @@ -20,29 +27,189 @@ env: -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache jobs: + eligibility: + if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + sha: ${{ steps.gate.outputs.sha }} + pr: ${{ steps.gate.outputs.pr }} + head: ${{ steps.gate.outputs.head }} + relinker: ${{ steps.gate.outputs.relinker }} + full: ${{ steps.gate.outputs.full }} + steps: + - uses: actions/github-script@v8 + id: gate + env: + PR_NUMBER: ${{ inputs.pr }} + with: + script: | + if (context.eventName === 'push') { + if (context.ref !== 'refs/heads/main') { + throw new Error('Automatic full builds only run on main'); + } + core.setOutput('sha', context.sha); + core.setOutput('pr', ''); + core.setOutput('head', ''); + core.setOutput('relinker', true); + core.setOutput('full', true); + await core.summary.addRaw(`Building main at ${context.sha}: all builds and tests enabled`).write(); + return; + } + const { owner, repo } = context.repo; + const manual = context.eventName === 'workflow_dispatch'; + if (manual) { + for (const username of new Set([context.actor, process.env.GITHUB_TRIGGERING_ACTOR])) { + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ owner, repo, username }); + if (!['write', 'maintain', 'admin'].includes(data.permission)) { + throw new Error('Only collaborators with write access can start or rerun builds'); + } + } + if (!/^[1-9][0-9]*$/.test(process.env.PR_NUMBER)) { + throw new Error('Enter a positive pull request number'); + } + } + const pull_number = manual ? Number(process.env.PR_NUMBER) : context.payload.pull_request.number; + if (!Number.isSafeInteger(pull_number) || pull_number < 1) { + throw new Error('Invalid pull request number'); + } + let { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number }); + for (let attempt = 0; pr.mergeable === null && attempt < 10; ++attempt) { + await new Promise((resolve) => setTimeout(resolve, 3000)); + ({ data: pr } = await github.rest.pulls.get({ owner, repo, pull_number })); + } + if (pr.state !== 'open' || pr.base.ref !== 'main') { + throw new Error('The pull request must be open and target main'); + } + if (!manual && pr.head.sha !== context.payload.pull_request.head.sha) { + throw new Error('The pull request has newer commits; use the latest run'); + } + if (pr.mergeable === false) { + throw new Error('The pull request does not merge cleanly into main; rebase it first'); + } + let relinker = manual; + if (!manual) { + const { data: commits } = await github.rest.repos.listCommits({ + owner, repo, sha: 'main', author: pr.user.login, per_page: 1 + }); + relinker = commits.length > 0; + } + const { data: main } = await github.rest.repos.getBranch({ owner, repo, branch: 'main' }); + core.setOutput('sha', main.commit.sha); + core.setOutput('pr', String(pull_number)); + core.setOutput('head', pr.head.sha); + core.setOutput('relinker', relinker); + core.setOutput('full', manual); + core.info(`PR #${pull_number} at ${pr.head.sha} merged into main at ${main.commit.sha}`); + await core.summary.addRaw( + `PR #${pull_number} at ${pr.head.sha} merged into main at ${main.commit.sha}: full builds ${manual ? 'enabled' : 'disabled'}, ` + + `relinker ${relinker ? 'enabled' : 'skipped because the author has no commit in main'}` + ).write(); + + relinker: + needs: eligibility + if: ${{ needs.eligibility.outputs.relinker == 'true' }} + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + compiler: g++ + - os: windows-latest + compiler: g++ + - os: macos-latest + compiler: clang++ + env: + WINLIBS_RELEASE: 15.2.0posix-14.0.0-ucrt-r7 + WINLIBS_ZIP: winlibs-x86_64-posix-seh-gcc-15.2.0-mingw-w64ucrt-14.0.0-r7.zip + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.eligibility.outputs.sha }} + persist-credentials: false + - name: Merge the pull request into main + if: ${{ needs.eligibility.outputs.head != '' }} + shell: bash + env: + PR_NUMBER: ${{ needs.eligibility.outputs.pr }} + PR_HEAD: ${{ needs.eligibility.outputs.head }} + run: | + git fetch --no-tags --unshallow origin "refs/pull/$PR_NUMBER/head" + if [ "$(git rev-parse FETCH_HEAD)" != "$PR_HEAD" ]; then echo "::error::The pull request has newer commits; start a new build"; exit 1; fi + if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit FETCH_HEAD; then echo "::error::The pull request does not merge cleanly into main; rebase it first"; exit 1; fi + - if: runner.os == 'Windows' + id: toolchain + uses: actions/cache@v6 + with: + path: C:\winlibs + key: ${{ env.WINLIBS_ZIP }} + - if: runner.os == 'Windows' && steps.toolchain.outputs.cache-hit != 'true' + run: | + curl -sSfL -o "$RUNNER_TEMP/winlibs.zip" "https://github.com/brechtsanders/winlibs_mingw/releases/download/$WINLIBS_RELEASE/$WINLIBS_ZIP" + 7z x "$RUNNER_TEMP/winlibs.zip" -oC:/winlibs > /dev/null + - if: runner.os == 'Windows' + run: echo 'C:\winlibs\mingw64\bin' >> "$GITHUB_PATH" + - run: cmake -S . -B build-relinker -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DANYPS5_RELINKER_ONLY=ON -DBUILD_TESTING=ON + - run: cmake --build build-relinker --parallel 3 + - run: ctest --test-dir build-relinker --output-on-failure --timeout 120 + linux: + needs: eligibility + if: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && needs.eligibility.outputs.full == 'true' }} runs-on: ubuntu-latest timeout-minutes: 30 steps: - uses: actions/checkout@v7 with: + ref: ${{ needs.eligibility.outputs.sha }} + persist-credentials: false submodules: true + - name: Merge the pull request into main + if: ${{ needs.eligibility.outputs.head != '' }} + shell: bash + env: + PR_NUMBER: ${{ needs.eligibility.outputs.pr }} + PR_HEAD: ${{ needs.eligibility.outputs.head }} + run: | + git fetch --no-tags --unshallow origin "refs/pull/$PR_NUMBER/head" + if [ "$(git rev-parse FETCH_HEAD)" != "$PR_HEAD" ]; then echo "::error::The pull request has newer commits; start a new build"; exit 1; fi + if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit FETCH_HEAD; then echo "::error::The pull request does not merge cleanly into main; rebase it first"; exit 1; fi + git submodule update --init --recursive - uses: hendrikmuhs/ccache-action@v1.2.24 with: key: linux - save: ${{ github.event_name == 'push' }} + save: false evict-old-files: job - name: Install lavapipe for the GPU execution tests run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libvulkan1 mesa-vulkan-drivers - - run: cmake -S . -B build $CONFIGURE + - run: cmake -S . -B build $CONFIGURE -DANYPS5_ENABLE_SPIRV_TOOLS=ON + - name: Check that every test executable is registered with ctest + run: python3 tools/check_test_registration.py build - run: cmake --build build - run: cmake --build build --target libs - - run: ctest --test-dir build --output-on-failure --timeout 120 --repeat until-fail:3 + - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log env: ANYPS5_REQUIRE_VULKAN: 1 GALLIUM_OVERRIDE_CPU_CAPS: avx + - name: Report tests slower than 30 s + if: ${{ !cancelled() }} + shell: bash + run: | + [ -f ctest.log ] || exit 0 + awk '/Test +#[0-9]+: / && $NF == "sec" { for (i = 1; i <= NF; i++) if ($i ~ /^#[0-9]+:$/) name = $(i + 1); t = $(NF - 1) + 0; if (t > max[name]) max[name] = t } + END { for (name in max) if (max[name] > 30) printf "::warning title=Slow test::%s took %.1f s on %s; tests must finish in 30 s (CONTRIBUTING.md)\n", name, max[name], ENVIRON["RUNNER_OS"] }' ctest.log windows: + needs: eligibility + if: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && needs.eligibility.outputs.full == 'true' }} runs-on: windows-latest timeout-minutes: 45 env: @@ -55,7 +222,20 @@ jobs: steps: - uses: actions/checkout@v7 with: + ref: ${{ needs.eligibility.outputs.sha }} + persist-credentials: false submodules: true + - name: Merge the pull request into main + if: ${{ needs.eligibility.outputs.head != '' }} + shell: bash + env: + PR_NUMBER: ${{ needs.eligibility.outputs.pr }} + PR_HEAD: ${{ needs.eligibility.outputs.head }} + run: | + git fetch --no-tags --unshallow origin "refs/pull/$PR_NUMBER/head" + if [ "$(git rev-parse FETCH_HEAD)" != "$PR_HEAD" ]; then echo "::error::The pull request has newer commits; start a new build"; exit 1; fi + if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit FETCH_HEAD; then echo "::error::The pull request does not merge cleanly into main; rebase it first"; exit 1; fi + git submodule update --init --recursive - id: toolchain uses: actions/cache@v6 with: @@ -69,13 +249,23 @@ jobs: - uses: hendrikmuhs/ccache-action@v1.2.24 with: key: windows - save: ${{ github.event_name == 'push' }} + save: false evict-old-files: job - shell: cmd run: cmake -S . -B build %CONFIGURE% + - name: Check that every test executable is registered with ctest + shell: cmd + run: python tools/check_test_registration.py build - shell: cmd run: cmake --build build - shell: cmd run: cmake --build build --target libs - shell: cmd - run: ctest --test-dir build --output-on-failure --timeout 120 --repeat until-fail:3 + run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log + - name: Report tests slower than 30 s + if: ${{ !cancelled() }} + shell: bash + run: | + [ -f ctest.log ] || exit 0 + awk '/Test +#[0-9]+: / && $NF == "sec" { for (i = 1; i <= NF; i++) if ($i ~ /^#[0-9]+:$/) name = $(i + 1); t = $(NF - 1) + 0; if (t > max[name]) max[name] = t } + END { for (name in max) if (max[name] > 30) printf "::warning title=Slow test::%s took %.1f s on %s; tests must finish in 30 s (CONTRIBUTING.md)\n", name, max[name], ENVIRON["RUNNER_OS"] }' ctest.log diff --git a/.github/workflows/conventions.yml b/.github/workflows/conventions.yml index a6ef53eb0..2a14c1c2a 100644 --- a/.github/workflows/conventions.yml +++ b/.github/workflows/conventions.yml @@ -1,14 +1,14 @@ name: Conventions on: - pull_request: + pull_request_target: types: [opened, synchronize, reopened, edited] permissions: contents: read concurrency: - group: conventions-${{ github.ref }} + group: conventions-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: @@ -18,5 +18,15 @@ jobs: steps: - uses: actions/checkout@v7 with: + ref: ${{ github.event.pull_request.base.sha }} fetch-depth: 0 - - run: python3 tools/check_conventions.py --base ${{ github.event.pull_request.base.sha }} --head ${{ github.event.pull_request.head.sha }} + persist-credentials: false + - name: Fetch pull request objects without checking out its code + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: git fetch --no-tags origin "$HEAD_SHA" + - name: Check conventions with the trusted base script + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: python3 -I tools/check_conventions.py --base "$BASE_SHA" --head "$HEAD_SHA" diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 3429cb6ff..21fac27ac 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -23,6 +23,7 @@ jobs: force_label() { gh label create "$1" --color "$2" --description "$3" --force; } force_label 'needs rebase' b60205 'Conflicts with main, rebase needed' + force_label 'intake paused' d93f0b 'Closed while PR intake is limited to authors with at least 20 PRs merged into main' force_label ci 5319e7 '.github' force_label agc 1d76db 'libSceAgc*' force_label shader 1d76db 'core/shader' diff --git a/.github/workflows/pr-intake.yml b/.github/workflows/pr-intake.yml new file mode 100644 index 000000000..a632bd61c --- /dev/null +++ b/.github/workflows/pr-intake.yml @@ -0,0 +1,68 @@ +name: PR intake + +on: + pull_request_target: + branches: [main] + types: [opened] + +permissions: {} + +concurrency: + group: pr-intake-${{ github.event.pull_request.number }} + +jobs: + intake: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/github-script@v8 + with: + script: | + const { owner, repo } = context.repo; + const pull_number = context.payload.pull_request.number; + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number }); + if (pr.state !== 'open' || pr.base.ref !== 'main') { + return; + } + const minimumMergedPRs = 20; + const { data: mergedPRs } = await github.rest.search.issuesAndPullRequests({ + q: `repo:${owner}/${repo} is:pr is:merged base:main author:${pr.user.login}`, + per_page: 1 + }); + if (mergedPRs.incomplete_results || !Number.isSafeInteger(mergedPRs.total_count) || mergedPRs.total_count < 0) { + throw new Error('Could not reliably count merged pull requests'); + } + if (mergedPRs.total_count >= minimumMergedPRs) { + return; + } + const name = 'intake paused'; + try { + await github.rest.issues.getLabel({ owner, repo, name }); + } catch (error) { + if (error.status !== 404) { + throw error; + } + try { + await github.rest.issues.createLabel({ + owner, repo, name, color: 'd93f0b', + description: 'Closed while PR intake is limited to authors with at least 20 PRs merged into main' + }); + } catch (createError) { + if (createError.status !== 422) { + throw createError; + } + await github.rest.issues.getLabel({ owner, repo, name }); + } + } + await github.rest.issues.addLabels({ owner, repo, issue_number: pull_number, labels: [name] }); + const body = 'Thank you for your interest in the project! We are sorry, but we cannot accept more PRs while we work through the existing backlog. This PR has been automatically closed and labeled "intake paused". This is not a judgment of the quality of your changes.'; + const comments = await github.paginate(github.rest.issues.listComments, { + owner, repo, issue_number: pull_number, per_page: 100 + }); + if (!comments.some((comment) => comment.user.login === 'github-actions[bot]' && comment.body === body)) { + await github.rest.issues.createComment({ owner, repo, issue_number: pull_number, body }); + } + await github.rest.pulls.update({ owner, repo, pull_number, state: 'closed' }); diff --git a/.github/workflows/progress-report.yml b/.github/workflows/progress-report.yml index 3fbf7361b..1a69d7338 100644 --- a/.github/workflows/progress-report.yml +++ b/.github/workflows/progress-report.yml @@ -2,6 +2,10 @@ name: Progress report on: pull_request: + paths: + - 'core/libs/prx/**' + - 'tools/progress.py' + - '.github/workflows/progress-report.yml' permissions: contents: read diff --git a/CMakeLists.txt b/CMakeLists.txt index 765d268e9..4c6923e7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,7 @@ function(add_test_executable target) endfunction() option(BUILD_TESTING "Build and register relinker tests" OFF) +option(ANYPS5_RELINKER_ONLY "Build the relinker without system libraries or third-party dependencies" OFF) if(BUILD_TESTING) enable_testing() if(MINGW) @@ -23,13 +24,34 @@ if(BUILD_TESTING) endif() option(ANYPS5_ENABLE_SPIRV_TOOLS "Enable SPIRV-Tools validation and optimization" OFF) -option(APS5_ENABLE_TIMING_LOG "Enable frame timing logging" OFF) +option(APS5_ENABLE_TIMING_LOG "Write frame timings asynchronously to frame-timing.log" OFF) add_compile_definitions(APS5_ENABLE_TIMING_LOG=$) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -static-libgcc -static-libstdc++") +if(NOT APPLE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -static-libgcc -static-libstdc++") +endif() + +if(BUILD_TESTING) + find_package(Python3 QUIET COMPONENTS Interpreter) + if(Python3_Interpreter_FOUND) + add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py) + add_test(NAME progress_report COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress.py) + add_test(NAME import_audit COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py) + add_test(NAME hardware_oracle COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/hw-oracle/test_hw_oracle.py) + endif() +endif() + +if(ANYPS5_RELINKER_ONLY) + add_subdirectory(core/relinker) + return() +endif() set(SDL_SHARED OFF CACHE BOOL "" FORCE) set(SDL_STATIC ON CACHE BOOL "" FORCE) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a901ee126..fc1d7a685 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,10 +13,13 @@ - The relinker uses only the C++20 standard library. - Third-party code is added as a submodule under `3rdparty/` and built from source, not found on the system. - Don't add tests that only check that a symbol is exported: a missing export already fails at startup. +- Each test finishes in 30 s on the CI runners, Vulkan tests included (Linux runs them on lavapipe), counting the first run with no shader cache. ctest runs in parallel there, so a slow test competes for the cores and times out on some runs and not others. Split or shrink a test that gets close instead of raising its timeout. CI reports every test over 30 s as a warning on the pull request. ## Build and test -Toolchains are listed in the [README](README.md#build). +Toolchains are listed in the [build instructions](docs/dev/BUILD.md). + +For relinker changes, use the [relinker-only build](docs/dev/BUILD.md#relinker-only) to build and test without third-party submodules, system libraries or a GPU. It also works on macOS. Changes to system libraries or shaders still need the full build and relevant runtime tests. ``` cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON @@ -26,7 +29,11 @@ ctest --test-dir build --output-on-failure Python 3 is optional; without it some relinker tests are not registered. -The Conventions check runs on every pull request and fails when a rule on this page is broken. It accepts code comments only when the pull request also changes [TechnicalDebt](docs/dev/TechnicalDebt.md), and only UTF-8 text files. Run it locally before pushing: +Every push to `main`, including a merged pull request, automatically runs full Linux and Windows builds and tests and the relinker matrix on Linux, Windows and macOS, using the pushed commit without contributor checks. + +For pull requests, full Linux and Windows builds and tests run only when a collaborator with write access selects Actions > Build > Run workflow on `main` and enters an open pull request number targeting `main`. This manual run also builds and tests the relinker on Linux, Windows and macOS, regardless of the author's previous contributions or the changed files. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main`; full builds never run automatically for pull requests. All build jobs build the selected pull request merged into the current `main` (GitHub's merge commit), so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds. + +The Conventions check runs automatically for every contributor, including first-time contributors. It uses the base branch checker and reads pull request Git objects without executing pull request code. The Conventions check runs on every pull request and fails when a rule on this page is broken. It accepts code comments only when the pull request also changes [TechnicalDebt](docs/dev/TechnicalDebt.md), and only UTF-8 text files. Run it locally before pushing: ``` python3 tools/check_conventions.py --base origin/main diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index f6c6df5e0..5569c3a1b 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -131,6 +131,22 @@ endforeach() add_custom_target(libs DEPENDS ${PATCHED_OUTPUTS}) if(BUILD_TESTING) + add_test_executable(patched_libraries_tests tests/PatchedLibraries.cpp) + add_dependencies(patched_libraries_tests libs) + if(MINGW) + target_compile_options(patched_libraries_tests PRIVATE -fasynchronous-unwind-tables) + target_link_options(patched_libraries_tests PRIVATE -static -static-libgcc -static-libstdc++) + endif() + add_test(NAME patched_libraries COMMAND patched_libraries_tests ${PATCHED_DIR}) + add_test_executable(guest_offline_imports_tests tests/GuestOfflineImports.cpp) + target_include_directories(guest_offline_imports_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(guest_offline_imports_tests PRIVATE libSceHttp libSceNpEntitlementAccess libSceRudp libc) + configure_windows_unwind(guest_offline_imports_tests) + add_test(NAME guest_offline_imports COMMAND guest_offline_imports_tests) + if(WIN32) + set_tests_properties(guest_offline_imports PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(savedata_memory_metadata_tests tests/SaveDataMemoryMetadata.cpp) target_include_directories(savedata_memory_metadata_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(savedata_memory_metadata_tests PRIVATE libSceSaveData.native) diff --git a/core/libs/SceTypes.hpp b/core/libs/SceTypes.hpp index 5bd1b5553..887b13d57 100644 --- a/core/libs/SceTypes.hpp +++ b/core/libs/SceTypes.hpp @@ -1219,6 +1219,9 @@ using NetCtlCallback = void (*)(int, void*); struct HttpEpoll {}; using HttpEpollHandle = HttpEpoll*; using HttpsCallback = int (*)(int, unsigned int, void* const*, int, void*); +using HttpRedirectCallback = int (*)(int, std::int32_t, std::int32_t*, const char*, void*); +using HttpCookieRecvCallback = int (*)(int, const char*, const char*, std::uint64_t, void*); +using HttpAuthInfoCallback = int (*)(int, int, const char*, char*, char*, int, std::uint8_t**, std::uint64_t*, int*, void*); struct HttpNBEvent { std::uint8_t opaque[64]; }; @@ -1243,6 +1246,14 @@ struct Http2AsyncResult { void* reserved; }; +struct Http2AsyncOption { + KernelEqueue equeue; + int user_event_id; + std::uint8_t padding[4]; + void* user_data; + void* reserved; +}; + struct NpTitleId { char data[13]; char pad[3]; }; struct NpTitleSecret { std::uint8_t data[128]; }; struct NpContentRestriction { std::uint8_t opaque[128]; }; @@ -1266,6 +1277,10 @@ struct NpEntitlementAccessAddcontEntitlementInfo { std::uint32_t download_status; }; +struct NpEntitlementAccessEntitlementKey { + std::uint8_t data[16]; +}; + struct NpUniversalDataSystemInitParam { @@ -1375,7 +1390,7 @@ struct SaveDataMountInfo { }; struct SceSaveDataTitleId { char data[10]; char pad[2]; }; -struct SceSaveDataDirName { char data[33]; char pad[3]; }; +struct SceSaveDataDirName { char data[32]; }; struct SaveDataSearchInfo { std::uint8_t opaque[128]; }; struct SaveDataMemoryData { void* buf; std::size_t buf_size; std::size_t offset; }; @@ -1838,12 +1853,6 @@ struct LibcHeapInfo { using Info = LibcHeapInfo; -#define VA_ARGS \ - std::uint64_t rdi, std::uint64_t rsi, std::uint64_t rdx, std::uint64_t rcx, \ - std::uint64_t r8, std::uint64_t r9, std::uint64_t overflow_arg_area, \ - __m128 xmm0, __m128 xmm1, __m128 xmm2, __m128 xmm3, \ - __m128 xmm4, __m128 xmm5, __m128 xmm6, __m128 xmm7, ... - struct Packet { std::uint32_t* addr; std::uint32_t dw_num; diff --git a/core/libs/nid/include/nid/ElfPatcher.hpp b/core/libs/nid/include/nid/ElfPatcher.hpp index 8dbc7d4bf..a0194d7fa 100644 --- a/core/libs/nid/include/nid/ElfPatcher.hpp +++ b/core/libs/nid/include/nid/ElfPatcher.hpp @@ -91,6 +91,7 @@ inline bool IsDynamicNameTag(std::int64_t tag) { return tag == kDtNeeded || tag == kDtSoname || tag == kDtRpath || tag == kDtRunpath; } constexpr std::uint32_t kShtDynamic = 6u; +constexpr std::uint32_t kShtGnuVerdef = 0x6ffffffdu; constexpr std::uint32_t kShtGnuVerneed = 0x6ffffffeu; constexpr std::uint32_t kShtDynsym = 11u; constexpr std::uint32_t kShtGnuHash = 0x6ffffff6u; diff --git a/core/libs/nid/src/ElfNidPatcher.cpp b/core/libs/nid/src/ElfNidPatcher.cpp index 319cf1feb..4f341bbd2 100644 --- a/core/libs/nid/src/ElfNidPatcher.cpp +++ b/core/libs/nid/src/ElfNidPatcher.cpp @@ -337,6 +337,8 @@ void ElfNidPatcher::PatchNids(std::vector& elf, const std::string& } if (shdr.sh_type == kShtGnuVersym) versymOffset = static_cast(shdr.sh_offset); + if (shdr.sh_type == kShtGnuVerdef) + throw std::runtime_error("unsupported .gnu.version_d section: symbol version definitions cannot be preserved"); } if (dynSymOffset == 0u) throw std::runtime_error("no .dynsym section"); diff --git a/core/libs/nid/tests/test_symbol_versions.py b/core/libs/nid/tests/test_symbol_versions.py new file mode 100644 index 000000000..b824a374c --- /dev/null +++ b/core/libs/nid/tests/test_symbol_versions.py @@ -0,0 +1,209 @@ +import re +import struct +import subprocess +import sys +import tempfile +from pathlib import Path + +SHT_NULL = 0 +SHT_STRTAB = 3 +SHT_DYNSYM = 11 +SHT_GNU_HASH = 0x6FFFFFF6 +SHT_GNU_VERDEF = 0x6FFFFFFD +SHT_GNU_VERSYM = 0x6FFFFFFF +SHN_UNDEF = 0 +SHN_ABS = 0xFFF1 +STB_GLOBAL = 1 +STT_FUNC = 2 + +LIBRARY_NAME = "libsymbols" +DYNSYM_OFFSET = 0x200 +DYNSTR_OFFSET = 0x300 +HASH_OFFSET = 0x500 +VERSYM_OFFSET = 0x580 +VERDEF_OFFSET = 0x600 +SHOFF = 0x800 +BUCKETS = 4 +HASHED_FROM = 3 +NID_NAME = re.compile(r"^[A-Za-z0-9+-]{11}$") + +SONAME = "libsymbols.so" +VERSIONS = [(1, [SONAME]), (2, ["VERS_ONE"]), (3, ["VERS_TWO", "VERS_ONE"])] +MARKERS = [("VERS_ONE", 2, 0x1100), ("VERS_TWO", 3, 0x1101)] +DEFINITIONS = [ + ("sceAlphaComputeLength", 2, 0x1200), + ("sceAlphaCompareStrings", 2, 0x1220), + ("sceAlphaFindCharacter", 2, 0x1240), + ("sceBetaCopyString", 3, 0x1260), + ("sceBetaConcatenateStrings", 3, 0x1280), + ("sceBetaSearchCharacter", 3, 0x12A0), +] +IMPORTS = ["jeJEvW13vxk", "e0m0E5Sq2N8"] + + +def gnu_hash(name): + value = 5381 + for byte in name.encode("utf-8"): + value = (value * 33 + byte) & 0xFFFFFFFF + return value + + +def symbols(versioned): + entries = [(None, 1, SHN_UNDEF, 0, 0)] + entries += [(name, 1, SHN_UNDEF, 0, 0) for name in IMPORTS] + if versioned: + entries += [(name, version, SHN_ABS, 0, value) for name, version, value in MARKERS] + entries += [(name, version, 9, STT_FUNC, value) for name, version, value in DEFINITIONS] + else: + entries += [(name, 1, 9, STT_FUNC, value) for name, _, value in DEFINITIONS] + return entries + + +def string_table(names): + offsets = {} + table = bytearray(b"\x00") + for name in names: + offsets.setdefault(name, len(table)) + table += name.encode("utf-8") + b"\x00" + return table, offsets + + +def build_image(entries, versioned): + image = bytearray(0x1000) + image[:16] = b"\x7fELF\x02\x01\x01" + bytes(9) + names = [entry[0] for entry in entries if entry[0]] + if versioned: + names += [name for _, version_names in VERSIONS for name in version_names] + strings, offsets = string_table(names) + for index, (name, version, shndx, symbol_type, value) in enumerate(entries): + info = (STB_GLOBAL << 4) | symbol_type + struct.pack_into("") @@ -236,9 +237,13 @@ if(BUILD_TESTING) endif() endif() -add_test_executable(agc_interpolant_mapping_tests EXCLUDE_FROM_ALL tests/InterpolantMapping.cpp) +add_test_executable(agc_interpolant_mapping_tests EXCLUDE_FROM_ALL + tests/InterpolantMapping.cpp + Shader/src/InterpolantMapping.cpp + Shader/src/ShaderUtils.cpp +) target_include_directories(agc_interpolant_mapping_tests PRIVATE ${LIBS_INCLUDE_DIR}) -target_link_libraries(agc_interpolant_mapping_tests PRIVATE libSceAgc libc) +target_link_libraries(agc_interpolant_mapping_tests PRIVATE libSceAgcDriver libc) set_target_properties(agc_interpolant_mapping_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_interpolant_mapping_tests) @@ -252,9 +257,15 @@ if(BUILD_TESTING) endif() if(BUILD_TESTING) - add_test_executable(guest_shader_link_tests ${LIBS_INCLUDE_DIR}/tests/GuestShaderLink.cpp) + add_test_executable(guest_shader_link_tests + ${LIBS_INCLUDE_DIR}/tests/GuestShaderLink.cpp + Shader/src/LinkShaders.cpp + Shader/src/PrimState.cpp + Shader/src/InterpolantMapping.cpp + Shader/src/ShaderUtils.cpp + ) target_include_directories(guest_shader_link_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_shader_link_tests PRIVATE libSceAgc libc) + target_link_libraries(guest_shader_link_tests PRIVATE libSceAgc libSceAgcDriver libc) configure_windows_unwind(guest_shader_link_tests) add_test(NAME guest_shader_link COMMAND guest_shader_link_tests) if(WIN32) @@ -358,3 +369,22 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +add_test_executable(agc_shader_helpers_tests EXCLUDE_FROM_ALL + tests/ShaderHelpers.cpp + Shader/src/LinkShaders.cpp + Shader/src/PrimState.cpp + Shader/src/InterpolantMapping.cpp + Shader/src/ShaderUtils.cpp +) +target_include_directories(agc_shader_helpers_tests PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(agc_shader_helpers_tests PRIVATE libc libSceAgcDriver) +configure_windows_unwind(agc_shader_helpers_tests) +if(BUILD_TESTING) + set_target_properties(agc_shader_helpers_tests PROPERTIES EXCLUDE_FROM_ALL FALSE) + add_test(NAME agc_shader_helpers COMMAND agc_shader_helpers_tests) + if(WIN32) + set_tests_properties(agc_shader_helpers PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceAgc/Command/src/Packet.cpp b/core/libs/prx/libSceAgc/Command/src/Packet.cpp index 9a9a1827a..4a4e9e318 100644 --- a/core/libs/prx/libSceAgc/Command/src/Packet.cpp +++ b/core/libs/prx/libSceAgc/Command/src/Packet.cpp @@ -40,7 +40,7 @@ std::uint32_t available(const CommandBuffer& buffer, const char* function) { const auto bottom = reinterpret_cast(buffer.bottom); const auto top = reinterpret_cast(buffer.top); const auto up = reinterpret_cast(buffer.cursor_up); - const auto down = reinterpret_cast(buffer.cursor_down); + const auto down = buffer.cursor_down != nullptr ? reinterpret_cast(buffer.cursor_down) : top; Require(((bottom | top | up | down) & 3u) == 0, function, "misaligned command buffer"); Require(bottom <= up && up <= down && down <= top, function, "invalid command buffer cursors"); Require(bottom != 0 || top == 0, function, "null command buffer storage"); diff --git a/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp b/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp index e93ad3279..f304439f4 100644 --- a/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp +++ b/core/libs/prx/libSceAgc/DcbDraw/src/IndexBuffer.cpp @@ -35,8 +35,7 @@ std::uint32_t APS5_VABI sceAgcDcbSetIndexSizeGetSize() { return 12; } -APS5_EXPORT("-KRzWekV120", sceAgcUnknown__MKRzWekV120); -std::uint32_t* APS5_VABI sceAgcUnknown__MKRzWekV120(CommandBuffer* buf, std::uint8_t indexSize, std::uint8_t cachePolicy, std::uint8_t perInstanceObjectId) { +std::uint32_t* APS5_VABI sceAgcDcbSetIndexSize_0600(CommandBuffer* buf, std::uint8_t indexSize, std::uint8_t cachePolicy, std::uint8_t perInstanceObjectId) { Agc::Command::CheckBits(perInstanceObjectId, 1, __func__); auto* packet = sceAgcDcbSetIndexSize(buf, indexSize, cachePolicy); packet[2] |= static_cast(perInstanceObjectId) << 14u; diff --git a/core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp b/core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp index 8e78f4864..ffdf58c81 100644 --- a/core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp +++ b/core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp @@ -26,8 +26,7 @@ uint64_t APS5_VABI sceAgcDcbContextStateOpGetSize(uint32_t operation) { return 27 * sizeof(std::uint32_t); } -APS5_EXPORT("qj7QZpgr9Uw", sceAgcDcbContextStateAnotherOp); -uint32_t* APS5_VABI sceAgcDcbContextStateAnotherOp(CommandBuffer* buf, uint32_t operation) { +uint32_t* APS5_VABI sceAgcDcbContextStateOp_0100(CommandBuffer* buf, uint32_t operation) { Agc::Command::Require(buf != nullptr, __func__, "null command buffer"); Agc::Command::Require(operation <= 3, __func__, "invalid context state operation"); const auto* function = __func__; diff --git a/core/libs/prx/libSceAgc/DcbState/src/Init.cpp b/core/libs/prx/libSceAgc/DcbState/src/Init.cpp index 1b63094d1..7ad33c6d9 100644 --- a/core/libs/prx/libSceAgc/DcbState/src/Init.cpp +++ b/core/libs/prx/libSceAgc/DcbState/src/Init.cpp @@ -9,8 +9,7 @@ extern "C" { -APS5_EXPORT("23LRUSvYu1M", sceAgcUnknownInitState); -int APS5_VABI sceAgcUnknownInitState(std::uint32_t* state, std::uint32_t version) { +int APS5_VABI sceAgcInit_0090(std::uint32_t* state, std::uint32_t version) { Agc::Command::CheckAddress(reinterpret_cast(state), alignof(std::uint32_t), __func__); Agc::Command::Require(version < 14, __func__, "unsupported AGC initialization version"); return 0; diff --git a/core/libs/prx/libSceAgc/Misc/include/ShaderFusion.hpp b/core/libs/prx/libSceAgc/Misc/include/ShaderFusion.hpp index 0be677ef1..0927dc9e5 100644 --- a/core/libs/prx/libSceAgc/Misc/include/ShaderFusion.hpp +++ b/core/libs/prx/libSceAgc/Misc/include/ShaderFusion.hpp @@ -10,7 +10,7 @@ struct SizeAlign { std::size_t m_align; }; -extern "C" int APS5_VABI sceAgcUnknownFuseShaderHalves(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem); -extern "C" int APS5_VABI sceAgcUnknownGetFusedShaderSize(SizeAlign* dst, const Shader* front, const Shader* back); +extern "C" int APS5_VABI sceAgcFuseShaderHalves_0200(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem); +extern "C" int APS5_VABI sceAgcGetFusedShaderSize_0080(SizeAlign* dst, const Shader* front, const Shader* back); #endif diff --git a/core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp b/core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp index 7b7dff0f4..655ce3fae 100644 --- a/core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp +++ b/core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp @@ -17,8 +17,7 @@ uint32_t APS5_VABI sceAgcGetPacketSize(uint32_t* packet) { return ((packet[0] >> 16u) & 0x3fffu) + 2u; } -APS5_EXPORT("V++UgBtQhn0", sceAgcGetDataPacketPayloadAddressUnk); -int APS5_VABI sceAgcGetDataPacketPayloadAddressUnk(uint32_t** addr, uint32_t* cmd, int type) { +int APS5_VABI sceAgcGetDataPacketPayloadAddress_0090(uint32_t** addr, uint32_t* cmd, int type) { Agc::Command::CheckAddress(reinterpret_cast(addr), alignof(uint32_t*), __func__); Agc::Command::CheckAddress(reinterpret_cast(cmd), alignof(uint32_t), __func__); if (type != 0) { diff --git a/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp b/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp index 44c79a8f7..499217f13 100644 --- a/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp +++ b/core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp @@ -28,6 +28,7 @@ constexpr std::uint32_t SPI_SHADER_PGM_RSRC2_HS = 0x10Bu; constexpr std::uint32_t SCodeEnd = 0xbf9f0000u; constexpr std::uint32_t SNop = 0xbf800000u; constexpr std::uint32_t SSetpcS6 = 0xbe802006u; +constexpr std::uint32_t SSwappcNullS6 = 0xbefd2106u; constexpr std::uint8_t FusedCodeAlignmentLog2 = 8; constexpr std::size_t FusedCodeAlignment = std::size_t{1} << FusedCodeAlignmentLog2; constexpr char TrailerMagic[8] = {'b', 'a', 'r', 'e', 'f', 'o', 'o', 't'}; @@ -51,7 +52,7 @@ ShaderRegister& FindRegister(ShaderRegister* regs, std::uint32_t count, std::uin for (std::uint32_t i = 0; regs != nullptr && i < count; ++i) { if (regs[i].offset == offset && occurrence-- == 0) return regs[i]; } - throw std::runtime_error("sceAgcUnknownFuseShaderHalves: shader half lacks register " + std::to_string(offset)); + throw std::runtime_error("sceAgcFuseShaderHalves_0200: shader half lacks register " + std::to_string(offset)); } void MergeMax(ShaderRegister& dst, const ShaderRegister& src, std::uint32_t shift, std::uint32_t mask) { @@ -114,7 +115,7 @@ std::uint32_t FrontProgramBytes(const char* function, const Shader* front) { return value; }; while (end > 0 && word(end - 1) == SCodeEnd) --end; - if (end == 0 || word(end - 1) != SSetpcS6) Fail(function, "front half does not end with s_setpc_b64 s[6:7]"); + if (end == 0 || (word(end - 1) != SSetpcS6 && word(end - 1) != SSwappcNullS6)) Fail(function, "front half does not end with s_setpc_b64 s[6:7] or s_swappc_b64 null, s[6:7]"); return static_cast((end - 1) * 4); } @@ -187,11 +188,11 @@ void SetProgramAddress(std::vector& regs, std::uint32_t loOffset regs[i + 1].value = (regs[i + 1].value & 0xFFFFFF00u) | static_cast((address >> 40u) & 0xFFu); return; } - throw std::runtime_error("sceAgcUnknownFuseShaderHalves: back half has no program address register pair"); + throw std::runtime_error("sceAgcFuseShaderHalves_0200: back half has no program address register pair"); } int FuseGeometryHalves(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { - constexpr auto fn = "sceAgcUnknownFuseShaderHalves"; + constexpr auto fn = "sceAgcFuseShaderHalves_0200"; if (scratch_mem == nullptr) Fail(fn, "geometry halves need fused shader memory"); auto layout = ComputeGeometryLayout(fn, front, back); const auto scratch = reinterpret_cast(scratch_mem); @@ -251,8 +252,7 @@ int FuseGeometryHalves(Shader* fused_result, const Shader* front, const Shader* extern "C" { -APS5_EXPORT("fd5Bp5tGTgo", sceAgcUnknownFuseShaderHalves); -int APS5_VABI sceAgcUnknownFuseShaderHalves(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { +int APS5_VABI sceAgcFuseShaderHalves_0200(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { if (fused_result == nullptr || front == nullptr || back == nullptr) APS5_INVALID_ARG_EX; if (!ValidHalves(front, back)) return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; const bool isGs = GeometryHalves(front); @@ -298,15 +298,14 @@ int APS5_VABI sceAgcUnknownFuseShaderHalves(Shader* fused_result, const Shader* } int APS5_VABI sceAgcFuseShaderHalves_nid_postfix(Shader* fused_result, const Shader* front, const Shader* back, void* scratch_mem) { - return sceAgcUnknownFuseShaderHalves(fused_result, front, back, scratch_mem); + return sceAgcFuseShaderHalves_0200(fused_result, front, back, scratch_mem); } -APS5_EXPORT("dolOmWH+huQ", sceAgcUnknownGetFusedShaderSize); -int APS5_VABI sceAgcUnknownGetFusedShaderSize(SizeAlign* dst, const Shader* front, const Shader* back) { +int APS5_VABI sceAgcGetFusedShaderSize_0080(SizeAlign* dst, const Shader* front, const Shader* back) { if (dst == nullptr || front == nullptr || back == nullptr) APS5_INVALID_ARG_EX; if (!ValidHalves(front, back)) return GRAPHICS5_ERROR_INVALID_SHADER_HALVES; if (GeometryHalves(front)) { - const auto layout = ComputeGeometryLayout("sceAgcUnknownGetFusedShaderSize", front, back); + const auto layout = ComputeGeometryLayout("sceAgcGetFusedShaderSize_0080", front, back); dst->m_size = layout.totalBytes + FusedCodeAlignment - 1; dst->m_align = FusedCodeAlignmentLog2; return 0; @@ -316,8 +315,7 @@ int APS5_VABI sceAgcUnknownGetFusedShaderSize(SizeAlign* dst, const Shader* fron return 0; } -APS5_EXPORT("k0E7vkgqAuE", sceAgcCreateInterpolantMappingVsPs); -int APS5_VABI sceAgcCreateInterpolantMappingVsPs(ShaderRegister* regs, const Shader* vs, const Shader* ps) { +int APS5_VABI sceAgcUpdateInterpolantMapping_0100(ShaderRegister* regs, const Shader* vs, const Shader* ps) { (void)regs; (void)vs; (void)ps; diff --git a/core/libs/prx/libSceAgc/Patch/src/Jump.cpp b/core/libs/prx/libSceAgc/Patch/src/Jump.cpp index e31894dd9..833c22210 100644 --- a/core/libs/prx/libSceAgc/Patch/src/Jump.cpp +++ b/core/libs/prx/libSceAgc/Patch/src/Jump.cpp @@ -28,8 +28,7 @@ int APS5_VABI sceAgcJumpPatchSetTarget(uint32_t* cmd, const volatile uint32_t* t return 0; } -APS5_EXPORT("Ikfdt-rIqCE", sceAgcUnknown_Ikfdt_MrIqCE); -int APS5_VABI sceAgcUnknown_Ikfdt_MrIqCE(uint32_t* cmd, uint64_t cache_policy, const volatile uint32_t* target, uint32_t size_in_dwords) { +int APS5_VABI sceAgcJumpPatchSetTarget_0300(uint32_t* cmd, uint64_t cache_policy, const volatile uint32_t* target, uint32_t size_in_dwords) { Agc::Command::CheckBits(cache_policy, 3u, __func__); PatchJump(cmd, target, size_in_dwords, __func__); cmd[3] = (cmd[3] & ~(3u << 28u)) | (static_cast(cache_policy) << 28u); diff --git a/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp b/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp index 5ce4d1555..3e70bd95b 100644 --- a/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp +++ b/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp @@ -1,10 +1,14 @@ #include "prx/libSceAgc/Shader/include/InterpolantMapping.hpp" #include +#include +#include #include #include #include "SceShaders.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgc/Shader/include/ShaderUtils.hpp" #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" @@ -53,9 +57,22 @@ int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader* if (regs == nullptr) { throw std::runtime_error(std::string(fn) + ": regs is null"); } + if (ps != nullptr && ps->num_input_semantics > 32) { + throw std::runtime_error(std::string(fn) + ": input semantic count exceeds 32"); + } + AgcDriver::ShaderPreparationScope transaction; + std::array values{}; + auto* output = regs; + regs = values.data(); if (ps == nullptr || ps->num_input_semantics == 0) { FillIdentityInterpolants(regs, 0); + if (ps != nullptr) { + AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, 32}, {}); + if (gs != nullptr) AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0); + } + transaction.Commit(); + std::copy(values.begin(), values.end(), output); return 0; } @@ -82,6 +99,10 @@ int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader* } FillIdentityInterpolants(regs, ps->num_input_semantics); + AgcDriverResolveShaderAbi_nid_postfix(ps, {regs, 32}, {}); + AgcDriverResolveGraphicsAbi_nid_postfix(gs, ps, 0); + transaction.Commit(); + std::copy(values.begin(), values.end(), output); return 0; } diff --git a/core/libs/prx/libSceAgc/Shader/src/LinkShaders.cpp b/core/libs/prx/libSceAgc/Shader/src/LinkShaders.cpp index 8f0b7e266..b58f19b40 100644 --- a/core/libs/prx/libSceAgc/Shader/src/LinkShaders.cpp +++ b/core/libs/prx/libSceAgc/Shader/src/LinkShaders.cpp @@ -1,4 +1,6 @@ #include "SceShaders.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" #include @@ -20,10 +22,13 @@ extern "C" int APS5_VABI sceAgcLinkShaders(ShaderRegister* context, ShaderRegist // The simple vertex/pixel pipeline combines two primitive context registers // and 32 interpolant registers. Publish only after both helpers succeed. + AgcDriver::ShaderPreparationScope transaction; std::array contextValues{}; std::array primitiveValues{}; sceAgcCreatePrimState(contextValues.data(), primitiveValues.data(), nullptr, vertex, primitiveType); sceAgcCreateInterpolantMapping(contextValues.data() + 2, vertex, pixel); + AgcDriverResolveGraphicsAbi_nid_postfix(vertex, pixel, primitiveType); + transaction.Commit(); std::copy(contextValues.begin(), contextValues.end(), context); std::copy(primitiveValues.begin(), primitiveValues.end(), primitive); return 0; diff --git a/core/libs/prx/libSceAgc/Shader/src/PrimState.cpp b/core/libs/prx/libSceAgc/Shader/src/PrimState.cpp index 365a70a3d..8d3564c76 100644 --- a/core/libs/prx/libSceAgc/Shader/src/PrimState.cpp +++ b/core/libs/prx/libSceAgc/Shader/src/PrimState.cpp @@ -1,10 +1,13 @@ #include "prx/libSceAgc/Shader/include/PrimState.hpp" #include +#include +#include #include #include #include "SceShaders.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgc/Shader/include/ShaderUtils.hpp" #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" @@ -22,44 +25,26 @@ int APS5_VABI sceAgcCreatePrimState(ShaderRegister* cx_regs, ShaderRegister* uc_ throw std::runtime_error(std::string(__func__) + ": invalid shader type or missing special registers"); } (void)GraphicsPrimTypeToGsOut(prim_type); - if (cx_regs != nullptr) { - const auto valid = [](const Shader* shader) { return shader->specials->vgt_shader_stages_en.offset == ShaderRegs::VGT_SHADER_STAGES_EN && shader->specials->vgt_gs_out_prim_type.offset == ShaderRegs::VGT_GS_OUT_PRIM_TYPE; }; - if (!valid(gs) || (hs != nullptr && !valid(hs))) { - throw std::runtime_error(std::string(__func__) + ": invalid context register offsets"); - } + const auto valid = [](const Shader* shader) { return shader->specials->vgt_shader_stages_en.offset == ShaderRegs::VGT_SHADER_STAGES_EN && shader->specials->vgt_gs_out_prim_type.offset == ShaderRegs::VGT_GS_OUT_PRIM_TYPE; }; + if (!valid(gs) || (hs != nullptr && !valid(hs))) { + throw std::runtime_error(std::string(__func__) + ": invalid context register offsets"); } - if (uc_regs != nullptr && (gs->specials->ge_cntl.offset != ShaderRegs::GE_CNTL || gs->specials->ge_user_vgpr_en.offset != ShaderRegs::GE_USER_VGPR_EN || (hs != nullptr && hs->specials->ge_user_vgpr_en.offset != ShaderRegs::GE_USER_VGPR_EN))) { + if (gs->specials->ge_cntl.offset != ShaderRegs::GE_CNTL || gs->specials->ge_user_vgpr_en.offset != ShaderRegs::GE_USER_VGPR_EN || (hs != nullptr && hs->specials->ge_user_vgpr_en.offset != ShaderRegs::GE_USER_VGPR_EN)) { throw std::runtime_error(std::string(__func__) + ": invalid user configuration register offsets"); } - - if (cx_regs != nullptr) { - cx_regs[0] = gs->specials->vgt_shader_stages_en; - if ((cx_regs[0].value & ShaderRegs::VGT_SHADER_STAGES_GS_BIT) != 0) { - cx_regs[1] = gs->specials->vgt_gs_out_prim_type; - } else { - cx_regs[1].offset = ShaderRegs::VGT_GS_OUT_PRIM_TYPE; - cx_regs[1].value = GraphicsPrimTypeToGsOut(prim_type); - } - - if (hs != nullptr) { - cx_regs[0].value |= hs->specials->vgt_shader_stages_en.value; - if ((cx_regs[0].value & ShaderRegs::VGT_SHADER_STAGES_GS_BIT) == 0) { - cx_regs[1] = hs->specials->vgt_gs_out_prim_type; - } - } + std::array contextValues{gs->specials->vgt_shader_stages_en, gs->specials->vgt_gs_out_prim_type}; + if ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_GS_BIT) == 0) { + contextValues[1] = {ShaderRegs::VGT_GS_OUT_PRIM_TYPE, GraphicsPrimTypeToGsOut(prim_type)}; } - - if (uc_regs != nullptr) { - uc_regs[0] = gs->specials->ge_cntl; - uc_regs[1] = gs->specials->ge_user_vgpr_en; - uc_regs[2].offset = ShaderRegs::VGT_PRIMITIVE_TYPE; - uc_regs[2].value = prim_type; - - if (hs != nullptr) { - uc_regs[1] = hs->specials->ge_user_vgpr_en; - } + if (hs != nullptr) { + contextValues[0].value |= hs->specials->vgt_shader_stages_en.value; + if ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_GS_BIT) == 0) contextValues[1] = hs->specials->vgt_gs_out_prim_type; } - + std::array primitiveValues{gs->specials->ge_cntl, hs != nullptr ? hs->specials->ge_user_vgpr_en : gs->specials->ge_user_vgpr_en, ShaderRegister{ShaderRegs::VGT_PRIMITIVE_TYPE, prim_type}}; + const std::array stages{hs, gs}; + AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span(stages).subspan(hs == nullptr ? 1u : 0u), contextValues, primitiveValues); + if (cx_regs != nullptr) std::copy(contextValues.begin(), contextValues.end(), cx_regs); + if (uc_regs != nullptr) std::copy(primitiveValues.begin(), primitiveValues.end(), uc_regs); return 0; } diff --git a/core/libs/prx/libSceAgc/Unimplemented.cpp b/core/libs/prx/libSceAgc/Unimplemented.cpp index 8c6244952..4841b8d7a 100644 --- a/core/libs/prx/libSceAgc/Unimplemented.cpp +++ b/core/libs/prx/libSceAgc/Unimplemented.cpp @@ -80,8 +80,7 @@ int APS5_VABI sceAgcCbMemsetExclusive() { return 0; } -APS5_EXPORT("7Wa3aeJgeVU", sceAgcUnknown_7Wa3aeJgeVU); -int APS5_VABI sceAgcUnknown_7Wa3aeJgeVU() { +int APS5_VABI sceAgcBranchPatchSetThenTarget_0300() { NotImplemented_nid_no_patch(__func__); return 0; } @@ -101,8 +100,7 @@ int APS5_VABI sceAgcGetShaderInstrumentation() { return 0; } -APS5_EXPORT("rP5xLdOf26k", sceAgcUnknown_rP5xLdOf26k); -int APS5_VABI sceAgcUnknown_rP5xLdOf26k() { +int APS5_VABI sceAgcBranchPatchSetElseTarget_0300() { NotImplemented_nid_no_patch(__func__); return 0; } diff --git a/core/libs/prx/libSceAgc/tests/Commands.cpp b/core/libs/prx/libSceAgc/tests/Commands.cpp index 8abf0f24a..58b9f1026 100644 --- a/core/libs/prx/libSceAgc/tests/Commands.cpp +++ b/core/libs/prx/libSceAgc/tests/Commands.cpp @@ -18,15 +18,15 @@ extern "C" int APS5_VABI sceAgcInit(std::uint32_t version); extern "C" void* APS5_VABI sceAgcGetRegisterDefaults(); extern "C" void* APS5_VABI sceAgcGetRegisterDefaultsInternal(); extern "C" void* APS5_VABI sceAgcGetRegisterDefaults2Internal(std::uint32_t version); -extern "C" int APS5_VABI sceAgcUnknownInitState(std::uint32_t* state, std::uint32_t version); +extern "C" int APS5_VABI sceAgcInit_0090(std::uint32_t* state, std::uint32_t version); extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexAuto(CommandBuffer* buf, std::uint32_t indexCount, std::uint64_t modifier); extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexIndirect(CommandBuffer* buf, std::uint32_t dataOffsetInBytes, std::uint64_t modifier); extern "C" std::uint32_t* APS5_VABI sceAgcDcbDrawIndexIndirectMulti(CommandBuffer* buf, std::uint32_t dataOffsetInBytes, std::uint32_t countIndirect, std::uint32_t maxCountOrCount, const volatile void* countAddress, std::uint32_t strideInBytes, std::uint64_t modifier); extern "C" int APS5_VABI sceAgcWaitRegMemPatchReference(std::uint32_t* cmd, std::uint64_t reference); extern "C" int APS5_VABI sceAgcWaitRegMemPatchMask(std::uint32_t* cmd, std::uint64_t mask); -extern "C" int APS5_VABI sceAgcGetDataPacketPayloadAddressUnk(std::uint32_t** addr, std::uint32_t* cmd, int type); +extern "C" int APS5_VABI sceAgcGetDataPacketPayloadAddress_0090(std::uint32_t** addr, std::uint32_t* cmd, int type); extern "C" std::uint32_t* APS5_VABI sceAgcCbSetShRegisterRangeDirect(CommandBuffer* buf, std::uint32_t offset, const std::uint32_t* values, std::uint32_t numValues); -extern "C" std::uint32_t* APS5_VABI sceAgcDcbContextStateAnotherOp(CommandBuffer* buf, std::uint32_t operation); +extern "C" std::uint32_t* APS5_VABI sceAgcDcbContextStateOp_0100(CommandBuffer* buf, std::uint32_t operation); extern "C" std::uint32_t* APS5_VABI sceAgcDcbPushMarker(CommandBuffer* buf, const char* str, std::uint32_t color); extern "C" std::uint32_t* APS5_VABI sceAgcDcbPopMarker(CommandBuffer* buf); extern "C" std::uint32_t* APS5_VABI sceAgcDcbSetMarker(CommandBuffer* buf, const char* str, std::uint32_t color); @@ -101,6 +101,12 @@ void testPackets() { exhausted.buffer.cursor_down = exhausted.words.data() + 2; expectFailure([&] { Agc::Command::WriteNop(&exhausted.buffer, 3, __func__); }); check(exhausted.buffer.cursor_up == exhausted.words.data(), "failed allocation advanced cursor"); + std::array scratch{}; + CommandBuffer noDown{scratch.data(), scratch.data() + scratch.size(), scratch.data(), nullptr, nullptr, nullptr, 0}; + Agc::Command::WriteNop(&noDown, 6, __func__); + check(noDown.cursor_up == scratch.data() + 6, "buffer without a down cursor did not use its top as the limit"); + expectFailure([&] { Agc::Command::WriteNop(&noDown, 3, __func__); }); + check(noDown.cursor_up == scratch.data() + 6, "allocation past the top of a buffer without a down cursor advanced its cursor"); } void testClearState() { @@ -222,7 +228,7 @@ void testContextState() { source.buffer.reserved_dw = 2; source.buffer.callback = growContext; source.buffer.user_data = &growth; - auto* first = sceAgcDcbContextStateAnotherOp(&source.buffer, operation); + auto* first = sceAgcDcbContextStateOp_0100(&source.buffer, operation); check(first == (split == 0 ? growth.destination.words.data() : source.words.data()), "incorrect first context packet address"); check(growth.calls == (requested == 0 ? 0u : 1u), "incorrect context callback count"); auto* end = requested == 0 ? source.words.data() + totals[operation] : growth.destination.words.data() + totals[operation] - split; @@ -243,24 +249,24 @@ void testContextState() { } } } - expectFailure([] { sceAgcDcbContextStateAnotherOp(nullptr, 0); }); + expectFailure([] { sceAgcDcbContextStateOp_0100(nullptr, 0); }); Storage invalid; - expectFailure([&] { sceAgcDcbContextStateAnotherOp(&invalid.buffer, 4); }); + expectFailure([&] { sceAgcDcbContextStateOp_0100(&invalid.buffer, 4); }); check(invalid.buffer.cursor_up == invalid.words.data(), "invalid context operation advanced cursor"); invalid.buffer.cursor_up = invalid.words.data() + 1; invalid.buffer.cursor_down = invalid.words.data(); - expectFailure([&] { sceAgcDcbContextStateAnotherOp(&invalid.buffer, 0); }); + expectFailure([&] { sceAgcDcbContextStateOp_0100(&invalid.buffer, 0); }); invalid.buffer.cursor_up = invalid.words.data(); - expectFailure([&] { sceAgcDcbContextStateAnotherOp(&invalid.buffer, 1); }); + expectFailure([&] { sceAgcDcbContextStateOp_0100(&invalid.buffer, 1); }); ContextGrowth growth{{}, invalid.words.data(), 22}; invalid.buffer.callback = growContext; invalid.buffer.user_data = &growth; growth.success = false; - expectFailure([&] { sceAgcDcbContextStateAnotherOp(&invalid.buffer, 1); }); + expectFailure([&] { sceAgcDcbContextStateOp_0100(&invalid.buffer, 1); }); growth.calls = 0; growth.success = true; growth.destination.buffer.cursor_down = growth.destination.words.data() + 21; - expectFailure([&] { sceAgcDcbContextStateAnotherOp(&invalid.buffer, 1); }); + expectFailure([&] { sceAgcDcbContextStateOp_0100(&invalid.buffer, 1); }); check(invalid.buffer.cursor_up == growth.destination.words.data(), "failed reservation advanced cursor"); } @@ -322,19 +328,19 @@ void testPacketPayloadAddress() { Storage storage; auto* packet = sceAgcCbSetShRegisterRangeDirect(&storage.buffer, 0x8c, nullptr, 4); std::uint32_t* payload = nullptr; - check(sceAgcGetDataPacketPayloadAddressUnk(&payload, packet, 1) == 0 && payload == packet + 2, "incorrect register packet payload address"); + check(sceAgcGetDataPacketPayloadAddress_0090(&payload, packet, 1) == 0 && payload == packet + 2, "incorrect register packet payload address"); const std::array values{11, 22, 33, 44}; std::copy(values.begin(), values.end(), payload); const std::array expected{0xc0047600u, 0x8c, 11, 22, 33, 44}; check(std::equal(expected.begin(), expected.end(), packet), "payload write corrupted register packet"); - check(sceAgcGetDataPacketPayloadAddressUnk(&payload, packet, 0) == 0 && payload == packet + 1, "incorrect generic packet payload address"); + check(sceAgcGetDataPacketPayloadAddress_0090(&payload, packet, 0) == 0 && payload == packet + 1, "incorrect generic packet payload address"); packet[0] = 0xffff1000u; - check(sceAgcGetDataPacketPayloadAddressUnk(&payload, packet, 0) == 0 && payload == nullptr, "empty payload marker was not recognized"); - check(sceAgcGetDataPacketPayloadAddressUnk(&payload, packet, -1) == 0 && payload == packet + 2, "nonzero payload type did not skip two words"); - expectFailure([&] { sceAgcGetDataPacketPayloadAddressUnk(nullptr, packet, 1); }); - expectFailure([&] { sceAgcGetDataPacketPayloadAddressUnk(&payload, nullptr, 1); }); + check(sceAgcGetDataPacketPayloadAddress_0090(&payload, packet, 0) == 0 && payload == nullptr, "empty payload marker was not recognized"); + check(sceAgcGetDataPacketPayloadAddress_0090(&payload, packet, -1) == 0 && payload == packet + 2, "nonzero payload type did not skip two words"); + expectFailure([&] { sceAgcGetDataPacketPayloadAddress_0090(nullptr, packet, 1); }); + expectFailure([&] { sceAgcGetDataPacketPayloadAddress_0090(&payload, nullptr, 1); }); auto* misaligned = reinterpret_cast(reinterpret_cast(packet) + 1); - expectFailure([&] { sceAgcGetDataPacketPayloadAddressUnk(&payload, misaligned, 0); }); + expectFailure([&] { sceAgcGetDataPacketPayloadAddress_0090(&payload, misaligned, 0); }); check(payload == packet + 2, "invalid packet changed output address"); } @@ -367,10 +373,10 @@ void testMemory() { void testDefaults() { std::uint32_t state = 0x12345678; - check(sceAgcUnknownInitState(&state, 8) == 0 && state == 0x12345678, "AGC initialization failed or modified caller state"); - check(sceAgcUnknownInitState(&state, 13) == 0 && state == 0x12345678, "AGC version 13 initialization changed caller state"); - expectFailure([] { sceAgcUnknownInitState(nullptr, 8); }); - expectFailure([&] { sceAgcUnknownInitState(&state, 14); }); + check(sceAgcInit_0090(&state, 8) == 0 && state == 0x12345678, "AGC initialization failed or modified caller state"); + check(sceAgcInit_0090(&state, 13) == 0 && state == 0x12345678, "AGC version 13 initialization changed caller state"); + expectFailure([] { sceAgcInit_0090(nullptr, 8); }); + expectFailure([&] { sceAgcInit_0090(&state, 14); }); check(sceAgcInit(8) == 0, "AGC version initialization failed"); expectFailure([] { sceAgcInit(14); }); expectFailure([] { sceAgcInit(0xffffffffu); }); diff --git a/core/libs/prx/libSceAgc/tests/ContextStateSize.cpp b/core/libs/prx/libSceAgc/tests/ContextStateSize.cpp index 130b00848..415b81e36 100644 --- a/core/libs/prx/libSceAgc/tests/ContextStateSize.cpp +++ b/core/libs/prx/libSceAgc/tests/ContextStateSize.cpp @@ -8,7 +8,7 @@ #include extern "C" std::uint64_t APS5_VABI sceAgcDcbContextStateOpGetSize(std::uint32_t operation); -extern "C" std::uint32_t* APS5_VABI sceAgcDcbContextStateAnotherOp(CommandBuffer* buf, std::uint32_t operation); +extern "C" std::uint32_t* APS5_VABI sceAgcDcbContextStateOp_0100(CommandBuffer* buf, std::uint32_t operation); namespace { @@ -34,7 +34,7 @@ void testSizes() { std::array words{}; CommandBuffer buffer{words.data(), words.data() + words.size(), words.data(), words.data() + words.size(), nullptr, nullptr, 0}; - check(sceAgcDcbContextStateAnotherOp(&buffer, operation) == words.data(), "context state op did not start at the cursor"); + check(sceAgcDcbContextStateOp_0100(&buffer, operation) == words.data(), "context state op did not start at the cursor"); const auto written = static_cast(buffer.cursor_up - words.data()) * sizeof(std::uint32_t); check(written == sizes[operation], "context state size differs from the written packets"); } diff --git a/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp b/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp index bee89bb8d..95e97ae37 100644 --- a/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp +++ b/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp @@ -4,10 +4,12 @@ #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" #include +#include #include #include #include #include +#include extern "C" int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps); extern "C" int APS5_VABI sceAgcUnknownCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps); @@ -15,6 +17,10 @@ extern "C" int APS5_VABI sceAgcUnknownCreateInterpolantMapping(ShaderRegister* r namespace { using Registers = std::array; +const Shader* mappedPixel = nullptr; +Registers preparedMapping{}; +unsigned mappings = 0; +unsigned links = 0; void check(bool condition, const char* message) { if (!condition) throw std::runtime_error(message); @@ -82,6 +88,7 @@ void testMapping() { auto split = filled(); check(sceAgcUnknownCreateInterpolantMapping(split.data(), &gs, &ps) == 0, "split f16 mapping failed"); + check(std::memcmp(split.data(), preparedMapping.data(), sizeof(split)) == 0, "prepared split mapping differs from published registers"); for (std::uint32_t i = 0; i < inputs.size(); ++i) { check(split[i].offset == ShaderRegs::SPI_PS_INPUT_CNTL_0 + i, "interpolant register offset changed"); } @@ -94,6 +101,7 @@ void testMapping() { auto regular = filled(); check(sceAgcCreateInterpolantMapping(regular.data(), &gs, &ps) == 0, "mapping failed"); + check(std::memcmp(regular.data(), preparedMapping.data(), sizeof(regular)) == 0, "prepared mapping differs from published registers"); check(regular[3].value == split[3].value && regular[4].value == split[4].value, "mappings differ outside the high f16 half mode"); checkIdentity(regular, static_cast(inputs.size()), "unused interpolants are not the identity"); } @@ -117,11 +125,24 @@ void testRejections() { } +extern "C" void AgcDriverResolveShaderAbi_nid_postfix(const Shader* shader, std::span context, std::span primitive) { + check(shader != nullptr && context.size() == preparedMapping.size() && primitive.empty(), "invalid mapping preparation call"); + mappedPixel = shader; + std::copy(context.begin(), context.end(), preparedMapping.begin()); + ++mappings; +} + +extern "C" void AgcDriverResolveGraphicsAbi_nid_postfix(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType) { + check(vertex != nullptr && pixel == mappedPixel && primitiveType == 0u, "invalid mapping link call"); + ++links; +} + int main() { try { testIdentity(); testMapping(); testRejections(); + check(mappings == 3u && links == 2u, "mapping preparation call count changed"); LibcRunShutdown_nid_postfix(); std::puts("AGC interpolant mapping tests passed"); return 0; diff --git a/core/libs/prx/libSceAgc/tests/PayloadRange.cpp b/core/libs/prx/libSceAgc/tests/PayloadRange.cpp index 97a386cc3..94f33c2ad 100644 --- a/core/libs/prx/libSceAgc/tests/PayloadRange.cpp +++ b/core/libs/prx/libSceAgc/tests/PayloadRange.cpp @@ -9,7 +9,7 @@ #include extern "C" int APS5_VABI sceAgcGetDataPacketPayloadRange(SceAgcMemoryRange* range, std::uint32_t* cmd, int type); -extern "C" int APS5_VABI sceAgcGetDataPacketPayloadAddressUnk(std::uint32_t** addr, std::uint32_t* cmd, int type); +extern "C" int APS5_VABI sceAgcGetDataPacketPayloadAddress_0090(std::uint32_t** addr, std::uint32_t* cmd, int type); namespace { @@ -42,7 +42,7 @@ void testRanges() { check(range.size == size, "payload range size mismatch"); check(words[0] == header && words[1] == 0x12345678u, "payload range query modified the packet"); std::uint32_t* address = nullptr; - check(sceAgcGetDataPacketPayloadAddressUnk(&address, words.data(), type) == 0 && address == range.base, + check(sceAgcGetDataPacketPayloadAddress_0090(&address, words.data(), type) == 0 && address == range.base, "payload range base differs from the payload address"); } } diff --git a/core/libs/prx/libSceAgc/tests/ShaderFusion.cpp b/core/libs/prx/libSceAgc/tests/ShaderFusion.cpp index 71c081572..c0a6a4eda 100644 --- a/core/libs/prx/libSceAgc/tests/ShaderFusion.cpp +++ b/core/libs/prx/libSceAgc/tests/ShaderFusion.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgc/Misc/include/ShaderFusion.hpp" +#include "prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp" #include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" #include "prx/libc/include/Shutdown.hpp" @@ -15,6 +16,7 @@ namespace { constexpr std::uint32_t SCodeEnd = 0xbf9f0000u; constexpr std::uint32_t SNop = 0xbf800000u; constexpr std::uint32_t SSetpcS6 = 0xbe802006u; +constexpr std::uint32_t SSwappcNullS6 = 0xbefd2106u; constexpr std::uint32_t SMovS0 = 0xbe800380u; constexpr std::uint32_t VMovV0 = 0x7e000280u; constexpr std::uint32_t SEndpgm = 0xbf810000u; @@ -67,8 +69,8 @@ struct Halves { Shader front{}; Shader back{}; - Halves() { - frontCode = {SMovS0, VMovV0, SSetpcS6, SCodeEnd, SCodeEnd}; + explicit Halves(std::uint32_t terminator = SSetpcS6) { + frontCode = {SMovS0, VMovV0, terminator, SCodeEnd, SCodeEnd}; const auto codeBytes = static_cast(frontCode.size() * 4); std::array magic{}; std::memcpy(magic.data(), "barefoot", 8); @@ -123,16 +125,16 @@ std::uint8_t* aligned(std::vector& storage, std::size_t alignment) return storage.data() + ((alignment - address % alignment) % alignment); } -void testFusion(std::size_t misalignment) { - Halves halves; +void testFusion(std::size_t misalignment, std::uint32_t terminator = SSetpcS6) { + Halves halves(terminator); SizeAlign size{}; - check(sceAgcUnknownGetFusedShaderSize(&size, &halves.front, &halves.back) == 0, "fused size query failed"); + check(sceAgcGetFusedShaderSize_0080(&size, &halves.front, &halves.back) == 0, "fused size query failed"); check(size.m_align == 8, "fused shaders are not 256-byte aligned"); check(size.m_size > 256 + 255 + halves.back.shader_size + sizeof(Shader), "fused size does not cover the alignment padding, code and header"); std::vector storage(size.m_size + 512, 0xcd); auto* scratch = aligned(storage, 256) + misalignment; Shader fused{}; - check(sceAgcUnknownFuseShaderHalves(&fused, &halves.front, &halves.back, scratch) == 0, "fusion failed"); + check(sceAgcFuseShaderHalves_0200(&fused, &halves.front, &halves.back, scratch) == 0, "fusion failed"); for (std::size_t i = 0; i < 16; ++i) check(scratch[size.m_size + i] == 0xcd, "fusion wrote past the reported size"); auto* memory = scratch + (256 - misalignment) % 256; const auto base = reinterpret_cast(memory); @@ -173,38 +175,38 @@ void testRejections() { Halves halves; SizeAlign size{}; Shader fused{}; - expectFailure([&] { sceAgcUnknownGetFusedShaderSize(nullptr, &halves.front, &halves.back); }); - expectFailure([&] { sceAgcUnknownFuseShaderHalves(&fused, &halves.front, &halves.back, nullptr); }); + expectFailure([&] { sceAgcGetFusedShaderSize_0080(nullptr, &halves.front, &halves.back); }); + expectFailure([&] { sceAgcFuseShaderHalves_0200(&fused, &halves.front, &halves.back, nullptr); }); { Halves swapped; - check(sceAgcUnknownGetFusedShaderSize(&size, &swapped.back, &swapped.front) == InvalidShaderHalves, "swapped halves were accepted by the size query"); - check(sceAgcUnknownFuseShaderHalves(&fused, &swapped.back, &swapped.front, nullptr) == InvalidShaderHalves, "swapped halves were fused"); + check(sceAgcGetFusedShaderSize_0080(&size, &swapped.back, &swapped.front) == InvalidShaderHalves, "swapped halves were accepted by the size query"); + check(sceAgcFuseShaderHalves_0200(&fused, &swapped.back, &swapped.front, nullptr) == InvalidShaderHalves, "swapped halves were fused"); } { Halves stages; stages.backSpecials.vgt_shader_stages_en.value = 0; std::vector storage(4096); - check(sceAgcUnknownFuseShaderHalves(&fused, &stages.front, &stages.back, storage.data()) == InvalidShaderHalves, "halves with different GS stages were fused"); + check(sceAgcFuseShaderHalves_0200(&fused, &stages.front, &stages.back, storage.data()) == InvalidShaderHalves, "halves with different GS stages were fused"); } { Halves noJump; noJump.frontCode[2] = SNop; - expectFailure([&] { sceAgcUnknownGetFusedShaderSize(&size, &noJump.front, &noJump.back); }); + expectFailure([&] { sceAgcGetFusedShaderSize_0080(&size, &noJump.front, &noJump.back); }); } { Halves noTrailer; noTrailer.frontCode[5] = 0; - expectFailure([&] { sceAgcUnknownGetFusedShaderSize(&size, &noTrailer.front, &noTrailer.back); }); + expectFailure([&] { sceAgcGetFusedShaderSize_0080(&size, &noTrailer.front, &noTrailer.back); }); } { Halves hardware; hardware.frontSh[2].value |= 1u << 16u; - expectFailure([&] { sceAgcUnknownGetFusedShaderSize(&size, &hardware.front, &hardware.back); }); + expectFailure([&] { sceAgcGetFusedShaderSize_0080(&size, &hardware.front, &hardware.back); }); } { Halves constants; constants.back.embedded_constant_buffer_size_dqw = 1; - expectFailure([&] { sceAgcUnknownGetFusedShaderSize(&size, &constants.front, &constants.back); }); + expectFailure([&] { sceAgcGetFusedShaderSize_0080(&size, &constants.front, &constants.back); }); } } @@ -228,10 +230,10 @@ void testHullHalves() { back.specials = &specials; back.type = static_cast(ShaderRegs::ShaderBinaryType::HsBack); SizeAlign size{}; - check(sceAgcUnknownGetFusedShaderSize(&size, &front, &back) == 0 && size.m_size == backSh.size() * sizeof(ShaderRegister) && size.m_align == 4, "hull size query changed"); + check(sceAgcGetFusedShaderSize_0080(&size, &front, &back) == 0 && size.m_size == backSh.size() * sizeof(ShaderRegister) && size.m_align == 4, "hull size query changed"); std::vector scratch(backSh.size()); Shader fused{}; - check(sceAgcUnknownFuseShaderHalves(&fused, &front, &back, scratch.data()) == 0, "hull fusion failed"); + check(sceAgcFuseShaderHalves_0200(&fused, &front, &back, scratch.data()) == 0, "hull fusion failed"); check(fused.type == static_cast(ShaderRegs::ShaderBinaryType::Hs) && fused.code == back.code && fused.sh_registers == scratch.data() && fused.user_data == nullptr, "hull halves were not fused at the register level"); const auto address = reinterpret_cast(front.code); check(scratch[0].value == 0x11u && scratch[1].value == 0x22u, "hull checksums were not taken from the front half"); @@ -244,9 +246,12 @@ void testHullHalves() { int main() { try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; testFusion(0); testFusion(0xa0); testFusion(4); + testFusion(0, SSwappcNullS6); testRejections(); testHullHalves(); LibcRunShutdown_nid_postfix(); diff --git a/core/libs/prx/libSceAgc/tests/ShaderHelpers.cpp b/core/libs/prx/libSceAgc/tests/ShaderHelpers.cpp new file mode 100644 index 000000000..ab1541ded --- /dev/null +++ b/core/libs/prx/libSceAgc/tests/ShaderHelpers.cpp @@ -0,0 +1,129 @@ +#include "SceShaders.hpp" +#include "prx/libSceAgc/Shader/include/PrimState.hpp" +#include "prx/libSceAgc/Shader/include/InterpolantMapping.hpp" +#include "prx/libSceAgc/Shader/include/ShaderConstants.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include + +extern "C" int APS5_VABI sceAgcLinkShaders(ShaderRegister*, ShaderRegister*, const void*, const Shader*, const Shader*, std::uint32_t); +extern "C" int APS5_VABI sceAgcCreatePrimState(ShaderRegister*, ShaderRegister*, const Shader*, const Shader*, std::uint32_t); +extern "C" int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister*, const Shader*, const Shader*); + +namespace { +std::array preparedContext; +std::array preparedPrimitive; +std::size_t stageCount = 0; +unsigned preparations = 0; +unsigned mappings = 0; +unsigned links = 0; +bool rejectLink = false; +const Shader* mappedPixel = nullptr; + +void Require(bool condition) { + if (!condition) throw std::runtime_error("shader helper preparation mismatch"); +} +} + +extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span stages, std::span context, std::span primitive) { + Require(context.size() == 2 && primitive.size() == 3); + std::copy(context.begin(), context.end(), preparedContext.begin()); + std::copy(primitive.begin(), primitive.end(), preparedPrimitive.begin()); + stageCount = stages.size(); + ++preparations; +} + +extern "C" void AgcDriverResolveShaderAbi_nid_postfix(const Shader* shader, std::span context, std::span primitive) { + Require(context.size() == 32 && primitive.empty()); + mappedPixel = shader; + ++mappings; +} + +extern "C" void AgcDriverResolveGraphicsAbi_nid_postfix(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType) { + if (rejectLink) throw std::runtime_error("injected link failure"); + Require(vertex != nullptr && pixel == mappedPixel && (primitiveType == 0 || primitiveType == 7)); + ++links; +} + +int main() { + try { + using namespace ShaderRegs; + ShaderSpecialRegs special{}; + special.vgt_shader_stages_en = {VGT_SHADER_STAGES_EN, 0}; + special.vgt_gs_out_prim_type = {VGT_GS_OUT_PRIM_TYPE, 2}; + special.ge_cntl = {GE_CNTL, 0x123}; + special.ge_user_vgpr_en = {GE_USER_VGPR_EN, 7}; + Shader vertex{}; + vertex.type = static_cast(ShaderBinaryType::Gs); + vertex.specials = &special; + Shader hull{}; + hull.type = static_cast(ShaderBinaryType::Hs); + hull.specials = &special; + Shader pixel{}; + pixel.type = static_cast(ShaderBinaryType::Ps); + ShaderSemantic input{}; + input.semantic = 9; + pixel.input_semantics = &input; + std::array interpolants{}; + for (const auto count : {0u, 1u}) { + pixel.num_input_semantics = count; + for (const bool primitiveFirst : {false, true}) { + for (unsigned outputs = 1; outputs <= 3; ++outputs) { + for (const auto* hs : {static_cast(nullptr), static_cast(&hull)}) { + preparations = mappings = links = 0; + std::array context{}; + std::array primitive{}; + context.back().value = primitive.back().value = 0xdeadbeef; + const auto prepare = [&] { Require(sceAgcCreatePrimState(outputs & 1 ? context.data() : nullptr, outputs & 2 ? primitive.data() : nullptr, hs, &vertex, 7) == 0); }; + if (primitiveFirst) prepare(); + Require(sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, &pixel) == 0); + if (!primitiveFirst) prepare(); + Require(preparations == 1 && mappings == 1 && links == 1 && stageCount == (hs ? 2u : 1u)); + Require(preparedContext[0].offset == VGT_SHADER_STAGES_EN && preparedContext[1].offset == VGT_GS_OUT_PRIM_TYPE); + Require(preparedPrimitive[0].value == 0x123 && preparedPrimitive[1].value == 7 && preparedPrimitive[2].value == 7); + Require(context.back().value == 0xdeadbeef && primitive.back().value == 0xdeadbeef); + for (unsigned i = 0; i < 2; ++i) Require(context[i].value == (outputs & 1 ? preparedContext[i].value : 0)); + for (unsigned i = 0; i < 3; ++i) Require(primitive[i].value == (outputs & 2 ? preparedPrimitive[i].value : 0)); + } + } + } + } + std::array linkedContext{}; + std::array linkedPrimitive{}; + for (auto& value : linkedContext) value.value = 0xdeadbeef; + for (auto& value : linkedPrimitive) value.value = 0xdeadbeef; + for (auto& value : interpolants) value.value = 0xdeadbeef; + rejectLink = true; + bool rejected = false; + try { sceAgcLinkShaders(linkedContext.data(), linkedPrimitive.data(), nullptr, &vertex, &pixel, 7); } + catch (const std::runtime_error&) { rejected = true; } + Require(rejected); + for (const auto value : linkedContext) Require(value.value == 0xdeadbeef); + for (const auto value : linkedPrimitive) Require(value.value == 0xdeadbeef); + rejected = false; + try { sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, &pixel); } + catch (const std::runtime_error&) { rejected = true; } + Require(rejected); + for (const auto value : interpolants) Require(value.value == 0xdeadbeef); + rejectLink = false; + pixel.num_input_semantics = 33; + rejected = false; + try { sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, &pixel); } + catch (const std::runtime_error&) { rejected = true; } + Require(rejected); + for (const auto value : interpolants) Require(value.value == 0xdeadbeef); + pixel.num_input_semantics = 1; + Require(sceAgcLinkShaders(linkedContext.data(), linkedPrimitive.data(), nullptr, &vertex, &pixel, 7) == 0); + preparations = mappings = links = 0; + Require(sceAgcCreatePrimState(nullptr, nullptr, nullptr, nullptr, 7) == 0); + Require(sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, nullptr) == 0); + Require(preparations == 0 && mappings == 0 && links == 0); + std::cout << "shader helper preparation tests passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 3490c5852..f55651972 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -126,6 +126,8 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/GuestMemory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/ShaderMemory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/BdaFeatures.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/SubgroupClock.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/ShaderDeviceProfile.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Pm4.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/VulkanDevice.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/AspectFit.cpp @@ -180,12 +182,12 @@ target_include_directories(libSceAgcDriver PRIVATE ${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include ) -file(GLOB_RECURSE agcShaderCacheGuardSources CONFIGURE_DEPENDS +file(GLOB_RECURSE agcShaderCacheGuardSources CONFIGURE_DEPENDS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/*.hpp" "${CMAKE_CURRENT_SOURCE_DIR}/*.h" ) -list(FILTER agcShaderCacheGuardSources EXCLUDE REGEX "^${CMAKE_CURRENT_SOURCE_DIR}/(tests|tools)/") +list(FILTER agcShaderCacheGuardSources EXCLUDE REGEX "^(tests|tools)/") add_shader_recompiler(shader_recompiler_agc_driver libc GUARD_SOURCES ${agcShaderCacheGuardSources} @@ -202,6 +204,16 @@ set_target_properties(libSceAgcDriver PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(libSceAgcDriver) if(BUILD_TESTING) + add_test_executable(agc_write_watch_coverage_tests tests/WriteWatchCoverage.cpp) + target_include_directories(agc_write_watch_coverage_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(agc_write_watch_coverage_tests PRIVATE libc) + configure_windows_unwind(agc_write_watch_coverage_tests) + add_test(NAME agc_write_watch_coverage COMMAND agc_write_watch_coverage_tests) + if(WIN32) + set_tests_properties(agc_write_watch_coverage PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(agc_profile_output_tests tests/ProfileOutput.cpp) target_include_directories(agc_profile_output_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_profile_output_tests PRIVATE libc) @@ -229,6 +241,8 @@ add_test_executable(agc_driver_tests EXCLUDE_FROM_ALL target_include_directories(agc_driver_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler + ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include + ${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include ) target_link_libraries(agc_driver_tests PRIVATE @@ -273,11 +287,35 @@ target_link_libraries(agc_structurizer_tests PRIVATE shader_recompiler_agc_drive set_target_properties(agc_structurizer_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_structurizer_tests) +add_test_executable(agc_subgroup_clock_tests EXCLUDE_FROM_ALL + tests/SubgroupClock.cpp + Execution/src/SubgroupClock.cpp +) +target_include_directories(agc_subgroup_clock_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) +set_target_properties(agc_subgroup_clock_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_subgroup_clock_tests) +if(BUILD_TESTING) + set_target_properties(agc_subgroup_clock_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_subgroup_clock COMMAND agc_subgroup_clock_tests) +endif() + add_test_executable(agc_spirv_module_tests EXCLUDE_FROM_ALL tests/SpirvModule.cpp) target_link_libraries(agc_spirv_module_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_spirv_module_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_spirv_module_tests) +add_test_executable(agc_spirv_narrow_constant_store_tests EXCLUDE_FROM_ALL tests/SpirvNarrowConstantStore.cpp) +target_link_libraries(agc_spirv_narrow_constant_store_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_spirv_narrow_constant_store_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_spirv_narrow_constant_store_tests) +if(ANYPS5_ENABLE_SPIRV_TOOLS) + target_link_libraries(agc_spirv_narrow_constant_store_tests PRIVATE SPIRV-Tools-static) + add_test_executable(agc_spirv_optimizer_tests EXCLUDE_FROM_ALL tests/SpirvOptimizer.cpp) + target_link_libraries(agc_spirv_optimizer_tests PRIVATE shader_recompiler_agc_driver libc SPIRV-Tools-static) + set_target_properties(agc_spirv_optimizer_tests PROPERTIES CXX_EXTENSIONS OFF) + configure_windows_unwind(agc_spirv_optimizer_tests) +endif() + add_test_executable(agc_dpp_write_tests EXCLUDE_FROM_ALL tests/DppWrite.cpp) target_link_libraries(agc_dpp_write_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_dpp_write_tests PROPERTIES CXX_EXTENSIONS OFF) @@ -288,6 +326,20 @@ target_link_libraries(agc_masked_select_tests PRIVATE shader_recompiler_agc_driv set_target_properties(agc_masked_select_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_masked_select_tests) +add_test_executable(agc_wave_control_decode_tests EXCLUDE_FROM_ALL tests/WaveControlDecode.cpp) +target_include_directories(agc_wave_control_decode_tests PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(agc_wave_control_decode_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_wave_control_decode_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_wave_control_decode_tests) +if(BUILD_TESTING) + set_target_properties(agc_wave_control_decode_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_wave_control_decode COMMAND agc_wave_control_decode_tests) + if(WIN32) + set_tests_properties(agc_wave_control_decode PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_vop1_control_decode_tests EXCLUDE_FROM_ALL tests/Vop1ControlDecode.cpp) target_link_libraries(agc_vop1_control_decode_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_vop1_control_decode_tests PROPERTIES CXX_EXTENSIONS OFF) @@ -307,6 +359,20 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_buffer_pool_tests EXCLUDE_FROM_ALL tests/BufferPool.cpp) +target_include_directories(agc_buffer_pool_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) +target_link_libraries(agc_buffer_pool_tests PRIVATE libSceAgcDriver libc) +set_target_properties(agc_buffer_pool_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_buffer_pool_tests) +if(BUILD_TESTING) + set_target_properties(agc_buffer_pool_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_buffer_pool COMMAND agc_buffer_pool_tests) + if(WIN32) + set_tests_properties(agc_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_display_buffer_dcc_tests EXCLUDE_FROM_ALL tests/DisplayBufferDcc.cpp) target_include_directories(agc_display_buffer_dcc_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) target_link_libraries(agc_display_buffer_dcc_tests PRIVATE libSceAgcDriver libc) @@ -332,6 +398,12 @@ target_link_libraries(agc_shader_disk_cache_tests PRIVATE shader_recompiler_agc_ set_target_properties(agc_shader_disk_cache_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_shader_disk_cache_tests) +add_test_executable(agc_shader_device_profile_tests EXCLUDE_FROM_ALL tests/ShaderDeviceProfile.cpp Execution/src/ShaderDeviceProfile.cpp) +target_include_directories(agc_shader_device_profile_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) +target_link_libraries(agc_shader_device_profile_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_shader_device_profile_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_shader_device_profile_tests) + add_test_executable(agc_shader_control_flow_tests EXCLUDE_FROM_ALL tests/ControlFlow.cpp ) @@ -370,10 +442,24 @@ add_custom_command( VERBATIM ) +set(agcSampleArrayShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/SampleArray.comp) +set(agcSampleArrayShaderSpv ${agcTestShaderGeneratedRoot}/SampleArray.spv) +set(agcSampleArrayShaderHeader ${agcTestShaderGeneratedRoot}/SampleArray_spv.h) +add_custom_command( + OUTPUT "${agcSampleArrayShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTestShaderGeneratedRoot}" + COMMAND "$" -V --target-env vulkan1.1 -o "${agcSampleArrayShaderSpv}" "${agcSampleArrayShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcSampleArrayShaderSpv} -DOUTPUT=${agcSampleArrayShaderHeader} + -DSYMBOL=SAMPLE_ARRAY_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcSampleArrayShaderSource}" glslang-standalone + VERBATIM +) + add_test_executable(agc_driver_recorder_tests EXCLUDE_FROM_ALL tests/Recorder.cpp tests/ResidentPresent.cpp ${agcSampleLodShaderHeader} + ${agcSampleArrayShaderHeader} ) target_include_directories(agc_driver_recorder_tests PRIVATE @@ -392,6 +478,20 @@ target_link_libraries(agc_driver_recorder_tests PRIVATE set_target_properties(agc_driver_recorder_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_driver_recorder_tests) +add_test_executable(agc_recorder_kept_bytes_tests EXCLUDE_FROM_ALL tests/RecorderKeptBytes.cpp) +target_include_directories(agc_recorder_kept_bytes_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) +target_link_libraries(agc_recorder_kept_bytes_tests PRIVATE libSceAgcDriver libc) +set_target_properties(agc_recorder_kept_bytes_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_recorder_kept_bytes_tests) +if(BUILD_TESTING) + set_target_properties(agc_recorder_kept_bytes_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_recorder_kept_bytes COMMAND agc_recorder_kept_bytes_tests) + if(WIN32) + set_tests_properties(agc_recorder_kept_bytes PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_driver_pm4_tests EXCLUDE_FROM_ALL tests/Pm4.cpp ) @@ -515,6 +615,7 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources) ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include + ${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include ) target_link_libraries(${agcExecutionTest} PRIVATE libSceAgcDriver SDL2-static libc) set_target_properties(${agcExecutionTest} PROPERTIES CXX_EXTENSIONS OFF) @@ -522,17 +623,103 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources) list(APPEND agcExecutionTests ${agcExecutionTest}) endforeach() +target_include_directories(agc_driver_prepared_exports_tests PRIVATE $) + +target_include_directories(agc_driver_prepared_publication_tests PRIVATE $) + +target_include_directories(agc_driver_prepared_graphics_tests PRIVATE $) +target_link_libraries(agc_driver_interpolation_modes_tests PRIVATE shader_recompiler_agc_driver) + +target_include_directories(agc_driver_prepared_shaders_tests PRIVATE + $ +) + if(BUILD_TESTING) - foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests ${agcExecutionTests}) + add_test(NAME agc_shader_bindless COMMAND agc_shader_memory_tests --bindless) + set_tests_properties(agc_shader_bindless PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_shader_bindless PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() + foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests ${agcExecutionTests}) + if(agcTest STREQUAL "agc_driver_constant_store_tests") + continue() + endif() string(REGEX REPLACE "_tests$" "" agcTestName "${agcTest}") set_target_properties(${agcTest} PROPERTIES EXCLUDE_FROM_ALL OFF) add_test(NAME ${agcTestName} COMMAND ${agcTest}) - set_tests_properties(${agcTestName} PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77) + set_tests_properties(${agcTestName} PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") if(WIN32) set_tests_properties(${agcTestName} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endforeach() + set_tests_properties(agc_driver_flat_store PROPERTIES RUN_SERIAL TRUE) + add_test(NAME agc_driver_sampled_cache_budget_memory COMMAND agc_driver_sampled_cache_budget_tests memory) + set_tests_properties(agc_driver_sampled_cache_budget_memory PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77) + if(WIN32) + set_tests_properties(agc_driver_sampled_cache_budget_memory PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test(NAME agc_driver_pm4_failure COMMAND agc_driver_pm4_tests failure) + set_tests_properties(agc_driver_pm4_failure PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77) + if(WIN32) + set_tests_properties(agc_driver_pm4_failure PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test(NAME agc_driver_flip_compute COMMAND agc_driver_flip_tests compute) + set_tests_properties(agc_driver_flip_compute PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77) + if(WIN32) + set_tests_properties(agc_driver_flip_compute PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + + foreach(agcTest IN ITEMS agc_spirv_narrow_constant_store_tests agc_spirv_optimizer_tests agc_driver_constant_store_tests) + if(NOT TARGET ${agcTest}) + continue() + endif() + set_target_properties(${agcTest} PROPERTIES EXCLUDE_FROM_ALL OFF) + string(REGEX REPLACE "_tests$" "" agcTestBaseName "${agcTest}") + foreach(agcSpirvMode IN ITEMS default full none) + if(agcSpirvMode STREQUAL "default") + set(agcTestName "${agcTestBaseName}") + set(agcSpirvEnvironment "--unset=APS5_SPIRV_OPT") + else() + set(agcTestName "${agcTestBaseName}_${agcSpirvMode}") + set(agcSpirvEnvironment "APS5_SPIRV_OPT=${agcSpirvMode}") + endif() + add_test(NAME ${agcTestName} COMMAND ${CMAKE_COMMAND} -E env + ${agcSpirvEnvironment} "$") + set_tests_properties(${agcTestName} PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" TIMEOUT 60 SKIP_RETURN_CODE 77) + if(agcTest STREQUAL "agc_driver_constant_store_tests") + set_tests_properties(${agcTestName} PROPERTIES RUN_SERIAL TRUE) + endif() + if(WIN32) + set_tests_properties(${agcTestName} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() + endforeach() + foreach(preparedMode IN ITEMS indirect uncached debug debug_failure) + set(preparedTest "agc_driver_prepared_shaders_${preparedMode}") + if(preparedMode STREQUAL "indirect") + add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --indirect) + elseif(preparedMode STREQUAL "debug_failure") + add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --fail-before-registration) + set_tests_properties(${preparedTest} PROPERTIES WILL_FAIL TRUE TIMEOUT 30) + else() + add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests) + endif() + set_tests_properties(${preparedTest} PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(preparedMode STREQUAL "uncached") + set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_NO_DISPATCH_CACHE=1") + elseif(preparedMode STREQUAL "debug" OR preparedMode STREQUAL "debug_failure") + set_property(TEST ${preparedTest} APPEND PROPERTY ENVIRONMENT "APS5_PROFILE_DRAW=1" "APS5_NO_UNLOCKED_DEVICE=1") + endif() + if(WIN32) + set_tests_properties(${preparedTest} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() + endforeach() endif() option(AGC_BUILD_VISUAL_TEST "Enable the standalone AGC SPIR-V visual test" OFF) @@ -570,6 +757,8 @@ if(AGC_BUILD_VISUAL_TEST) Execution/src/PresentationScaler.cpp Execution/src/GuestMemory.cpp Execution/src/BdaFeatures.cpp + Execution/src/SubgroupClock.cpp + Execution/src/ShaderDeviceProfile.cpp Graphics/src/Draw.cpp Graphics/src/DepthSurface.cpp Graphics/src/ColorTargetLayout.cpp @@ -771,6 +960,14 @@ if(BUILD_TESTING) set_tests_properties(scalar_saveexec PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(scalar_swappc_tests ${LIBS_INCLUDE_DIR}/tests/ScalarSwappc.cpp) + target_link_libraries(scalar_swappc_tests PRIVATE shader_recompiler_agc_driver libc) + configure_windows_unwind(scalar_swappc_tests) + add_test(NAME scalar_swappc COMMAND scalar_swappc_tests) + if(WIN32) + set_tests_properties(scalar_swappc PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(scalar_bitcount64_tests ${LIBS_INCLUDE_DIR}/tests/ScalarBitCount64.cpp) target_link_libraries(scalar_bitcount64_tests PRIVATE shader_recompiler_agc_driver libc) configure_windows_unwind(scalar_bitcount64_tests) @@ -787,6 +984,14 @@ if(BUILD_TESTING) set_tests_properties(scalar_scratch_load PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(scalar_call_tests ${LIBS_INCLUDE_DIR}/tests/ScalarCall.cpp) + target_link_libraries(scalar_call_tests PRIVATE shader_recompiler_agc_driver libc) + configure_windows_unwind(scalar_call_tests) + add_test(NAME scalar_call COMMAND scalar_call_tests) + if(WIN32) + set_tests_properties(scalar_call PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(image_sample_gather_tests ${LIBS_INCLUDE_DIR}/tests/ImageSampleGather.cpp) target_link_libraries(image_sample_gather_tests PRIVATE shader_recompiler_agc_driver libc) configure_windows_unwind(image_sample_gather_tests) @@ -855,6 +1060,23 @@ if(BUILD_TESTING) endif() if(BUILD_TESTING) + add_test_executable(image_by_tests ${LIBS_INCLUDE_DIR}/tests/ImageBy.cpp) + target_link_libraries(image_by_tests PRIVATE shader_recompiler_agc_driver libc) + configure_windows_unwind(image_by_tests) + add_test(NAME image_by COMMAND image_by_tests) + if(WIN32) + set_tests_properties(image_by PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(image_pck_n_tests ${LIBS_INCLUDE_DIR}/tests/ImagePckN.cpp) + target_link_libraries(image_pck_n_tests PRIVATE shader_recompiler_agc_driver libc) + configure_windows_unwind(image_pck_n_tests) + add_test(NAME image_pck_n COMMAND image_pck_n_tests) + if(WIN32) + set_tests_properties(image_pck_n PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(image_msaa_load_tests ${LIBS_INCLUDE_DIR}/tests/ImageMsaaLoad.cpp) target_link_libraries(image_msaa_load_tests PRIVATE shader_recompiler_agc_driver libc) configure_windows_unwind(image_msaa_load_tests) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp index 5c3ef004f..24a7be45b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp @@ -28,6 +28,8 @@ std::size_t DisplayBufferSize(const DisplayBuffer& buffer); std::vector DecodeDisplayBuffer(const DisplayBuffer& buffer, std::span source); std::vector ReadDisplayBuffer(const DisplayBuffer& buffer); std::array DisplayBufferClearPixel(const DisplayBuffer& buffer, Graphics::DccKeys keys); +std::size_t DisplayBufferKeyBytes(const DisplayBuffer& buffer); +Graphics::DccKeys DisplayBufferKeys(const DisplayBuffer& buffer); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp index 1f643db73..16ef18fbe 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver.hpp @@ -1,6 +1,7 @@ #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_HPP #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_HPP +#include #include "SceTypes.hpp" #include "SceShaders.hpp" @@ -16,5 +17,8 @@ extern "C" void AgcDriverUnlockVulkanLoader_nid_postfix(); extern "C" void AgcDriverShutdown_nid_postfix(); extern "C" void AgcDriverSuspendPoint_nid_postfix(); extern "C" void AgcDriverRegisterShader_nid_postfix(const Shader* shader); +extern "C" void AgcDriverResolveShaderAbi_nid_postfix(const Shader* shader, std::span context, std::span primitive); +extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span stages, std::span context, std::span primitive); +extern "C" void AgcDriverResolveGraphicsAbi_nid_postfix(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType); #endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp index 8bbb76d59..1591221fd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp @@ -82,6 +82,8 @@ struct DataMask { enum class EntryOutcome { Equal, EqualData, Differing, Inaccessible, QueuedLabel, FlushingImage, PublishMoved, PendingMoved, ForgetMoved }; +bool CacheableResult(const ShaderRecompiler::RecompileResult& compiled); + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp index d5ae73e55..16d7a286e 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp @@ -32,6 +32,15 @@ struct DrawDecode { std::vector roles; }; +void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const ShaderRegistry& registry, bool staticAbi, bool includeFragment); + +struct PreparedGraphicsStage { + std::shared_ptr snapshot; + PreparedShaders::Entry entry; +}; + +std::vector PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target); + struct DrawRecipeRecord { std::vector> stages; std::shared_ptr recipe; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index cf2941e62..2f12dae2e 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -53,9 +53,13 @@ public: void SuspendPoint(); void RegisterVideoOutput(std::uint32_t handle, const std::shared_ptr& output); void UnregisterVideoOutput(std::uint32_t handle, const std::shared_ptr& output); + void AttachWindow(const PresentationWindow& window); void Present(const PresentationWindow& window, const DisplayBuffer* buffer, bool opaque, void (*gpuReady)(void*), void* context); void ReleaseWindow(void* window); void RegisterShader(const Shader* shader); + void ResolveShaderAbi(const Shader* shader, std::span context, std::span primitive); + void ResolveGraphicsStagesAbi(std::span stages, std::span context, std::span primitive); + void ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType); private: friend class SampledReadScope; @@ -68,6 +72,7 @@ private: static bool& onWorkerThread(); static void copyCommands(Submission& submission, const std::uint32_t* guest, std::size_t words); static bool copySegment(Submission& submission, const std::uint32_t* guest, std::size_t words, std::size_t& budget); + static void readRegisterLists(Submission& submission); void waitForFlipRoom(const Submission& submission); void reserveOutputs(Submission& submission); void executeRewindTail(const Submission& stalled); @@ -93,7 +98,7 @@ private: void lookupDispatch(std::uint64_t address, const Submission& submission, std::uint64_t key, bool noDispatchCache, bool traceCache, bool profile, std::span memory, DispatchPhaseTiming& phaseTiming, std::array& phaseMs, std::shared_ptr& compiledResult, std::shared_ptr& keepVariant, std::vector& captured, std::vector& liveWords, bool& dataHit, bool& cached, bool& validated, std::shared_ptr& missedEntry, bool& missedDiffering); void insertDispatch(std::uint64_t address, std::uint64_t key, bool noDispatchCache, bool profile, const std::shared_ptr& registeredShader, std::uint64_t forgetAtCapture, std::span memory, const std::shared_ptr& shaderMemory, const std::vector& captured, const std::shared_ptr& capture, const std::shared_ptr& compiledResult, const std::shared_ptr& missedEntry, bool missedDiffering, std::shared_ptr& attachVariant, DispatchPhaseTiming& phaseTiming); void dispatch(QueueState& queue, std::span packet, const Submission& submission, std::uint64_t indirectArguments = 0); - void verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffset, std::uint64_t deviceSerial, ShaderRecompiler::RecompileRequest request, std::span memory, std::uint64_t address, const DispatchVariant& variant, std::span liveWords, const ShaderRecompiler::RecompileResult& patched); + void verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffset, ShaderRecompiler::RecompileRequest request, std::span memory, std::uint64_t address, const DispatchVariant& variant, std::span liveWords, const ShaderRecompiler::RecompileResult& patched); static bool drawEntries(); static bool verifyDrawEntries(); static bool registerKeyEnabled(); @@ -110,7 +115,7 @@ private: std::shared_ptr decodeDraw(const QueueState& queue, const Submission& submission); void resolveDrawDecode(const QueueState& queue, const Submission& submission, std::shared_ptr& decode, bool registerKey, std::uint64_t drawKey, bool profile); void lookupDraw(const Submission& submission, const std::shared_ptr& localDevice, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector& programs, const std::vector& roles, const std::vector>& vertexInfos, bool useDrawEntries, bool registerKey, bool profile, std::uint64_t& drawKey, std::shared_ptr& entry, std::vector>& matched, std::vector>& matchedRegions, bool& drawHit, bool& verifyHit, DrawPhaseTiming& phaseTiming, std::array& phaseMs); - ShaderRecompiler::RecompileResult compileDrawStage(std::size_t i, std::uint32_t pushOffset, const QueueState& queue, const Submission& submission, const std::vector& programs, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector>& vertexInfos, std::vector& memory, const std::vector& linked, const Pm4::DrawParameters& drawParameters, const std::shared_ptr& localDevice, ShaderMemory& shaderMemory, std::vector& stageCaptures, std::vector& recompiled, bool drawHit, const std::vector>& matched, const std::vector>& matchedRegions, bool profile, std::uint64_t dumpTarget, std::uint64_t dumpSlot1, std::uint64_t& captures, DrawPhaseTiming& phaseTiming, std::array& phaseMs, std::string& rejected); + ShaderRecompiler::RecompileResult materializeDrawStage(std::size_t i, std::uint32_t pushOffset, const QueueState& queue, const Submission& submission, const std::vector& programs, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector>& vertexInfos, std::vector& memory, const std::vector& linked, const Pm4::DrawParameters& drawParameters, const std::shared_ptr& localDevice, ShaderMemory& shaderMemory, std::vector& stageCaptures, std::vector& recompiled, bool drawHit, const std::vector>& matched, const std::vector>& matchedRegions, bool profile, std::uint64_t dumpTarget, std::uint64_t dumpSlot1, std::uint64_t& captures, DrawPhaseTiming& phaseTiming, std::array& phaseMs, std::string& rejected); void cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawParameters& drawParameters, const std::optional& indirectCpu, const std::vector& programs, const std::vector& stageCaptures, const std::vector>& vertexInfos, const std::vector>& decodeReads, bool verifyHit, const std::vector>& matched, std::vector>& fresh, std::uint64_t drawKey, bool registerKey, const std::shared_ptr& decode, DrawPhaseTiming& phaseTiming); static bool drawPrecheck(); std::optional precheckDraw(const QueueState& queue, const Submission& submission, std::span packet, const Pm4::DrawParameters& drawParameters, std::string& rejected, bool& traceIndirect); @@ -215,8 +220,8 @@ private: void dumpSampleCounters(std::uint64_t address); template static void timed(double WorkerProfile::*bucket, TWork&& work); - template - void tolerate(const char* kind, TWork&& work); + FrameTiming* frameTiming(); + FrameTiming* includeTimingSubmission(const Submission& submission, bool firstSegment); void execute(const Submission& submission); void markCompleted(std::uint64_t serial); static const std::atomic*& workerQueued(); @@ -229,6 +234,7 @@ private: std::map workers; std::uint64_t frameSerial = 0; + std::shared_ptr pendingFrameTiming; std::atomic flipsCounted{0}; std::atomic flipSerial{0}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp index cdc9f1683..1b15bcfe9 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp @@ -26,13 +26,20 @@ struct Submission { std::map> flips; std::map> renderingWaits; std::map conditionalEnds; + std::map> registerLists; bool suspend = false; bool waitFree = false; std::uint64_t received = 0; std::vector labelWrites; std::set heldAtSubmit; + std::chrono::steady_clock::time_point receivedAt{}; + std::chrono::steady_clock::time_point copiedAt{}; + std::chrono::steady_clock::time_point validatedAt{}; + std::chrono::steady_clock::time_point roomReadyAt{}; std::chrono::steady_clock::time_point enqueuedAt{}; + std::chrono::steady_clock::time_point dequeuedAt{}; + std::chrono::steady_clock::time_point orderedAt{}; const std::uint32_t* rewindTail = nullptr; std::size_t rewindWords = 0; }; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp index 1f6012ccb..846729868 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp @@ -2,7 +2,9 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_SHADERREGISTRY_HPP #include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/QueueState.hpp" #include +#include #include #include #include @@ -14,16 +16,45 @@ namespace AgcDriver::DriverDetail { -struct HandleMemos { +struct ShaderSnapshot; +using ShaderRegistry = std::map>; + +struct PreparedShaderState { struct Entry { - std::uint64_t key = 0; + std::size_t codeOffset; std::shared_ptr handle; - std::shared_ptr failure; }; + struct Rectangle { + std::uint64_t vertexId; + std::uint64_t fragmentId; + std::shared_ptr shaders; + }; + struct RectangleProgress { + std::weak_ptr fragment; + std::size_t vertexCount; + std::size_t fragmentCount; + }; + struct GraphicsAbi { + std::vector key; + std::weak_ptr registry; + std::vector> stages; + }; + std::vector entries; + std::vector> registeredAbis; + std::vector graphicsAbis; + std::vector rectangles; + std::vector rectangleProgress; + std::vector> fragments; + bool rectangleRequested = false; +}; +struct PreparedShaders : PreparedShaderState { std::mutex mutex; - std::array entries; - std::size_t next = 0; - std::atomic poisoned{0}; +}; + +struct RegisteredShaderState { + Registers shader; + Registers context; + Registers userConfig; }; struct ShaderSnapshot { @@ -32,17 +63,23 @@ struct ShaderSnapshot { std::uint8_t type; std::vector code; std::vector header; - std::unique_ptr handles = std::make_unique(); + std::shared_ptr prepared = std::make_shared(); + std::shared_ptr registeredState; }; -using ShaderRegistry = std::map>; -bool FailureMemo(); std::shared_ptr ReadRawComputeShader(std::uint64_t address); std::uint64_t NullPixelProgramAddress(); +std::optional RegisteredFloatMode(const ShaderSnapshot& snapshot); +void PublishRegisteredShader(std::shared_ptr& registry, const std::shared_ptr& snapshot); -std::shared_ptr SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, std::uint64_t deviceSerial, const ShaderRecompiler::RecompileRequest& request, bool bypass, const std::string** poisoned = nullptr); +void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target); + +ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapshot, std::uint64_t vertexId, std::uint64_t fragmentId); + +std::shared_ptr SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request); +ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index 966fb6c04..d45085da8 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -92,6 +92,7 @@ bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generati // for every collect: nothing is reused without an ordering point. APS5_PACKET_EPOCH=1 makes the // workers bump before every packet as before. CollectEpochBumps counts the bumps ([guestmem] line). void BumpCollectEpoch(); +std::uint64_t CollectEpoch(); std::uint64_t CollectEpochBumps(); std::uint64_t CollectWritesUncached(std::uint64_t address, std::size_t bytes); // The tracker's current generation (every collect and MarkWritten bumps it): a stamp taken after diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp index 6ce0b2d54..1f6bb75a0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp @@ -5,17 +5,15 @@ #include #include #include -#include +#include #include -#include #include #include #include -#include +#include #include #include #include -#include "prx/libc/include/general/LogMacros.hpp" namespace AgcDriver { @@ -30,23 +28,68 @@ public: std::uint64_t bytes = 0; }; - explicit FrameTiming(std::uint64_t id) : id(id) {} + explicit FrameTiming(std::uint64_t id, bool localMetrics = false) : localMetrics(localMetrics), id(id), workerStart(Clock::now()) {} + + static FrameTiming* Preparation() { + if (!APS5_ENABLE_TIMING_LOG) return nullptr; + static FrameTiming timing(0); + return &timing; + } + + static FrameTiming* Async() { + if (!APS5_ENABLE_TIMING_LOG) return nullptr; + static FrameTiming timing(0); + return &timing; + } + + void CollectBackground() { + const auto collect = [&](FrameTiming* source, auto& destination) { + if (source == nullptr) return; + std::scoped_lock lock(mutex, source->mutex); + for (auto& [key, metric] : source->metrics) { + if (metric.count == 0) continue; + auto& target = destination[key]; + target.total += metric.total; + target.maximum = std::max(target.maximum, metric.maximum); + target.count += metric.count; + target.bytes += metric.bytes; + metric = {}; + } + }; + collect(Preparation(), metrics); + collect(Async(), asyncMetrics); + } Metric* Get(const char* scope, const char* stage) { - std::lock_guard lock(mutex); + std::unique_lock lock(mutex, std::defer_lock); + if (!localMetrics) lock.lock(); return &metrics[{scope, stage}]; } void Add(Metric* metric, Clock::duration elapsed, std::uint64_t bytes = 0) { - std::lock_guard lock(mutex); + std::unique_lock lock(mutex, std::defer_lock); + if (!localMetrics) lock.lock(); metric->total += elapsed; metric->maximum = std::max(metric->maximum, elapsed); ++metric->count; metric->bytes += bytes; } + void NotePacket(std::uint64_t serial, std::size_t offset, std::uint32_t opcode, std::size_t words, Clock::duration elapsed) { + std::unique_lock lock(mutex, std::defer_lock); + if (!localMetrics) lock.lock(); + ++packetCount; + packetWords += words; + if (elapsed > slowPacketTime) { + slowPacketTime = elapsed; + slowPacketSerial = serial; + slowPacketOffset = offset; + slowPacketOpcode = opcode; + } + } + void IncludeSubmission(std::uint64_t serial, Clock::time_point received, Clock::time_point enqueued, Clock::time_point dequeued, bool firstSegment) { - if (serial == 0 || received > enqueued || enqueued > dequeued) throw std::runtime_error("Frame timing: invalid submission timestamps"); + if (serial == 0 || received == Clock::time_point{} || received > enqueued || enqueued > dequeued) throw std::runtime_error("Frame timing: invalid submission timestamps"); if (firstSerial == 0) { firstSerial = serial; start = received; @@ -66,6 +109,7 @@ public: flipOffset = offset; flipReceived = received; flipReached = reached; + localMetrics = false; } Clock::time_point FlipReached() { @@ -107,39 +151,52 @@ public: gpuGapMs += gapMs; } - void Print(std::uint32_t outputHandle, std::int32_t buffer, std::int64_t argument, Clock::time_point finished, Clock::duration interval) { + std::function Capture(std::uint32_t outputHandle, std::int32_t buffer, std::int64_t argument, Clock::time_point finished, Clock::duration interval) { std::lock_guard lock(mutex); if (firstSerial == 0 || flipSerial == 0) throw std::runtime_error("Frame timing: incomplete submission lineage"); - std::ostringstream output; - output.imbue(std::locale::classic()); - output << std::fixed << std::setprecision(3); - output << "[FrameTiming] frame=" << id << " submissions=" << firstSerial << ':' << lastSerial; - output << " output=" << outputHandle; - output << " flip=" << flipSerial << ':' << flipOffset << " buffer=" << buffer << " argument=" << argument; - output << " endpoint=flip_complete display_confirmed=0"; - output << " first_submit_to_flip_ms=" << milliseconds(finished - start); - output << " flip_submit_to_complete_ms=" << milliseconds(finished - flipReceived); - output << " first_submit_to_execute_ms=" << milliseconds(executionStart - start); - output << " execute_to_flip_packet_ms=" << milliseconds(flipReached - executionStart); - output << " flip_packet_to_complete_ms=" << milliseconds(finished - flipReached); - auto accounted = Clock::duration::zero(); - for (const auto scope : {"Driver.Packet", "Driver.Suspend", "Driver.Worker", "Driver.Completion"}) { - const auto it = metrics.find({scope, "total"}); - if (it != metrics.end()) accounted += it->second.total; - } - output << " worker_unattributed_ms=" << milliseconds(flipReached - executionStart - accounted); - if (interval != Clock::duration::zero()) output << " flip_interval_ms=" << milliseconds(interval); - output << " batches_at_flip=" << batchesAtFlip << " unsignaled_at_flip=" << unsignaledAtFlip << " batches_at_blit=" << batchesAtBlit << " batches_after_flip=" << batchesAfterFlip; - output << " gpu_busy_ms=" << gpuBusyMs << " gpu_gap_ms=" << gpuGapMs; - output << " last_submit_after_flip_ms=" << milliseconds(lastSubmitAfterFlip) << " blit_submit_after_flip_ms=" << milliseconds(blitSubmitAfterFlip); - output << " metrics=inclusive(count,sum_ms,max_ms[,bytes])"; - for (const auto& [key, metric] : metrics) { - if (metric.count == 0) continue; - output << ' ' << key.first << '.' << key.second << "=(" << metric.count << ',' << milliseconds(metric.total) << ',' << milliseconds(metric.maximum); - if (metric.bytes != 0) output << ',' << metric.bytes; - output << ')'; - } - APS5_LOG_TIMING("%s", output.str().c_str()); + return [asyncMetrics = asyncMetrics, metrics = metrics, id = id, firstSerial = firstSerial, lastSerial = lastSerial, flipSerial = flipSerial, flipOffset = flipOffset, workerStart = workerStart, start = start, executionStart = executionStart, flipReceived = flipReceived, flipReached = flipReached, batchesAtFlip = batchesAtFlip, unsignaledAtFlip = unsignaledAtFlip, batchesAtBlit = batchesAtBlit, batchesAfterFlip = batchesAfterFlip, gpuBusyMs = gpuBusyMs, gpuGapMs = gpuGapMs, lastSubmitAfterFlip = lastSubmitAfterFlip, blitSubmitAfterFlip = blitSubmitAfterFlip, packetCount = packetCount, packetWords = packetWords, slowPacketTime = slowPacketTime, slowPacketSerial = slowPacketSerial, slowPacketOffset = slowPacketOffset, slowPacketOpcode = slowPacketOpcode, outputHandle, buffer, argument, finished, interval](std::ostream& output) { + output << std::fixed << std::setprecision(3); + output << "[FrameTiming] frame=" << id << " submissions=" << firstSerial << ':' << lastSerial; + output << " output=" << outputHandle; + output << " flip=" << flipSerial << ':' << flipOffset << " buffer=" << buffer << " argument=" << argument; + output << " endpoint=flip_complete display_confirmed=0 worker_queue=0 background_metrics=completed_since_previous_flip"; + output << " first_submit_at_ms=" << milliseconds(start.time_since_epoch()); + output << " first_execute_at_ms=" << milliseconds(executionStart.time_since_epoch()); + output << " flip_packet_at_ms=" << milliseconds(flipReached.time_since_epoch()); + output << " flip_complete_at_ms=" << milliseconds(finished.time_since_epoch()); + output << " first_submit_to_flip_ms=" << milliseconds(finished - start); + output << " flip_submit_to_complete_ms=" << milliseconds(finished - flipReceived); + output << " first_submit_to_execute_ms=" << milliseconds(executionStart - start); + output << " execute_to_flip_packet_ms=" << milliseconds(flipReached - executionStart); + output << " flip_packet_to_complete_ms=" << milliseconds(finished - flipReached); + auto accounted = Clock::duration::zero(); + for (const auto scope : {"Driver.Packet", "Driver.Suspend", "Driver.WorkerWait", "Driver.Completion", "Driver.EndSubmission", "Driver.Rewind"}) { + const auto it = metrics.find({scope, "total"}); + if (it != metrics.end()) accounted += it->second.total; + } + output << " worker_begin_at_ms=" << milliseconds(workerStart.time_since_epoch()); + output << " worker_span_ms=" << milliseconds(flipReached - workerStart); + output << " worker_accounted_ms=" << milliseconds(accounted); + output << " worker_unattributed_ms=" << milliseconds(flipReached - workerStart - accounted); + if (interval != Clock::duration::zero()) output << " flip_interval_ms=" << milliseconds(interval); + output << " batches_at_flip=" << batchesAtFlip << " unsignaled_at_flip=" << unsignaledAtFlip << " batches_at_blit=" << batchesAtBlit << " batches_after_flip=" << batchesAfterFlip; + output << " gpu_busy_ms=" << gpuBusyMs << " gpu_gap_ms=" << gpuGapMs; + output << " last_submit_after_flip_ms=" << milliseconds(lastSubmitAfterFlip) << " blit_submit_after_flip_ms=" << milliseconds(blitSubmitAfterFlip); + output << " packet_count=" << packetCount << " packet_dwords=" << packetWords; + output << " slow_packet_submission=" << slowPacketSerial << " slow_packet_offset=" << slowPacketOffset << " slow_packet_opcode=" << slowPacketOpcode << " slow_packet_ms=" << milliseconds(slowPacketTime); + output << " metrics=inclusive(count,sum_ms,max_ms[,bytes])"; + const auto writeMetrics = [&](const auto& entries, const char* prefix) { + for (const auto& [key, metric] : entries) { + if (metric.count == 0) continue; + output << ' ' << prefix << key.first << '.' << key.second << "=(" << metric.count << ',' << milliseconds(metric.total) << ',' << milliseconds(metric.maximum); + if (metric.bytes != 0) output << ',' << metric.bytes; + output << ')'; + } + }; + writeMetrics(metrics, ""); + writeMetrics(asyncMetrics, "Async."); + output << '\n'; + }; } private: @@ -148,12 +205,21 @@ private: } std::mutex mutex; + bool localMetrics = false; std::map, Metric> metrics; + std::map, Metric> asyncMetrics; + std::uint64_t packetCount = 0; + std::uint64_t packetWords = 0; + Clock::duration slowPacketTime{}; + std::uint64_t slowPacketSerial = 0; + std::size_t slowPacketOffset = 0; + std::uint32_t slowPacketOpcode = 0; std::uint64_t id; std::uint64_t firstSerial = 0; std::uint64_t lastSerial = 0; std::uint64_t flipSerial = 0; std::size_t flipOffset = 0; + Clock::time_point workerStart{}; Clock::time_point start{}; Clock::time_point executionStart{}; Clock::time_point flipReceived{}; @@ -196,7 +262,7 @@ public: if (frame != nullptr) { total = frame->Get(scope, "total"); } - start = Clock::now(); + if (frame != nullptr) start = Clock::now(); previous = start; } @@ -204,10 +270,20 @@ public: PerformanceTimer& operator=(const PerformanceTimer&) = delete; ~PerformanceTimer() { + Finish(); + } + + void Finish() { if (frame == nullptr) return; const auto end = Clock::now(); if (tail != nullptr) frame->Add(tail, end - previous); frame->Add(total, end - start); + frame = nullptr; + } + + void FinishPacket(std::uint64_t serial, std::size_t offset, std::uint32_t opcode, std::size_t words) { + if (frame != nullptr) frame->NotePacket(serial, offset, opcode, words, Clock::now() - start); + Finish(); } void Mark(const char* stage, std::uint64_t bytes = 0) { @@ -225,8 +301,8 @@ private: const char* scope; FrameTiming::Metric* total = nullptr; FrameTiming::Metric* tail = nullptr; - Clock::time_point start; - Clock::time_point previous; + Clock::time_point start{}; + Clock::time_point previous{}; }; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp index 7bc44cece..e6d2cbe13 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp @@ -86,6 +86,9 @@ inline std::size_t PacketWords(std::uint32_t header) { return FillerPacket(heade std::string_view UnsupportedReason(std::uint32_t header); void Validate(std::span packet, std::uint32_t queue); void Execute(std::span packet, QueueState& queue); +inline bool IndirectRegisterOpcode(std::uint32_t opcode) { return opcode == 0x63 || opcode == 0x64 || opcode == 0x9f; } +std::vector ReadIndirectRegisters(std::span packet); +void ExecuteIndirectRegisters(std::span packet, std::span pairs, QueueState& queue); bool AccessesMemory(std::uint32_t header); // Whether an ACQUIRE_MEM packet asks only for GPU cache actions (no CPU-visible memory // synchronization): such a packet needs a pipeline barrier, not a device drain. diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Presentation.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Presentation.hpp index 88dd83ead..3f771fbb1 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Presentation.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Presentation.hpp @@ -25,6 +25,7 @@ struct PresentationWindow { } +extern "C" void AgcDriverAttachWindow_nid_postfix(const AgcDriver::PresentationWindow& window); extern "C" void AgcDriverPresentClear_nid_postfix(const AgcDriver::PresentationWindow& window, bool opaque, void (*gpuReady)(void*), void* context); extern "C" void AgcDriverPresentBuffer_nid_postfix(const AgcDriver::PresentationWindow& window, const AgcDriver::DisplayBuffer& buffer, void (*gpuReady)(void*), void* context); extern "C" void AgcDriverReleaseWindow_nid_postfix(void* window); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp index 9c714409e..6daec3ba8 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -24,12 +25,17 @@ public: } ~ProfileOutput() { + Stop(); + } + + void Stop() { + std::lock_guard stopLock(stopMutex); { std::lock_guard lock(mutex); stopping = true; } changed.notify_one(); - worker.join(); + if (worker.joinable()) worker.join(); } bool Write(std::string text) { @@ -117,6 +123,7 @@ private: std::function writer; const std::size_t capacity; + std::mutex stopMutex; std::mutex mutex; std::condition_variable changed; std::condition_variable idle; @@ -129,14 +136,21 @@ private: std::thread worker; }; +inline std::atomic createdProfileOutput{nullptr}; + inline ProfileOutput& ProfileOutput_nid_no_patch() { static ProfileOutput output([](std::string_view text) { if (std::fwrite(text.data(), 1, text.size(), stderr) != text.size() || std::fflush(stderr) != 0) throw std::runtime_error("profile output write failed"); }); + createdProfileOutput.store(&output, std::memory_order_release); return output; } +inline void StopProfileOutput_nid_no_patch() { + if (auto* output = createdProfileOutput.load(std::memory_order_acquire)) output->Stop(); +} + template void ProfilePrint_nid_no_patch(const char* format, TArgs... args) { ProfileOutput_nid_no_patch().Print(format, args...); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp index 19493fc5e..ff24be825 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp @@ -148,15 +148,18 @@ inline Registers InitialContextRegisters() { {0xc, 0}, {0xd, 0x40004000}, {0x81, 0x80000000}, {0x82, 0x40004000}, {0x90, 0x80000000}, {0x91, 0x40004000}, {0x105, 0}, {0x106, 0}, {0x107, 0}, {0x108, 0}, - {0x1b1, 0}, {0x1b6, 0}, {0x1c3, 0}, {0x1c4, 0}, {0x1c5, 0}, + {0x1b1, 0}, {0x1b3, 0}, {0x1b4, 0}, {0x1b6, 0}, {0x1c3, 0}, {0x1c4, 0}, {0x1c5, 0}, {0x1ff, 0}, {0x292, 2}, {0x293, 0}, {0x29b, 0}, {0x2ce, 0}, {0x2d3, 0}, {0x2d5, 0}, {0x2d6, 0}, {0x2db, 0}, - {0x2dc, 0xaa00}, {0x2e4, 0}, {0x2f8, 0}, {0x2f9, 0x2d}, + {0x2dc, 0xaa00}, {0x2df, 0}, {0x2e4, 0}, {0x2f8, 0}, {0x2f9, 0x2d}, {0x30e, 0xffffffff}, {0x30f, 0xffffffff}, {0x313, 0x6000}, {0x318, 0}, {0x31b, 0}, {0x31c, 0}, {0x31d, 0}, {0x390, 0}, {0x3b0, 0}, {0x3b8, 0} }; - for (std::uint32_t i = 0; i < 8; ++i) result.emplace(0x1e0 + i, 0x20010001); + for (std::uint32_t i = 0; i < 8; ++i) { + result.emplace(0x1e0 + i, 0x20010001); + result.emplace(0x31c + 0xfu * i, 0); + } for (std::uint32_t i = 0; i < 16; ++i) { result.emplace(0x94 + 2 * i, 0x80000000); result.emplace(0x95 + 2 * i, 0x40004000); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp new file mode 100644 index 000000000..a390c288e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp @@ -0,0 +1,31 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SHADERDEVICEPROFILE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SHADERDEVICEPROFILE_HPP + +#include "Recompiler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include +#include + +namespace AgcDriver { + +class ShaderDeviceProfile { +public: + ShaderDeviceProfile(const ShaderRecompiler::SpirvTarget& target, const VkDeviceCreateInfo& deviceInfo, const VkPhysicalDeviceLimits& limits); + ShaderDeviceProfile(const ShaderDeviceProfile&) = delete; + ShaderDeviceProfile& operator=(const ShaderDeviceProfile&) = delete; + ShaderRecompiler::SpirvTarget Target() const { return target; } + const VkPhysicalDeviceLimits& Limits() const { return limits; } + bool NullDescriptors() const { return nullDescriptors; } + +private: + ShaderRecompiler::SpirvTarget target; + const VkPhysicalDeviceLimits limits; + std::vector capabilities; + std::vector extensions; + std::vector extensionViews; + bool nullDescriptors = false; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp index 2c41a0684..75f597a2d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp @@ -14,6 +14,7 @@ namespace ShaderRecompiler { struct SourceHandle; +class PreparedShaderInvocation; } namespace AgcDriver { @@ -61,6 +62,7 @@ public: // (the driver's memoized ShaderRecompiler::ResolveSource result) the capture skips the source // resolution. std::shared_ptr Capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle = nullptr); + std::shared_ptr Capture(const ShaderRecompiler::PreparedShaderInvocation& invocation); [[nodiscard]] std::vector Regions() const; // The page regions read since the previous call (or construction), a word read again // included, the initial regions excluded: one stage's own reads on the draw path's shared @@ -70,6 +72,7 @@ public: static void CountHandleMemo(bool hit); private: + std::shared_ptr capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle, const ShaderRecompiler::PreparedShaderInvocation* invocation); static constexpr std::size_t PageBytes = 4096; static constexpr std::size_t PageWords = PageBytes / sizeof(std::uint32_t); @@ -82,6 +85,7 @@ private: }; static bool read(void* context, std::uint64_t address, std::uint32_t* value); + static bool accessible(void* context, std::uint64_t address, std::uint64_t bytes); Page& page(std::uint64_t base); // Regions given at construction (the registered shader's code and header), referenced as given: diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp new file mode 100644 index 000000000..57ce88164 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp @@ -0,0 +1,30 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SHADERPREPARATION_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SHADERPREPARATION_HPP + +#include + +namespace AgcDriver::DriverDetail { + +struct ShaderSnapshot; +struct PreparedShaderState; + +class ShaderPreparationTransaction { +public: + ShaderPreparationTransaction(); + ~ShaderPreparationTransaction(); + ShaderPreparationTransaction(const ShaderPreparationTransaction&) = delete; + ShaderPreparationTransaction& operator=(const ShaderPreparationTransaction&) = delete; + PreparedShaderState& Edit(const ShaderSnapshot& snapshot); + const PreparedShaderState& Read(const ShaderSnapshot& snapshot) const; + void Commit(); + +private: + struct State; + std::unique_ptr state; + State* root = nullptr; + bool committed = false; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp new file mode 100644 index 000000000..0c3148030 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp @@ -0,0 +1,28 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SHADERPREPARATIONSCOPE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SHADERPREPARATIONSCOPE_HPP + +namespace AgcDriver::DriverDetail { +class ShaderPreparationTransaction; +} + +extern "C" AgcDriver::DriverDetail::ShaderPreparationTransaction* AgcDriverBeginShaderPreparation_nid_postfix(); +extern "C" void AgcDriverCommitShaderPreparation_nid_postfix(AgcDriver::DriverDetail::ShaderPreparationTransaction* transaction); +extern "C" void AgcDriverEndShaderPreparation_nid_postfix(AgcDriver::DriverDetail::ShaderPreparationTransaction* transaction) noexcept; + +namespace AgcDriver { + +class ShaderPreparationScope { +public: + ShaderPreparationScope() : transaction(AgcDriverBeginShaderPreparation_nid_postfix()) {} + ~ShaderPreparationScope() { AgcDriverEndShaderPreparation_nid_postfix(transaction); } + ShaderPreparationScope(const ShaderPreparationScope&) = delete; + ShaderPreparationScope& operator=(const ShaderPreparationScope&) = delete; + void Commit() { AgcDriverCommitShaderPreparation_nid_postfix(transaction); } + +private: + DriverDetail::ShaderPreparationTransaction* transaction; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp new file mode 100644 index 000000000..6c1c74209 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp @@ -0,0 +1,13 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SUBGROUPCLOCK_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SUBGROUPCLOCK_HPP + +#include +#include + +namespace AgcDriver { + +bool NarrowSubgroupClock(VkDriverId driver, std::string_view deviceName); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 8828e54a3..696cfbdc1 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -161,6 +161,7 @@ public: bool Presentable() const; bool PrimitiveListRestart() const; bool SamplerFilterMinmax() const; + bool ConservativeRasterization() const; // A presentation is a few steps so the presenter holds GuestMemory::GpuMutex only while it // touches the queue. Presentations are slots (FlipInFlight() + 1, each with its own command // buffer, fence, kept resident image and dump buffer): RetirePresents(keep) (no mutex) retires @@ -299,6 +300,7 @@ private: // The device's Graphics::Context: a copy of the one built at setup (its instance functions // resolved then, its function table filled then), or with APS5_NO_CONTEXT_CACHE=1 built anew. Graphics::Context graphicsContext() const; + ShaderRecompiler::SpirvTarget buildTarget() const; Graphics::Context buildContext() const; // Body of Dispatch and DispatchIndirect: `arguments` 0 dispatches x, y, z groups. IndirectOutcome dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipe); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/WriteWatchCoverage.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/WriteWatchCoverage.hpp new file mode 100644 index 000000000..a35d3fb19 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/WriteWatchCoverage.hpp @@ -0,0 +1,85 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_WRITEWATCHCOVERAGE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_WRITEWATCHCOVERAGE_HPP + +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::GuestMemory { + +class WriteWatchCoverage { +public: + void Initialize(std::uint64_t address, std::size_t bytes) { + base = address; + size = bytes; + excluded.assign(bytes / BlockBytes + (bytes % BlockBytes != 0), 0); + excludedGenerations.clear(); + } + + bool Covers(std::uint64_t address, std::size_t bytes) const { + if (bytes == 0 || bytes > size || address < base || address - base > size - bytes) return false; + const auto first = (address - base) / BlockBytes; + const auto last = (address - base + bytes - 1) / BlockBytes; + return std::none_of(excluded.begin() + first, excluded.begin() + last + 1, [](auto generation) { return generation != 0; }); + } + + bool Exclude(std::uint64_t address, std::size_t bytes, std::uint64_t generation = 1) { + if (bytes == 0 || size == 0) return false; + if (address < base) { + if (base - address >= bytes) return false; + bytes -= base - address; + address = base; + } + const auto offset = address - base; + if (offset >= size) return false; + bytes = static_cast(std::min(bytes, size - offset)); + const auto first = offset / BlockBytes; + const auto last = (offset + bytes - 1) / BlockBytes; + bool changed = false; + for (auto block = first; block <= last; ++block) { + changed = changed || excluded[block] == 0; + auto& stamp = excludedGenerations[block]; + stamp = std::max(stamp, std::max(generation, 1)); + excluded[block] = 1; + } + return changed; + } + + bool Restore(std::uint64_t address, std::size_t bytes, std::uint64_t generation = std::numeric_limits::max()) { + if (bytes == 0 || size == 0) return false; + if (address < base) { + if (base - address >= bytes) return false; + bytes -= base - address; + address = base; + } + const auto offset = address - base; + if (offset >= size) return false; + bytes = static_cast(std::min(bytes, size - offset)); + const auto first = offset / BlockBytes + (offset % BlockBytes != 0); + const auto stop = offset + bytes; + const auto end = stop / BlockBytes + (stop == size && stop % BlockBytes != 0); + bool changed = false; + for (auto block = first; block < end; ++block) { + if (excluded[block] != 0 && excludedGenerations.at(block) < generation) { + changed = true; + excluded[block] = 0; + excludedGenerations.erase(block); + } + } + return changed; + } + +private: + static constexpr std::size_t BlockBytes = 65536; + std::uint64_t base = 0; + std::size_t size = 0; + std::vector excluded; + std::unordered_map excludedGenerations; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp index 459b6e228..8bc53af7d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/DisplayBuffer.cpp @@ -3,6 +3,7 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include #include #include #include @@ -10,6 +11,7 @@ #include #include #include +#include namespace AgcDriver { namespace { @@ -50,6 +52,40 @@ void decodeTexel(const std::byte* texel, std::byte* pixel, bool rgba, bool tenBi pixel[3] = texel[3]; } +constexpr std::uint32_t MetaBlockEdge = 512; +constexpr std::size_t MetaBlockKeys = 4096; +constexpr std::uint32_t KeyEdge = 8; + +std::array displayableKeyPixel(const DisplayBuffer& buffer, std::size_t key) { + const auto pitch = (buffer.width + MetaBlockEdge - 1u) / MetaBlockEdge; + const auto block = static_cast(key / MetaBlockKeys); + const auto offset = static_cast(key % MetaBlockKeys); + std::uint32_t x = 0; + std::uint32_t y = 0; + for (std::uint32_t bit = 0; bit < 6; ++bit) { + x |= ((offset >> (2u * bit)) & 1u) << bit; + y |= ((offset >> (2u * bit + 1u)) & 1u) << bit; + } + return {block % pitch * MetaBlockEdge + x * KeyEdge, block / pitch * MetaBlockEdge + y * KeyEdge}; +} + +void traceDisplayKeys(const DisplayBuffer& buffer, const std::uint8_t* keys, std::size_t count) { + static const bool trace = std::getenv("APS5_TRACE_DCC_KEYS") != nullptr; + if (!trace) return; + constexpr std::size_t MaxRuns = 256; + std::string runs; + std::size_t begin = 0; + for (std::size_t run = 0; begin < count && run < MaxRuns; ++run) { + const auto end = static_cast(std::find_if(keys + begin, keys + count, [&](std::uint8_t key) { return key != keys[begin]; }) - keys); + char text[32]; + std::snprintf(text, sizeof(text), "%s%02x x%zu", run == 0 ? "" : ", ", keys[begin], end - begin); + runs += text; + begin = end; + } + if (begin < count) runs += ", ..."; + std::fprintf(stderr, "[dcc-keys] display keys 0x%llx+0x%zx: %s\n", static_cast(buffer.dccAddress), count, runs.c_str()); +} + } std::size_t DisplayBufferSize(const DisplayBuffer& buffer) { @@ -117,6 +153,35 @@ std::array DisplayBufferClearPixel(const DisplayBuffer& buffer, Gr return pixel; } +std::size_t DisplayBufferKeyBytes(const DisplayBuffer& buffer) { + const auto bytes = DisplayBufferSize(buffer); + require(buffer.tilingMode == 0, "VideoOut: a linear display buffer has no DCC keys"); + return Graphics::DccKeyCount(Graphics::TextureTileMode::kR64KBX, 4, buffer.width, buffer.height, bytes); +} + +Graphics::DccKeys DisplayBufferKeys(const DisplayBuffer& buffer) { + const auto count = DisplayBufferKeyBytes(buffer); + require(buffer.dccAddress != 0, "VideoOut: a display buffer without DCC metadata has no keys"); + const auto keys = Graphics::CurrentDccKeys(buffer.dccAddress, DisplayBufferSize(buffer), count); + if (keys == Graphics::DccKeys::Uncompressed || Graphics::IsDccClear(keys)) return keys; + char differing[128] = ""; + if (keys == Graphics::DccKeys::Mixed && GuestMemory::Accessible(reinterpret_cast(buffer.dccAddress), count)) { + const auto* bytes = reinterpret_cast(buffer.dccAddress); + traceDisplayKeys(buffer, bytes, count); + const auto* other = std::find_if(bytes + 1, bytes + count, [&](std::uint8_t key) { return key != bytes[0]; }); + if (other == bytes + count) { + std::snprintf(differing, sizeof(differing), ": every key is 0x%02x", bytes[0]); + } else { + const auto key = static_cast(other - bytes); + const auto [x, y] = displayableKeyPixel(buffer, key); + std::snprintf(differing, sizeof(differing), ": key %zu (pixels %u..%u x %u..%u in the displayable order) is 0x%02x, key 0 is 0x%02x", key, x, x + KeyEdge - 1u, y, y + KeyEdge - 1u, *other, bytes[0]); + } + } + char message[384]; + std::snprintf(message, sizeof(message), "VideoOut: display buffer 0x%llx reads %s DCC keys at 0x%llx over its 0x%zx key bytes%s: presenting DCC metadata that is not uniformly uncompressed or fast-cleared is not implemented", static_cast(buffer.address), Graphics::DccKeysName(keys), static_cast(buffer.dccAddress), count, differing); + throw std::runtime_error(message); +} + std::vector ReadDisplayBuffer(const DisplayBuffer& buffer) { const auto size = DisplayBufferSize(buffer); static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp index d5ab9cc25..a13acf5c8 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DataHitVerification.cpp @@ -1,22 +1,23 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "Optimization/ResourceProgram.hpp" #include #include namespace AgcDriver::DriverDetail { -void Driver::verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffset, std::uint64_t deviceSerial, ShaderRecompiler::RecompileRequest request, std::span memory, std::uint64_t address, const DispatchVariant& variant, std::span liveWords, const ShaderRecompiler::RecompileResult& patched) { +void Driver::verifyDataHit(const ShaderSnapshot& snapshot, std::size_t codeOffset, ShaderRecompiler::RecompileRequest request, std::span memory, std::uint64_t address, const DispatchVariant& variant, std::span liveWords, const ShaderRecompiler::RecompileResult& patched) { const auto fail = [&](const char* what, std::size_t position, std::size_t slot) { std::fprintf(stderr, "[dispatch-cache] APS5_VERIFY_DATA_HITS: %s disagrees (program 0x%llx, position %zu, slot %zu)\n", what, static_cast(address), position, slot); std::fflush(stderr); std::abort(); }; auto verifyMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); - const auto handle = SourceHandleFor(snapshot, codeOffset, deviceSerial, request, false); - const auto capture = verifyMemory->Capture(request, handle.get()); + const auto invocation = InvocationFor(snapshot, codeOffset, request); + const auto capture = verifyMemory->Capture(invocation); const auto regions = verifyMemory->Regions(); request.context.memory = regions; - const auto result = ShaderRecompiler::Recompile(request, *capture); + const auto result = invocation.Materialize(*capture); std::size_t position = 0, run = 0; for (const auto& region : regions) { const bool registered = std::any_of(memory.begin(), memory.end(), [&](const auto& known) { return region.guestAddress >= known.guestAddress && region.guestAddress < known.guestAddress + known.bytes.size(); }); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index 9f1d52007..975e68edb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" @@ -11,6 +12,7 @@ namespace AgcDriver::DriverDetail { void Driver::dispatch(QueueState& queue, std::span packet, const Submission& submission, std::uint64_t indirectArguments) { + PerformanceTimer timing("Driver.Dispatch"); const auto address = (static_cast(readRegister(queue.shader, 0x20c)) << 8u) | (static_cast(readRegister(queue.shader, 0x20d) & 0xffu) << 40u); auto it = submission.shaders->upper_bound(address); std::shared_ptr registeredShader; @@ -26,7 +28,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, for (std::uint32_t i = 0; i < userCount; ++i) { userData.push_back(readUserData(queue.shader, 0x240 + i)); } - auto compute = Graphics::DecodeComputeStageInfo(queue.shader); + auto compute = Graphics::DecodeComputeStageInfo(queue.shader, snapshot.header); std::vector memory{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}}; if (!snapshot.header.empty()) memory.push_back({snapshot.headerAddress, snapshot.header}); @@ -65,7 +67,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, } ShaderRecompiler::RecompileRequest request{ {ShaderRecompiler::ShaderStage::Compute, address, std::span(snapshot.code).subspan(codeOffset), snapshot.headerAddress, snapshot.header}, - {(packet[4] & 0x8000u) != 0 ? 32u : 64u, 0, userData, compute, std::nullopt, std::nullopt, memory}, + {(packet[4] & 0x8000u) != 0 ? 32u : 64u, 0, userData, compute, std::nullopt, std::nullopt, memory, RegisteredFloatMode(snapshot)}, localDevice->ComputeTarget((packet[4] & 0x8000u) != 0 ? 32u : 64u), {0, 0, 0, 128} }; @@ -98,13 +100,11 @@ void Driver::dispatch(QueueState& queue, std::span packet, if (probeThis) std::fprintf(stderr, "[gpu] probing dispatch %llu of 0x%llx\n", static_cast(probeDispatch.second), static_cast(address)); } - if (FailureMemo() && snapshot.handles->poisoned.load(std::memory_order_relaxed) != 0) { - const std::string* poisoned = nullptr; - if (SourceHandleFor(snapshot, codeOffset, localDevice->Serial(), request, probeThis, &poisoned) == nullptr && poisoned != nullptr) { - pendingDispatchPhases().outcome = DispatchOutcome::SkippedMemo; - return; - } - } + struct ProbeScope { + bool active; + explicit ProbeScope(bool active) : active(active) { if (active) ShaderRecompiler::SetDebugProbeActive(true); } + ~ProbeScope() { if (active) ShaderRecompiler::SetDebugProbeActive(false); } + } probeScope{probeThis}; const bool noDispatchCache = noDispatchCacheEnv || probeThis; std::uint64_t key = 0xcbf29ce484222325ull; const auto mix = [&](std::uint64_t value) { @@ -179,10 +179,13 @@ void Driver::dispatch(QueueState& queue, std::span packet, previous.key = key; } - if (!stampValidate()) mix(reinterpret_cast(registeredShader.get())); + mix(reinterpret_cast(registeredShader.get())); phaseTiming.Phase(PhaseKey); + timing.Mark("decode_key_device_setup"); lookupDispatch(address, submission, key, noDispatchCache, traceCache, profile, memory, phaseTiming, phaseMs, compiledResult, keepVariant, captured, liveWords, dataHit, cached, validated, missedEntry, missedDiffering); + timing.Mark("lookup_cache"); if (cached) { + require(keepVariant != nullptr && keepVariant->shader == registeredShader, "dispatch cache belongs to another registered shader"); captureMs += phaseTiming.Elapsed(); } else { shaderMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); @@ -190,18 +193,14 @@ void Driver::dispatch(QueueState& queue, std::span packet, static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; try { + const auto invocation = InvocationFor(snapshot, codeOffset, request); + timing.Mark("prepared_invocation"); - struct ProbeScope { - bool active; - explicit ProbeScope(bool active) : active(active) { if (active) ShaderRecompiler::SetDebugProbeActive(true); } - ~ProbeScope() { if (active) ShaderRecompiler::SetDebugProbeActive(false); } - } probeScope{probeThis}; const auto waitedBefore = traceCapSync() ? Graphics::Recorder::ThreadWaitedMs() : 0.0; forgetAtCapture = GuestMemory::ForgetSerial(); - const auto handle = SourceHandleFor(snapshot, codeOffset, localDevice->Serial(), request, probeThis); capture = [&] { const SampledReadScope sampling(evidenceReads); - return shaderMemory->Capture(request, handle.get()); + return shaderMemory->Capture(invocation); }(); captured = shaderMemory->Regions(); request.context.memory = captured; @@ -211,10 +210,11 @@ void Driver::dispatch(QueueState& queue, std::span packet, if (dumpShaders) static_cast(dumpRequest(address, request)); const auto started = std::chrono::steady_clock::now(); - static const bool reuseCapture = std::getenv("APS5_NO_CAPTURE_REUSE") == nullptr; - bool memoHit = false; - compiledResult = reuseCapture ? ShaderRecompiler::Recompile(request, *capture, &memoHit) : std::make_shared(ShaderRecompiler::Recompile(request)); - if (compiledResult->cacheHit || memoHit) ++cacheHits; + + timing.Mark("capture_resources"); + compiledResult = invocation.Materialize(*capture); + timing.Mark("materialize"); + if (compiledResult->cacheHit) ++cacheHits; const auto elapsed = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); static double totalMs = 0; totalMs += elapsed; @@ -231,9 +231,9 @@ void Driver::dispatch(QueueState& queue, std::span packet, } recompileMs += phaseTiming.Elapsed(); phaseTiming.Phase(PhaseRecompile); - insertDispatch(address, key, noDispatchCache, profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming); + insertDispatch(address, key, noDispatchCache || !CacheableResult(*compiledResult), profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming); } - if (verifyDataHits() && dataHit) verifyDataHit(snapshot, codeOffset, localDevice->Serial(), request, memory, address, *keepVariant, liveWords, *compiledResult); + if (verifyDataHits() && dataHit) verifyDataHit(snapshot, codeOffset, request, memory, address, *keepVariant, liveWords, *compiledResult); if (recordQueuedLabelsAfterCapture(submission.queue, captured)) { dispatch(queue, packet, submission, indirectArguments); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp index 459a69cf9..8478e84a6 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp @@ -6,6 +6,10 @@ namespace AgcDriver::DriverDetail { +bool CacheableResult(const ShaderRecompiler::RecompileResult& compiled) { + return compiled.poisonedSrtReads == 0u; +} + void Driver::insertDispatch(std::uint64_t address, std::uint64_t key, bool noDispatchCache, bool profile, const std::shared_ptr& registeredShader, std::uint64_t forgetAtCapture, std::span memory, const std::shared_ptr& shaderMemory, const std::vector& captured, const std::shared_ptr& capture, const std::shared_ptr& compiledResult, const std::shared_ptr& missedEntry, bool missedDiffering, std::shared_ptr& attachVariant, DispatchPhaseTiming& phaseTiming) { if (!noDispatchCache) { auto fresh = std::make_shared(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index 5634d788a..bf6ca03d1 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -91,7 +91,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe if (!graphics.stages.mesh) return; auto& words = programs.front().userData; require(programs.front().firstUserSgpr == 0 && words.size() >= ShaderRecompiler::MeshIndexBufferUserWord + 4, "mesh program lacks the hidden user words"); - const auto descriptor = Graphics::MeshIndexBufferDescriptor(parameters, programs.front().binary.codeAddress); + const auto descriptor = Graphics::MeshIndexBufferDescriptor(parameters); std::copy(descriptor.begin(), descriptor.end(), words.begin() + ShaderRecompiler::MeshIndexBufferUserWord); }; if (!drawParameters.indirect) setMeshIndexBuffer(drawParameters); @@ -204,7 +204,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe memory.insert(memory.end(), matchedRegions[i].begin(), matchedRegions[i].end()); } else { resultIndex[i] = results.size(); - results.push_back(compileDrawStage(i, pushCursorBytes, queue, submission, programs, graphics, pixel, vertexInfos, memory, linked, drawParameters, localDevice, shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, rejected)); + results.push_back(materializeDrawStage(i, pushCursorBytes, queue, submission, programs, graphics, pixel, vertexInfos, memory, linked, drawParameters, localDevice, shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, rejected)); if (!rejected.empty()) return DrawVerdict::Rejected; programResults[i] = &results.back(); } @@ -234,7 +234,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe const auto buildRectList = [&] { phaseTiming.Phase(DrawRowVectors); require(programs.size() == 2 && programResults[0] != nullptr && programResults[1] != nullptr, "rect-list requires vertex and fragment programs"); - auto rectangle = ShaderRecompiler::BuildRectListShaders(*programResults[0], *programResults[1], localDevice->Target()); + auto rectangle = PreparedRectangle(*programs[0].snapshot, programResults[0]->variantId, programResults[1]->variantId); if (rectListBuilt) { results[rectIndex] = std::move(rectangle.control); results[rectIndex + 1] = std::move(rectangle.evaluation); @@ -329,7 +329,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe auto& result = results[resultIndex[programIndex]]; const auto pushBytes = result.pushConstants.size(); decodeVertexInfo(programIndex); - result = compileDrawStage(programIndex, pushOffsets[programIndex], queue, submission, programs, graphics, pixel, vertexInfos, memory, linked, drawParameters, localDevice, shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, rejected); + result = materializeDrawStage(programIndex, pushOffsets[programIndex], queue, submission, programs, graphics, pixel, vertexInfos, memory, linked, drawParameters, localDevice, shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, rejected); if (!rejected.empty()) return DrawVerdict::Rejected; require(result.pushConstants.size() == pushBytes, "patched program changed its push constant layout"); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp index 8399fc997..ad07dcdc6 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp @@ -1,38 +1,37 @@ +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" +#include "Optimization/ResourceProgram.hpp" #include #include namespace AgcDriver::DriverDetail { -ShaderRecompiler::RecompileResult Driver::compileDrawStage(std::size_t i, std::uint32_t pushOffset, const QueueState& queue, const Submission& submission, const std::vector& programs, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector>& vertexInfos, std::vector& memory, const std::vector& linked, const Pm4::DrawParameters& drawParameters, const std::shared_ptr& localDevice, ShaderMemory& shaderMemory, std::vector& stageCaptures, std::vector& recompiled, bool drawHit, const std::vector>& matched, const std::vector>& matchedRegions, bool profile, std::uint64_t dumpTarget, std::uint64_t dumpSlot1, std::uint64_t& captures, DrawPhaseTiming& phaseTiming, std::array& phaseMs, std::string& rejected) { +ShaderRecompiler::RecompileResult Driver::materializeDrawStage(std::size_t i, std::uint32_t pushOffset, const QueueState& queue, const Submission& submission, const std::vector& programs, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector>& vertexInfos, std::vector& memory, const std::vector& linked, const Pm4::DrawParameters& drawParameters, const std::shared_ptr& localDevice, ShaderMemory& shaderMemory, std::vector& stageCaptures, std::vector& recompiled, bool drawHit, const std::vector>& matched, const std::vector>& matchedRegions, bool profile, std::uint64_t dumpTarget, std::uint64_t dumpSlot1, std::uint64_t& captures, DrawPhaseTiming& phaseTiming, std::array& phaseMs, std::string& rejected) { + PerformanceTimer timing("Shader.DrawStage"); using Stage = ShaderRecompiler::ShaderStage; phaseTiming.Phase(DrawRowVectors); const auto& program = programs[i]; const auto waveSize = program.binary.stage == Stage::Fragment ? graphics.stages.fragmentWaveSize : graphics.stages.vertexWaveSize; ShaderRecompiler::RecompileRequest request{ program.binary, - {waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(pixel) : std::nullopt, vertexInfos[i], memory}, + {waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(pixel) : std::nullopt, vertexInfos[i], memory, RegisteredFloatMode(*program.snapshot)}, localDevice->Target(), {0, 0, pushOffset, (graphics.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : Graphics::PipelinePushConstantBytes) - pushOffset}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, graphics.stages.mesh, graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}} }; const auto waitedBefore = traceCapSync() || profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; - const std::string* poisoned = nullptr; - const auto handle = SourceHandleFor(*program.snapshot, program.codeOffset, localDevice->Serial(), request, false, FailureMemo() ? &poisoned : nullptr); - if (handle == nullptr && poisoned != nullptr) { - rejected = *poisoned; - return {}; - } + const auto invocation = InvocationFor(*program.snapshot, program.codeOffset, request); + timing.Mark("prepared_invocation"); auto& stageCapture = stageCaptures[i]; stageCapture.forgetSerial = GuestMemory::ForgetSerial(); stageCapture.pushOffset = pushOffset; const auto capture = [&] { const SampledReadScope sampling(evidenceReads); - return shaderMemory.Capture(request, handle.get()); + return shaderMemory.Capture(invocation); }(); stageCapture.regions = shaderMemory.TakeRecentRegions(); @@ -73,10 +72,11 @@ ShaderRecompiler::RecompileResult Driver::compileDrawStage(std::size_t i, std::u } } - static const bool reuseCapture = std::getenv("APS5_NO_CAPTURE_REUSE") == nullptr; phaseTiming.Phase(DrawRowCapture); - stageCapture.compiled = reuseCapture ? ShaderRecompiler::Recompile(request, *capture) : std::make_shared(ShaderRecompiler::Recompile(request)); + timing.Mark("capture_resources"); + stageCapture.compiled = invocation.Materialize(*capture); + timing.Mark("materialize"); ShaderRecompiler::RecompileResult result = *stageCapture.compiled; phaseTiming.Phase(DrawRowRecompile); return result; @@ -88,7 +88,7 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP std::uint64_t unstable = 0, mismatches = 0; for (std::size_t i = 0; i < programs.size(); ++i) { const auto& stageCapture = stageCaptures[i]; - if (stageCapture.compiled == nullptr) continue; + if (stageCapture.compiled == nullptr || !CacheableResult(*stageCapture.compiled)) continue; auto variant = std::make_shared(); variant->compiled = stageCapture.compiled; variant->shader = programs[i].snapshot; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp index f3d2e0120..fac2d2bdc 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp @@ -5,27 +5,38 @@ namespace AgcDriver::DriverDetail { -std::shared_ptr Driver::decodeDraw(const QueueState& queue, const Submission& submission) { +namespace { + +std::uint32_t ReadGraphicsRegister(const Registers& registers, std::uint32_t offset) { + const auto found = registers.find(offset); + return found == registers.end() ? 0u : found->second; +} + +} + +void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const ShaderRegistry& registry, bool staticAbi, bool includeFragment) { using Stage = ShaderRecompiler::ShaderStage; using Role = ShaderRecompiler::ProgramRole; const auto programAddress = [&](std::uint32_t base) { Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, base); Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, base + 1); - const auto high = readRegister(queue.shader, base + 1); + const auto high = ReadGraphicsRegister(queue.shader, base + 1); require((high & ~0xffu) == 0, "reserved graphics program address bits are set"); - return (static_cast(readRegister(queue.shader, base)) << 8u) | (static_cast(high) << 40u); + return (static_cast(ReadGraphicsRegister(queue.shader, base)) << 8u) | (static_cast(high) << 40u); }; + const bool pixelSkipped = Graphics::PixelProgramSkipped(queue); const auto prepare = [&](std::uint64_t address, std::uint8_t type, Stage stage, std::uint32_t rsrc2, std::uint32_t userDataBase) { - const bool nullPixel = address == 0 && stage == Stage::Fragment; + const bool nullPixel = stage == Stage::Fragment && (address == 0 || pixelSkipped); if (nullPixel) address = NullPixelProgramAddress(); - auto it = submission.shaders->upper_bound(address); - require(it != submission.shaders->begin(), "graphics program does not belong to a registered shader"); + auto it = registry.upper_bound(address); + require(it != registry.begin(), "graphics program does not belong to a registered shader"); --it; const auto& snapshot = *it->second; require(address - snapshot.codeAddress < snapshot.code.size() * sizeof(std::uint32_t), "graphics program is outside registered shader code"); + require((address - snapshot.codeAddress) % sizeof(std::uint32_t) == 0, "graphics entry point is not dword aligned"); require(snapshot.type == type, "graphics program refers to an incompatible shader binary type"); Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, rsrc2); - const auto resources = nullPixel && !queue.shader.contains(rsrc2) ? 0u : readRegister(queue.shader, rsrc2); + const auto resources = nullPixel && !queue.shader.contains(rsrc2) ? 0u : ReadGraphicsRegister(queue.shader, rsrc2); const auto userCount = ((resources >> 1u) & 0x1fu) | (((resources >> 27u) & 1u) << 5u); require(userCount <= 32, "graphics user SGPR count exceeds the register bank"); const auto codeOffset = static_cast((address - snapshot.codeAddress) / sizeof(std::uint32_t)); @@ -40,13 +51,12 @@ std::shared_ptr Driver::decodeDraw(const QueueState& queue, const Su }; for (std::uint32_t i = 0; i < userCount; ++i) { Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, userDataBase + i); - result.userData.push_back(readUserData(queue.shader, userDataBase + i)); + result.userData.push_back((staticAbi ? ReadGraphicsRegister(queue.shader, userDataBase + i) : readUserData(queue.shader, userDataBase + i))); } return result; }; { - auto product = std::make_shared(); - product->state = Graphics::DecodeState(queue); + auto* product = &decoded; auto& programs = product->programs; auto& roles = product->roles; programs.reserve(5); @@ -60,9 +70,10 @@ std::shared_ptr Driver::decodeDraw(const QueueState& queue, const Su program.userData.insert(program.userData.begin(), 8, 0); Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, pointerBase); Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, pointerBase + 1); + if (staticAbi) return; if (!pointerRequired && !queue.shader.contains(pointerBase) && !queue.shader.contains(pointerBase + 1)) return; - const auto low = readRegister(queue.shader, pointerBase); - const auto high = readRegister(queue.shader, pointerBase + 1); + const auto low = ReadGraphicsRegister(queue.shader, pointerBase); + const auto high = ReadGraphicsRegister(queue.shader, pointerBase + 1); const auto address = static_cast(low) | (static_cast(high) << 32u); require(address != 0 || !pointerRequired, "merged shader user-data address is null"); if (address == 0) return; @@ -78,8 +89,8 @@ std::shared_ptr Driver::decodeDraw(const QueueState& queue, const Su append(0x0c8, 2, Stage::TessellationEvaluation, 0x08b, 0x08c, Role::Domain); } else if (graphics.stages.path == Graphics::ShaderPath::Geometry) { const auto frontAddress = programAddress(0xc8); - auto snapshot = submission.shaders->upper_bound(frontAddress); - require(snapshot != submission.shaders->begin(), "geometry front program is not registered"); + auto snapshot = registry.upper_bound(frontAddress); + require(snapshot != registry.begin(), "geometry front program is not registered"); --snapshot; const auto type = snapshot->second->type; require(type == 2 || type == 4, "invalid geometry front binary type"); @@ -89,18 +100,25 @@ std::shared_ptr Driver::decodeDraw(const QueueState& queue, const Su } else { append(0xc8, 2, Stage::Vertex, 0x8b, 0x8c, Role::Main); } - const bool nullPixel = Graphics::PixelProgramUnset(queue); - if (nullPixel) { - const auto rejection = Graphics::NullPixelProgramRejection(queue); - require(rejection.empty(), rejection.c_str()); + if (includeFragment) { + if (pixelSkipped) { + const auto rejection = Graphics::NullPixelProgramRejection(queue); + require(rejection.empty(), rejection.c_str()); + } + append(0x008, 1, Stage::Fragment, 0x00b, 0x00c, Role::Fragment); + programs.back().firstUserSgpr = 0; } - append(0x008, 1, Stage::Fragment, 0x00b, 0x00c, Role::Fragment); - programs.back().firstUserSgpr = 0; - product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(graphics), nullPixel); - return product; } } +std::shared_ptr Driver::decodeDraw(const QueueState& queue, const Submission& submission) { + auto product = std::make_shared(); + product->state = Graphics::DecodeState(queue); + DecodeGraphicsPrograms(*product, queue, *submission.shaders, false, true); + product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(product->state), Graphics::PixelProgramSkipped(queue)); + return product; +} + void Driver::resolveDrawDecode(const QueueState& queue, const Submission& submission, std::shared_ptr& decode, bool registerKey, std::uint64_t drawKey, bool profile) { if (decode == nullptr || verifyDrawRecipe()) { std::vector readLog; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp index cdc931174..8d42a6ae6 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp @@ -63,6 +63,7 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr(pixel.conservativeZExport)); + mix(pixel.orderedPixelShader); for (const auto value : pixel.targetOutputMode) mix(value); for (const auto value : pixel.targetExportMapping) mix(value); std::lock_guard cacheLock(drawCacheMutex); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp index f2842ad61..e75ed53ee 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp @@ -23,13 +23,15 @@ std::optional Driver::precheckDraw(const QueueState& queue, const S localDevice->ColorMetadataPass(*pass); return DrawVerdict::Drawn; } + rejected = Graphics::DepthMaintenanceRejection(queue); + if (!rejected.empty()) return DrawVerdict::Rejected; static const bool traceIndirectEnabled = std::getenv("APS5_TRACE_INDIRECT_DRAWS") != nullptr; traceIndirect = traceIndirectEnabled; if (traceIndirect && drawParameters.indirect) std::fprintf(stderr, "[draw] indirect packet %s args 0x%llx count %u reached\n", Pm4::Name(packet[0]).c_str(), static_cast(drawParameters.indirect->arguments), drawParameters.indirect->count); { const auto targetMask = queue.context.find(0x8e); const auto shaderMask = queue.context.find(0x8f); - const bool colorWrites = targetMask != queue.context.end() && shaderMask != queue.context.end() && (targetMask->second & shaderMask->second) != 0; + const bool colorWrites = targetMask != queue.context.end() && shaderMask != queue.context.end() && Graphics::ColorWriteMask(queue.context) != 0; if (!colorWrites && !queue.shader.contains(0x8)) { const auto word = [&](std::uint32_t offset) { const auto it = queue.context.find(offset); return it == queue.context.end() ? 0u : it->second; }; if ((word(0x200) & 3u) == 0 || ((word(0x010) & 3u) == 0 && (word(0x011) & 1u) == 0)) return DrawVerdict::Nothing; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp index 5181d61f5..29284743d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp @@ -1,20 +1,37 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include +#include +#include #include namespace AgcDriver::DriverDetail { std::uint64_t Driver::drawRegisterKey(const QueueState& queue, const ShaderRegistry& registry, std::uint64_t deviceSerial) { + static const bool allUserWords = std::getenv("APS5_DRAW_KEY_ALL_USER_WORDS") != nullptr; std::uint64_t key = 0xcbf29ce484222325ull; const auto mix = [&](std::uint64_t value) { key ^= value; key *= 0x100000001b3ull; }; + const auto userEnd = [&](std::uint32_t base) { + if (allUserWords) return base + 32u; + const auto resources = queue.shader.find(base - 1); + if (resources == queue.shader.end()) return base; + const auto count = ((resources->second >> 1u) & 0x1fu) | (((resources->second >> 27u) & 1u) << 5u); + return base + std::min(count, 32u); + }; + const std::array, 3> users{{{0x00cu, userEnd(0x00cu)}, {0x08cu, userEnd(0x08cu)}, {0x10cu, userEnd(0x10cu)}}}; + const auto unread = [&](std::uint32_t offset) { + return std::any_of(users.begin(), users.end(), [&](const auto& user) { return offset >= user.second && offset < user.first + 32u; }); + }; mix(deviceSerial); for (const auto& range : Graphics::DrawKeyRegisters) { const auto& bank = range.bank == Graphics::RegisterBank::Context ? queue.context : range.bank == Graphics::RegisterBank::Shader ? queue.shader : queue.userConfig; mix((static_cast(range.bank) << 32u) | range.first); const auto end = range.first + range.count; + const bool shader = range.bank == Graphics::RegisterBank::Shader; for (auto it = bank.lower_bound(range.first); it != bank.end() && it->first < end; ++it) { + if (shader && unread(it->first)) continue; mix(it->first); mix(it->second); } @@ -77,11 +94,11 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) { for (std::size_t i = 0; i < s.blends.size(); ++i) { if (!sameBlend(s.blends[i], t.blends[i])) return false; } - if (s.hasColorTarget != t.hasColorTarget || s.rectList != t.rectList || s.renderExtent.width != t.renderExtent.width || s.renderExtent.height != t.renderExtent.height || s.topology != t.topology || s.negativeOneToOne != t.negativeOneToOne || s.depthClamp != t.depthClamp || s.cullMode != t.cullMode || s.frontFace != t.frontFace || !sameBlend(s.blend, t.blend) || s.blendConstants != t.blendConstants) return false; + if (s.hasColorTarget != t.hasColorTarget || s.rectList != t.rectList || s.renderExtent.width != t.renderExtent.width || s.renderExtent.height != t.renderExtent.height || s.topology != t.topology || s.negativeOneToOne != t.negativeOneToOne || s.depthClamp != t.depthClamp || s.conservativeRasterization != t.conservativeRasterization || s.cullMode != t.cullMode || s.frontFace != t.frontFace || !sameBlend(s.blend, t.blend) || s.blendConstants != t.blendConstants) return false; if (std::memcmp(&s.viewport, &t.viewport, sizeof(VkViewport)) != 0 || std::memcmp(&s.scissor, &t.scissor, sizeof(VkRect2D)) != 0) return false; const auto& p = a.pixel; const auto& q = b.pixel; - if (p.interpolatorCount != q.interpolatorCount || p.interpolatorSettings != q.interpolatorSettings || p.wave32 != q.wave32 || p.inputAddr != q.inputAddr || p.hasPerspectiveCenterVgpr != q.hasPerspectiveCenterVgpr || p.perspectiveCentroid != q.perspectiveCentroid || p.posX != q.posX || p.posY != q.posY || p.posZ != q.posZ || p.posW != q.posW || p.frontFace != q.frontFace || p.ancillary != q.ancillary || p.sampleShading != q.sampleShading || p.noPerspective != q.noPerspective || p.linearCentroid != q.linearCentroid || p.pixelKillEnable != q.pixelKillEnable || p.depthExportEnable != q.depthExportEnable || p.sampleMaskExportEnable != q.sampleMaskExportEnable || p.earlyZ != q.earlyZ || p.executeOnNoop != q.executeOnNoop || p.conservativeZExport != q.conservativeZExport || p.targetOutputMode != q.targetOutputMode || p.targetExportMapping != q.targetExportMapping) return false; + if (p.interpolatorCount != q.interpolatorCount || p.interpolatorSettings != q.interpolatorSettings || p.wave32 != q.wave32 || p.inputAddr != q.inputAddr || p.hasPerspectiveCenterVgpr != q.hasPerspectiveCenterVgpr || p.perspectiveCentroid != q.perspectiveCentroid || p.posX != q.posX || p.posY != q.posY || p.posZ != q.posZ || p.posW != q.posW || p.frontFace != q.frontFace || p.ancillary != q.ancillary || p.sampleShading != q.sampleShading || p.noPerspective != q.noPerspective || p.linearCentroid != q.linearCentroid || p.pixelKillEnable != q.pixelKillEnable || p.depthExportEnable != q.depthExportEnable || p.sampleMaskExportEnable != q.sampleMaskExportEnable || p.earlyZ != q.earlyZ || p.executeOnNoop != q.executeOnNoop || p.conservativeZExport != q.conservativeZExport || p.orderedPixelShader != q.orderedPixelShader || p.targetOutputMode != q.targetOutputMode || p.targetExportMapping != q.targetExportMapping) return false; if (a.roles != b.roles || a.programs.size() != b.programs.size()) return false; for (std::size_t i = 0; i < a.programs.size(); ++i) { const auto& x = a.programs[i]; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp index 91e589164..526c7ab37 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/EntryPoints.cpp @@ -33,6 +33,10 @@ void UnregisterVideoOutput(std::uint32_t handle, const std::shared_ptr context, std::span primitive) { + AgcDriver::DriverDetail::Driver::Get().ResolveShaderAbi(shader, context, primitive); +} + +extern "C" void AgcDriverResolveGraphicsAbi_nid_postfix(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType) { + AgcDriver::DriverDetail::Driver::Get().ResolveGraphicsAbi(vertex, pixel, primitiveType); +} +extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span stages, std::span context, std::span primitive) { + AgcDriver::DriverDetail::Driver::Get().ResolveGraphicsStagesAbi(stages, context, primitive); +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp index 5854c488a..ebe789113 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp @@ -2,6 +2,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp" +#include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" namespace AgcDriver::DriverDetail { @@ -44,6 +45,7 @@ void Driver::stop() { device.Reset(); replacedDevices.clear(); Graphics::ShutdownGuestBufferWorkers(); + StopProfileOutput_nid_no_patch(); stopped = true; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index af3361308..3a2b72269 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -29,17 +29,11 @@ void Driver::timed(double WorkerProfile::*bucket, TWork&& work) { } } -template -void Driver::tolerate(const char* kind, TWork&& work) { - try { - work(); - } catch (const std::exception& error) { - reportSkip(kind, error.what()); - } -} - void Driver::execute(const Submission& submission) { + auto* submissionTiming = includeTimingSubmission(submission, true); if (submission.suspend) { + PerformanceContext timingContext(submissionTiming); + PerformanceTimer timing("Driver.Suspend"); static const bool suspendDrain = std::getenv("APS5_SUSPEND_DRAIN") != nullptr; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; @@ -113,9 +107,13 @@ void Driver::execute(const Submission& submission) { const auto packet = std::span(submission.commands).subspan(cursor, count); const auto opcode = (header >> 8u) & 0xffu; auto nextCursor = cursor + count; + if (APS5_ENABLE_TIMING_LOG && submission.queue == 0 && pendingFrameTiming == nullptr) includeTimingSubmission(submission, false); + PerformanceContext timingContext(submission.queue == 0 ? pendingFrameTiming.get() : FrameTiming::Async()); + PerformanceTimer timing("Driver.Packet"); std::shared_lock deviceUse(deviceReplacement, std::defer_lock); if (opcode != 0x3c && opcode != 0x93 && header != RenderingWaitPacketHeader && header != FlipPacketHeader) deviceUse.lock(); + timing.Mark("device_lock"); GuestMemory::SetCurrentPacket(header == FlipPacketHeader ? 0xffffu : opcode, submission.queue); CaptureTrace::Log("packet submission=%llu queue=%x offset=%zu header=%08x words=%zu", static_cast(submission.serial), submission.queue, cursor, header, packet.size()); if (Pm4::Predicated(header) && queue.predication.operation != 0) { @@ -132,6 +130,7 @@ void Driver::execute(const Submission& submission) { const auto flushStart = profilePackets ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; flushBetweenPackets(submission.queue, header, opcode == 0x49 || opcode == 0x37); + timing.Mark("flush_between_packets"); PacketTimer packetTimer{profilePackets, header == FlipPacketHeader ? 0xffffu : opcode, submission.queue, packetProfile, std::chrono::steady_clock::now()}; if (profilePackets) { @@ -170,6 +169,7 @@ void Driver::execute(const Submission& submission) { bool wroteOnGpu = false, endOfPipeInterrupt = false, interruptDeferred = false, drawPacket = false, sampleDump = false; const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, wroteOnGpu, endOfPipeInterrupt, interruptDeferred, drawPacket, sampleDump); traceLabel(packet, submission.queue); + timing.Mark("prepare_memory"); const bool waitPacket = opcode == 0x3c || opcode == 0x93 || header == RenderingWaitPacketHeader; struct Progress { @@ -201,10 +201,14 @@ void Driver::execute(const Submission& submission) { if (batchesAtFlip != 0) flipSerial = batchesAtFlip; flipBatchesUnsignaled += unsignaledAtFlip; - auto frame = std::make_shared(++frameSerial); + timing.Mark("flip_flush"); + timing.FinishPacket(submission.serial, cursor, 0xffffu, count); + auto frame = APS5_ENABLE_TIMING_LOG ? std::move(pendingFrameTiming) : std::make_shared(++frameSerial); + require(frame != nullptr, "flip has no frame timing"); const auto now = FrameTiming::Clock::now(); - frame->IncludeSubmission(submission.serial, now, now, now, true); - frame->SetFlip(submission.serial, cursor, now, now); + if (!APS5_ENABLE_TIMING_LOG) frame->IncludeSubmission(submission.serial, now, now, now, true); + frame->SetFlip(submission.serial, cursor, APS5_ENABLE_TIMING_LOG ? submission.receivedAt : now, now); + if (APS5_ENABLE_TIMING_LOG) frame->CollectBackground(); frame->NoteFlipBatches(batchesAtFlip, unsignaledAtFlip); CaptureTrace::Log("flip frame=%llu submission=%llu offset=%zu batch=%llu unsignaled=%llu", static_cast(frameSerial), static_cast(submission.serial), cursor, static_cast(batchesAtFlip), static_cast(unsignaledAtFlip)); submission.flips.at(cursor)->GpuReady(frame); @@ -234,7 +238,7 @@ void Driver::execute(const Submission& submission) { } } else if (drawPacket) { bool drawn = false; - timed(&WorkerProfile::drawMs, [&] { tolerate("draw", [&] { + timed(&WorkerProfile::drawMs, [&] { static const bool traceDraws = std::getenv("APS5_TRACE_DRAWS") != nullptr; static const bool profileDraws = std::getenv("APS5_PROFILE_DRAW") != nullptr; const auto color = (static_cast(readRegister(queue.context, 0x390)) << 40u) | (static_cast(readRegister(queue.context, 0x318)) << 8u); @@ -243,36 +247,14 @@ void Driver::execute(const Submission& submission) { if (profileDraws) Graphics::CountDrawSkip(kind, std::chrono::duration(std::chrono::steady_clock::now() - started).count()); }; - const auto skipped = [&](const std::string& what) { - if (traceDraws) std::fprintf(stderr, "[draw] target 0x%llx mask 0x%x failed: %.160s\n",static_cast(color), readRegister(queue.context, 0x8e), what.c_str()); - - char suffix[80]; - std::snprintf(suffix, sizeof(suffix), " [%s, color target 0x%llx]", Pm4::Name(header).c_str(), static_cast(color)); - - static std::set> dumpedTargets; - const std::string reason = what.substr(0, 48); - if (dumpedTargets.insert({color, reason}).second) { - char name[64]; - std::snprintf(name, sizeof(name), "draw_%llx_%08x.regs", static_cast(color), static_cast(std::hash{}(reason))); - if (std::FILE* file = std::fopen(name, "w")) { - std::fprintf(file, "# %s\n", what.c_str()); - recent.Print(file, "# packet %s\n"); - for (const auto& [offset, value] : queue.context) std::fprintf(file, "context %x %08x\n", offset, value); - for (const auto& [offset, value] : queue.userConfig) std::fprintf(file, "uconfig %x %08x\n", offset, value); - for (const auto& [offset, value] : queue.shader) std::fprintf(file, "shader %x %08x\n", offset, value); - std::fclose(file); - } - } - reportSkip("draw", what + suffix); - }; try { std::string rejected; const auto verdict = draw(queue, packet, submission, rejected); drawn = verdict == DrawVerdict::Drawn; CaptureTrace::Log("draw submission=%llu queue=%x offset=%zu target=%llx mask=%x verdict=%d reason=%.256s", static_cast(submission.serial), submission.queue, cursor, static_cast(color), readRegister(queue.context, 0x8e), static_cast(verdict), rejected.c_str()); if (verdict == DrawVerdict::Rejected) { - skipped(rejected); countSkip(Graphics::DrawSkip::Prechecked); + throw std::runtime_error(rejected); } else if (verdict == DrawVerdict::Nothing) { countSkip(Graphics::DrawSkip::Nothing); } else if (traceDraws) { @@ -280,10 +262,10 @@ void Driver::execute(const Submission& submission) { } } catch (const std::exception& error) { CaptureTrace::Log("draw-error submission=%llu offset=%zu reason=%.256s", static_cast(submission.serial), cursor, error.what()); - skipped(error.what()); countSkip(Graphics::DrawSkip::Thrown); + throw; } - }); }); + }); finishDrawPacket(drawn); } else if (opcode == 0x22) { recordQueuedLabelsBeforeRead(submission.queue); @@ -294,7 +276,8 @@ void Driver::execute(const Submission& submission) { dumpSampleCounters(packet[2] | (static_cast(packet[3]) << 32u)); } else if (opcode != 0x42 && opcode != 0x46 && opcode != 0x58) { if (!wroteOnGpu) { - Pm4::Execute(packet, queue); + if (Pm4::IndirectRegisterOpcode(opcode)) Pm4::ExecuteIndirectRegisters(packet, submission.registerLists.at(cursor), queue); + else Pm4::Execute(packet, queue); if (opcode == 0x49 || opcode == 0x37) { if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial); } @@ -302,9 +285,14 @@ void Driver::execute(const Submission& submission) { if (endOfPipeInterrupt && !interruptDeferred) AgcDriverDeliverEopInterrupt(submission.queue); } if (drawPacket || (sampleDump && wroteOnGpu)) Graphics::Recorder::CountRecordedWork(); + timing.Mark(header == RenderingWaitPacketHeader ? "rendering_wait" : opcode == 0x3c || opcode == 0x93 ? "memory_wait" : opcode == 0x15 || opcode == 0x16 ? "dispatch" : drawPacket ? "draw" : "execute"); + if (submission.queue == 0) timing.FinishPacket(submission.serial, cursor, opcode, count); + else timing.Finish(); cursor = nextCursor; } + PerformanceContext endContext(submission.queue == 0 ? frameTiming() : FrameTiming::Async()); + PerformanceTimer endTiming("Driver.EndSubmission"); static const bool submitAtEnd = std::getenv("APS5_SUBMIT_AT_END") != nullptr; if (!deferredLabels().labels.empty() || Graphics::Recorder::PendingLabelSince().has_value() || Graphics::Recorder::RecordedWorkSinceSubmit() != 0) { auto& costs = submissionCosts(submission.queue); @@ -320,6 +308,7 @@ void Driver::execute(const Submission& submission) { recordDeferredLabels(localDevice.get(), submission.queue); if (localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0); } + endTiming.Finish(); if (submission.rewindTail != nullptr) executeRewindTail(submission); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index 442cae836..de4b778f5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -18,7 +18,8 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue bool orderedAlready = false; - endOfPipeInterrupt = opcode == 0x49 && ((packet[2] >> 24u) & 7u) != 0; + const auto interruptSelect = (packet[2] >> 24u) & 7u; + endOfPipeInterrupt = opcode == 0x49 && interruptSelect != 0 && interruptSelect != 3; interruptDeferred = false; if (!drainAll && endOfPipeInterrupt) { const auto label = Pm4::DecodeLabelWrite(packet); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp index e42553239..e1c00238f 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Presentation.cpp @@ -23,6 +23,19 @@ void Driver::UnregisterVideoOutput(std::uint32_t handle, const std::shared_ptrWindow() == nullptr, "a window is already attached to the device"); + device->PrepareForReplacement(); + replacedDevices.push_back(device); + } + device = std::make_shared(&window); +} + void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buffer, bool opaque, void (*gpuReady)(void*), void* context) { PerformanceContext timingContext(window.timing.get()); @@ -48,19 +61,10 @@ void Driver::Present(const PresentationWindow& window, const DisplayBuffer* buff bool trailing = false; double waitedMs = 0; { - std::unique_lock replacing(deviceReplacement, std::defer_lock); - if (const auto current = device.Load(); current == nullptr || current->Window() == nullptr) replacing.lock(); GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Present); std::lock_guard lock(GuestMemory::GpuMutex()); timing.Mark("gpu_mutex_wait"); - if (device == nullptr || device->Window() == nullptr) { - if (device) { - device->PrepareForReplacement(); - replacedDevices.push_back(device); - } - device = std::make_shared(&window); - } - require(device->Window() == window.context, "presentation window does not match device surface"); + require(device != nullptr && device->Window() == window.context, "presentation window is not attached to the device"); presenting = device; timing.Mark("device_setup"); std::uint32_t drawableWidth = 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp index dca6dd040..9d9232a0a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp @@ -2,12 +2,42 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Queues/WorkerAffinity.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp" +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include namespace AgcDriver::DriverDetail { +FrameTiming* Driver::frameTiming() { + if (!APS5_ENABLE_TIMING_LOG) return nullptr; + if (pendingFrameTiming == nullptr) pendingFrameTiming = std::make_shared(++frameSerial, true); + return pendingFrameTiming.get(); +} + +FrameTiming* Driver::includeTimingSubmission(const Submission& submission, bool firstSegment) { + if (!APS5_ENABLE_TIMING_LOG) return nullptr; + auto* frame = submission.queue == 0 ? frameTiming() : FrameTiming::Async(); + require(submission.receivedAt != FrameTiming::Clock::time_point{} && submission.receivedAt <= submission.enqueuedAt && submission.enqueuedAt <= submission.dequeuedAt && submission.dequeuedAt <= submission.orderedAt, "invalid submission timing order"); + if (submission.queue == 0) frame->IncludeSubmission(submission.serial, submission.receivedAt, submission.enqueuedAt, submission.orderedAt, firstSegment); + else if (firstSegment) { + frame->Add(frame->Get("Submission", "accept"), submission.enqueuedAt - submission.receivedAt); + frame->Add(frame->Get("Submission", "queue"), submission.orderedAt - submission.enqueuedAt); + } + if (firstSegment) { + frame->Add(frame->Get("Submission", "dequeue"), submission.dequeuedAt - submission.enqueuedAt); + frame->Add(frame->Get("Submission", "order_wait"), submission.orderedAt - submission.dequeuedAt); + if (!submission.suspend) { + frame->Add(frame->Get("Submission", "copy"), submission.copiedAt - submission.receivedAt, submission.commands.size() * sizeof(std::uint32_t)); + frame->Add(frame->Get("Submission", "validate"), submission.validatedAt - submission.copiedAt); + frame->Add(frame->Get("Submission", "flip_room_wait"), submission.roomReadyAt - submission.validatedAt); + frame->Add(frame->Get("Submission", "reserve_enqueue"), submission.enqueuedAt - submission.roomReadyAt); + } + } + return frame; +} + void Driver::markCompleted(std::uint64_t serial) { completedOutOfOrder.insert(serial); while (!completedOutOfOrder.empty() && *completedOutOfOrder.begin() == completed + 1) { @@ -52,12 +82,25 @@ void Driver::run(std::uint32_t id) noexcept { submission = Submission{}; static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; { + PerformanceContext timingContext(id == 0 ? frameTiming() : FrameTiming::Async()); + PerformanceTimer timing("Driver.WorkerWait"); std::unique_lock lock(mutex); + timing.Mark("mutex_wait"); auto& worker = workers.at(id); auto& pending = worker.pending; workerQueued() = &worker.queued; if (traceGpu && pending.empty()) std::fprintf(stderr, "[gpu] %.1f idle queue=0x%x\n", TraceMs(), id); const auto ready = [&] { return failure || stopping || !pending.empty(); }; + const auto poll = [&](const auto& done) { +#ifdef _WIN32 + lock.unlock(); + PollSleep(); + lock.lock(); + return done(); +#else + return changed.wait_for(lock, std::chrono::milliseconds(1), done); +#endif + }; while (!ready()) { if (!completionsPending() || (id != 0 && Graphics::Recorder::PendingCompletionLabels() == 0)) { @@ -66,24 +109,29 @@ void Driver::run(std::uint32_t id) noexcept { if (id == 0) queue0Dormant.store(false, std::memory_order_relaxed); break; } - if (changed.wait_for(lock, std::chrono::milliseconds(1), ready)) break; + if (poll(ready)) break; lock.unlock(); reapCompletionLabels(); lock.lock(); } + timing.Mark("idle_and_completion_poll"); rethrowFailure(); if (stopping || shutdownToken.stop_requested() || pending.empty()) { break; } submission = std::move(pending.front()); pending.pop_front(); + if (APS5_ENABLE_TIMING_LOG) submission.dequeuedAt = std::chrono::steady_clock::now(); if (id == 0) queue0Executing = submission.suspend ? 0 : submission.received; if (submission.waitFree) { orderHolders.fetch_add(1, std::memory_order_acq_rel); - while (!failure && !stopping && !orderReleased(id, submission.received)) changed.wait_for(lock, std::chrono::milliseconds(1)); + const auto released = [&] { return failure || stopping || orderReleased(id, submission.received); }; + while (!released()) poll(released); orderHolders.fetch_sub(1, std::memory_order_acq_rel); rethrowFailure(); } + if (APS5_ENABLE_TIMING_LOG) submission.orderedAt = std::chrono::steady_clock::now(); + timing.Mark("dequeue_order"); worker.queued.fetch_sub(1, std::memory_order_acq_rel); if (profile && submission.enqueuedAt != std::chrono::steady_clock::time_point{}) costs.dequeueNs += static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - submission.enqueuedAt).count()); } @@ -97,6 +145,8 @@ void Driver::run(std::uint32_t id) noexcept { } if (traceGpu) std::fprintf(stderr, "[gpu] %.1f done serial=%llu queue=0x%x\n", TraceMs(), static_cast(submission.serial), id); const auto completeStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + PerformanceContext completionContext(id == 0 ? frameTiming() : FrameTiming::Async()); + PerformanceTimer completionTiming("Driver.Completion"); bool notify = true; { std::lock_guard lock(mutex); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp index 08f5cee6c..3df687bf6 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" @@ -103,6 +104,19 @@ bool Driver::copySegment(Submission& submission, const std::uint32_t* guest, std return false; } +void Driver::readRegisterLists(Submission& submission) { + submission.registerLists.clear(); + for (std::size_t cursor = 0; cursor < submission.commands.size(); cursor += Pm4::PacketWords(submission.commands[cursor])) { + const auto header = submission.commands[cursor]; + if ((header >> 30u) != 3u || !Pm4::IndirectRegisterOpcode((header >> 8u) & 0xffu)) continue; + try { + submission.registerLists.emplace(cursor, Pm4::ReadIndirectRegisters(std::span(submission.commands).subspan(cursor, Pm4::PacketWords(header)))); + } catch (const std::exception& error) { + throw std::runtime_error("AGC driver: " + Pm4::Name(header) + " at DWORD " + std::to_string(cursor) + ": " + error.what()); + } + } +} + void Driver::waitForFlipRoom(const Submission& submission) { for (std::size_t cursor = 0; cursor < submission.commands.size(); cursor += Pm4::PacketWords(submission.commands[cursor])) { if (submission.commands[cursor] != FlipPacketHeader) continue; @@ -141,6 +155,7 @@ void Driver::reserveOutputs(Submission& submission) { } void Driver::executeRewindTail(const Submission& stalled) { + PerformanceTimer timing("Driver.Rewind"); std::atomic_ref control(*const_cast(stalled.rewindTail - 1)); if ((control.load(std::memory_order_acquire) & 0x80000000u) == 0) { noteWaitBlocked(stalled.queue, reinterpret_cast(stalled.rewindTail - 1), true); @@ -157,9 +172,14 @@ void Driver::executeRewindTail(const Submission& stalled) { } Submission tail{}; tail.queue = stalled.queue; + if (APS5_ENABLE_TIMING_LOG) tail.receivedAt = std::chrono::steady_clock::now(); copyCommands(tail, stalled.rewindTail, stalled.rewindWords); + if (APS5_ENABLE_TIMING_LOG) tail.copiedAt = std::chrono::steady_clock::now(); validate(tail, stalled.rewindTail); + readRegisterLists(tail); + if (APS5_ENABLE_TIMING_LOG) tail.validatedAt = std::chrono::steady_clock::now(); waitForFlipRoom(tail); + if (APS5_ENABLE_TIMING_LOG) tail.roomReadyAt = std::chrono::steady_clock::now(); { std::lock_guard lock(mutex); rethrowFailure(); @@ -169,10 +189,13 @@ void Driver::executeRewindTail(const Submission& stalled) { tail.serial = stalled.serial; tail.received = ++eventSerial; } + if (APS5_ENABLE_TIMING_LOG) tail.enqueuedAt = tail.dequeuedAt = tail.orderedAt = std::chrono::steady_clock::now(); + timing.Finish(); execute(tail); } void Driver::Submit(const Packet* packet, std::uint32_t queue) { + const auto receivedAt = APS5_ENABLE_TIMING_LOG ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; CheckFailure(); require(queue == 0 || (queue >= 0x20 && queue < 0x58), "unsupported compute queue"); GuestMemory::CheckRange(packet, sizeof(Packet), alignof(Packet)); @@ -180,6 +203,7 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) { require(descriptor.flags == 0, "nonzero submission flags are not implemented"); Submission submission{}; submission.queue = queue; + submission.receivedAt = receivedAt; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; auto& costs = submissionCosts(queue); const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; @@ -189,8 +213,12 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) { copyCommands(submission, descriptor.addr, descriptor.dw_num); } const auto copied = profile ? std::chrono::steady_clock::now() : start; + if (APS5_ENABLE_TIMING_LOG) submission.copiedAt = std::chrono::steady_clock::now(); validate(submission, descriptor.addr); + readRegisterLists(submission); + if (APS5_ENABLE_TIMING_LOG) submission.validatedAt = std::chrono::steady_clock::now(); waitForFlipRoom(submission); + if (APS5_ENABLE_TIMING_LOG) submission.roomReadyAt = std::chrono::steady_clock::now(); static const bool trace = std::getenv("APS5_TRACE_GPU") != nullptr; if (trace) std::fprintf(stderr, "[gpu] %.1f submit queue=0x%x dwords=%zu at %p\n", TraceMs(), queue, submission.commands.size(), static_cast(descriptor.addr)); const auto validated = profile ? std::chrono::steady_clock::now() : start; @@ -211,6 +239,7 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) { costs.validateNs += static_cast(std::chrono::duration_cast(validated - copied).count()); costs.copyNs += static_cast(std::chrono::duration_cast((copied - start) + (now - validated)).count()); } + if (APS5_ENABLE_TIMING_LOG) submission.enqueuedAt = std::chrono::steady_clock::now(); enqueue(std::move(submission)); ++accepted; } @@ -218,6 +247,7 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) { } void Driver::SuspendPoint() { + const auto receivedAt = APS5_ENABLE_TIMING_LOG ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; require(!onWorkerThread(), "worker cannot suspend itself"); std::unique_lock lock(mutex); rethrowFailure(); @@ -228,6 +258,7 @@ void Driver::SuspendPoint() { boundary.suspend = true; boundary.queue = 0; + boundary.receivedAt = receivedAt; boundary.enqueuedAt = std::chrono::steady_clock::now(); enqueue(std::move(boundary)); ++accepted; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/WorkerAffinity.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/WorkerAffinity.cpp index 0615e1bc3..d9d04c893 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/WorkerAffinity.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/WorkerAffinity.cpp @@ -10,7 +10,9 @@ std::uint64_t WorkerAffinityMask() { if (std::getenv("APS5_NO_WORKER_AFFINITY") != nullptr) return std::uint64_t{0}; const auto requested = CpuTopology::MaskFromEnvironment("APS5_WORKER_AFFINITY_MASK"); if (requested != 0) return requested; +#ifndef __linux__ if (std::getenv("APS5_WORKER_AFFINITY") == nullptr) return std::uint64_t{0}; +#endif const auto& layout = CpuTopology::Get(); return layout.hybrid ? layout.performant : std::uint64_t{0}; }(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 85acc59f5..109b381a7 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -1,8 +1,15 @@ +#include "prx/libc/include/general/LogMacros.hpp" +#include "ControlFlow/GraphBuilder.hpp" +#include "ControlFlow/Structurizer.hpp" +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "CacheKey.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp" +#include "CompiledVariant.hpp" #include "Optimization/ResourceProgram.hpp" #include #include @@ -78,71 +85,475 @@ std::shared_ptr ReadRawComputeShader(std::uint64_t address throw std::runtime_error("AGC driver: raw compute program has no reachable end within mapped code or the size limit"); } -bool FailureMemo() { - static const bool memo = std::getenv("APS5_NO_FAILURE_MEMO") == nullptr; - return memo; + +struct ShaderPreparationTransaction::State { + struct Change { + std::shared_ptr destination; + PreparedShaderState prepared; + }; + static std::mutex writers; + static thread_local State* active; + std::unique_lock lock{writers}; + std::map changes; + bool failed = false; +}; + +std::mutex ShaderPreparationTransaction::State::writers; +thread_local ShaderPreparationTransaction::State* ShaderPreparationTransaction::State::active = nullptr; + +ShaderPreparationTransaction::ShaderPreparationTransaction() { + if (State::active == nullptr) { + state = std::make_unique(); + State::active = state.get(); + } + root = State::active; } -std::shared_ptr SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, std::uint64_t deviceSerial, const ShaderRecompiler::RecompileRequest& request, bool bypass, const std::string** poisoned) { - static const bool enabled = std::getenv("APS5_NO_SOURCE_HANDLE_CACHE") == nullptr && std::getenv("APS5_NO_CAPTURE_REUSE") == nullptr; - static const bool verify = std::getenv("APS5_VERIFY_SOURCE_HANDLE") != nullptr; - if (!enabled || bypass || ShaderRecompiler::DebugProbeActive() || !request.useCache) return nullptr; - std::uint64_t key = 0xcbf29ce484222325ull; - for (const auto value : {static_cast(codeOffset), deviceSerial, ShaderRecompiler::RecompileCacheKey::ContextHash(request)}) { - key ^= value; - key *= 0x100000001b3ull; +ShaderPreparationTransaction::~ShaderPreparationTransaction() { + if (!committed) root->failed = true; + if (state != nullptr) State::active = nullptr; +} + +PreparedShaderState& ShaderPreparationTransaction::Edit(const ShaderSnapshot& snapshot) { + require(!committed, "shader preparation transaction is already committed"); + auto& changes = root->changes; + const auto found = changes.find(snapshot.prepared.get()); + if (found != changes.end()) return found->second.prepared; + std::lock_guard lock(snapshot.prepared->mutex); + return changes.emplace(snapshot.prepared.get(), State::Change{snapshot.prepared, *snapshot.prepared}).first->second.prepared; +} + +const PreparedShaderState& ShaderPreparationTransaction::Read(const ShaderSnapshot& snapshot) const { + require(!committed, "shader preparation transaction is already committed"); + const auto found = root->changes.find(snapshot.prepared.get()); + return found != root->changes.end() ? found->second.prepared : *snapshot.prepared; +} + +void ShaderPreparationTransaction::Commit() { + require(!committed && !root->failed, "shader preparation transaction was aborted"); + if (state != nullptr) { + std::vector> locks; + locks.reserve(state->changes.size()); + for (const auto& [key, change] : state->changes) locks.emplace_back(change.destination->mutex); + for (auto& [key, change] : state->changes) { + auto& destination = *change.destination; + destination.entries.swap(change.prepared.entries); + destination.registeredAbis.swap(change.prepared.registeredAbis); + destination.graphicsAbis.swap(change.prepared.graphicsAbis); + destination.rectangles.swap(change.prepared.rectangles); + destination.rectangleProgress.swap(change.prepared.rectangleProgress); + destination.fragments.swap(change.prepared.fragments); + std::swap(destination.rectangleRequested, change.prepared.rectangleRequested); + } } - if (request.graphics.has_value()) { - for (const auto& linked : request.graphics->linkedPrograms) { - for (const auto value : {static_cast(linked.role), linked.binary.codeAddress}) { - key ^= value; - key *= 0x100000001b3ull; + committed = true; +} + +void PublishRegisteredShader(std::shared_ptr& registry, const std::shared_ptr& snapshot) { + ShaderPreparationTransaction transaction; + if (registry != nullptr) { + const auto found = registry->find(snapshot->codeAddress); + if (found != registry->end()) { + const auto& current = *found->second; + if (current.headerAddress == snapshot->headerAddress && current.type == snapshot->type && current.code == snapshot->code && current.header == snapshot->header) { + transaction.Commit(); + return; } } } - auto& memos = *snapshot.handles; + auto next = registry != nullptr ? std::make_shared(*registry) : std::make_shared(); + next->insert_or_assign(snapshot->codeAddress, snapshot); + registry = std::move(next); + transaction.Commit(); +} + +std::shared_ptr SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request) { + struct PreparedKeyStorage {}; + auto& key = HostThreadLocal, PreparedKeyStorage>(); + ShaderRecompiler::BuildPreparedShaderKey(request, key); + std::lock_guard lock(snapshot.prepared->mutex); + for (const auto& entry : snapshot.prepared->entries) { + if (entry.codeOffset == codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) return entry.handle; + } + if (snapshot.header.empty()) { + if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader"); + APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast(snapshot.codeAddress)); + auto handle = ShaderRecompiler::PrepareShader(request); + snapshot.prepared->entries.push_back({codeOffset, handle}); + return handle; + } + std::string layouts; + for (const auto& entry : snapshot.prepared->entries) { + if (entry.codeOffset != codeOffset || entry.handle == nullptr || entry.handle->artifact == nullptr) continue; + const auto& layout = entry.handle->artifact->layout; + layouts += " [" + std::to_string(layout.pushConstantOffsetBytes) + "," + std::to_string(layout.pushConstantSizeBytes) + "]"; + } + throw std::runtime_error("AGC driver: prepared shader artifact is missing for the requested static ABI: address=" + std::to_string(request.shader.codeAddress) + " stage=" + std::to_string(static_cast(request.shader.stage)) + " wave=" + std::to_string(request.context.waveSize) + " pushOffset=" + std::to_string(request.layout.pushConstantOffsetBytes) + " pushCapacity=" + std::to_string(request.layout.pushConstantSizeBytes) + " preparedLayouts=" + layouts); +} + +ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapshot, std::uint64_t vertexId, std::uint64_t fragmentId) { + std::lock_guard lock(snapshot.prepared->mutex); + for (const auto& entry : snapshot.prepared->rectangles) { + if (entry.vertexId == vertexId && entry.fragmentId == fragmentId) { + require(entry.shaders != nullptr, "prepared rectangle artifact is missing"); + return *entry.shaders; + } + } + throw std::runtime_error("AGC driver: prepared rectangle artifacts are missing"); +} + +ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request) { + require(codeOffset < snapshot.code.size(), "prepared shader code offset is outside the snapshot"); + const auto code = std::span(snapshot.code).subspan(codeOffset); + require(request.shader.code.data() == code.data() && request.shader.code.size() == code.size(), "prepared invocation does not refer to registered code"); + auto invocationRequest = request; + struct PreparedKeyStorage {}; + auto& key = HostThreadLocal, PreparedKeyStorage>(); + ShaderRecompiler::BuildPreparedShaderKey(request, key); + std::lock_guard lock(snapshot.prepared->mutex); + for (const auto& entry : snapshot.prepared->entries) { + if (entry.codeOffset != codeOffset) continue; + invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle); + if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) return std::move(*invocation); + } + if (snapshot.header.empty()) { + if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader"); + APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast(snapshot.codeAddress)); + auto handle = ShaderRecompiler::PrepareShader(request); + invocationRequest = request; + invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*handle); + auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, handle, key); + if (!invocation.has_value()) throw std::runtime_error("AGC driver: raw compute artifact does not match its invocation"); + snapshot.prepared->entries.push_back({codeOffset, std::move(handle)}); + return std::move(*invocation); + } + std::string layouts; + for (const auto& entry : snapshot.prepared->entries) { + if (entry.codeOffset != codeOffset || entry.handle == nullptr || entry.handle->artifact == nullptr) continue; + const auto& layout = entry.handle->artifact->layout; + layouts += " [" + std::to_string(layout.pushConstantOffsetBytes) + "," + std::to_string(layout.pushConstantSizeBytes) + "]"; + } + throw std::runtime_error("AGC driver: prepared shader artifact is missing for the requested static ABI: address=" + std::to_string(request.shader.codeAddress) + " stage=" + std::to_string(static_cast(request.shader.stage)) + " wave=" + std::to_string(request.context.waveSize) + " pushOffset=" + std::to_string(request.layout.pushConstantOffsetBytes) + " pushCapacity=" + std::to_string(request.layout.pushConstantSizeBytes) + " preparedLayouts=" + layouts); +} +std::optional RegisteredFloatMode(const ShaderSnapshot& snapshot) { + if (snapshot.registeredState == nullptr) return std::nullopt; + std::uint32_t rsrc1; + std::uint32_t fp16OverflowBit; + switch (snapshot.type) { + case 0: rsrc1 = 0x212; fp16OverflowBit = 26; break; + case 1: rsrc1 = 0x00a; fp16OverflowBit = 29; break; + case 2: case 4: case 6: rsrc1 = 0x08a; fp16OverflowBit = 31; break; + case 5: case 7: rsrc1 = 0x10a; fp16OverflowBit = 30; break; + default: return std::nullopt; + } + const auto found = snapshot.registeredState->shader.find(rsrc1); + if (found == snapshot.registeredState->shader.end()) return std::nullopt; + const auto value = found->second; + return ShaderRecompiler::ShaderFloatMode{(value >> 12u) & 0xffu, ((value >> 21u) & 1u) != 0u, ((value >> 23u) & 1u) != 0u, ((value >> fp16OverflowBit) & 1u) != 0u}; +} + +namespace { + +template +std::vector ReadHeaderArray(const ShaderSnapshot& snapshot, const TValue* pointer, std::size_t count) { + if (count == 0) return {}; + const auto address = reinterpret_cast(pointer); + if (address < snapshot.headerAddress || address - snapshot.headerAddress > snapshot.header.size()) throw std::runtime_error("AGC driver: shader metadata is outside the registered header"); + const auto offset = static_cast(address - snapshot.headerAddress); + if (count > (snapshot.header.size() - offset) / sizeof(TValue)) throw std::runtime_error("AGC driver: truncated shader metadata"); + std::vector result(count); + std::memcpy(result.data(), snapshot.header.data() + offset, count * sizeof(TValue)); + return result; +} + +Shader ReadHeader(const ShaderSnapshot& snapshot) { + Shader header; + std::memcpy(&header, snapshot.header.data(), sizeof(header)); + return header; +} + +RegisteredShaderState DecodeRegisteredState(const ShaderSnapshot& snapshot) { + const auto header = ReadHeader(snapshot); + RegisteredShaderState state{{}, InitialContextRegisters(), {{0x24a, 0}, {0x24b, 0}}}; + for (const auto reg : ReadHeaderArray(snapshot, header.sh_registers, header.num_sh_registers)) { + state.shader.insert_or_assign(reg.offset, reg.value); + } + for (const auto reg : ReadHeaderArray(snapshot, header.cx_registers, header.num_cx_registers)) { + state.context.insert_or_assign(reg.offset, reg.value); + } + if (header.specials != nullptr) { + const auto special = ReadHeaderArray(snapshot, header.specials, 1).front(); + state.context[special.vgt_shader_stages_en.offset] = special.vgt_shader_stages_en.value; + state.context[special.vgt_gs_out_prim_type.offset] = special.vgt_gs_out_prim_type.value; + state.userConfig[special.ge_cntl.offset] = special.ge_cntl.value; + state.userConfig[special.ge_user_vgpr_en.offset] = special.ge_user_vgpr_en.value; + } + return state; +} + +std::uint32_t RegisterValue(const Registers& registers, std::uint32_t offset) { + const auto found = registers.find(offset); + if (found == registers.end()) throw std::runtime_error("AGC driver: missing static shader ABI register " + std::to_string(offset)); + return found->second; +} + +void BuildRegisteredAbiKey(const QueueState& state, const VulkanDevice& device, std::vector& key) { + key.clear(); + key.reserve(7u + state.shader.size() + state.context.size() + state.userConfig.size()); + key.insert(key.end(), {device.Serial(), ShaderRecompiler::DebugProbeActive(), ShaderRecompiler::RayTracingStrict(), ShaderRecompiler::RayTracingMiss()}); + for (const auto* registers : {&state.shader, &state.context, &state.userConfig}) { + key.push_back(registers->size()); + for (const auto& [offset, value] : *registers) key.push_back((static_cast(offset) << 32u) | value); + } +} + +std::vector PrepareRegistered(const ShaderSnapshot& snapshot, const VulkanDevice& device, const QueueState& state, bool registration) { + using Stage = ShaderRecompiler::ShaderStage; + const auto header = ReadHeader(snapshot); + std::uint32_t programRegister; + std::uint32_t resourceRegister; + std::uint32_t firstUser = 0; + Stage stage; + switch (snapshot.type) { + case 0: stage = Stage::Compute; programRegister = 0x20c; resourceRegister = 0x213; break; + case 1: stage = Stage::Fragment; programRegister = 0x008; resourceRegister = 0x00b; break; + case 2: stage = Stage::Vertex; programRegister = 0x0c8; resourceRegister = 0x08b; firstUser = 8; break; + case 4: stage = Stage::Mesh; programRegister = 0x0c8; resourceRegister = 0x08b; break; + case 5: stage = Stage::Local; programRegister = 0x148; resourceRegister = 0x10b; firstUser = 8; break; + case 6: stage = Stage::Mesh; programRegister = 0x088; resourceRegister = 0x08b; break; + case 7: stage = Stage::TessellationControl; programRegister = 0x108; resourceRegister = 0x10b; break; + default: throw std::runtime_error("AGC driver: unsupported registered shader type"); + } + const auto high = RegisterValue(state.shader, programRegister + 1); + if ((high & ~0xffu) != 0) throw std::runtime_error("AGC driver: invalid registered program address"); + const auto address = (static_cast(RegisterValue(state.shader, programRegister)) << 8u) | (static_cast(high) << 40u); + if (address < snapshot.codeAddress || address - snapshot.codeAddress >= snapshot.code.size() * 4u) throw std::runtime_error("AGC driver: registered entry point is outside shader code"); + const auto codeOffset = static_cast((address - snapshot.codeAddress) / 4u); + const auto code = std::span(snapshot.code).subspan(codeOffset); + const auto decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code); + std::optional compute; + std::optional pixel; + std::optional vertex; + std::optional graphics; + std::uint32_t wave; + const auto resources = RegisterValue(state.shader, resourceRegister); + auto userCount = (resources >> 1u) & 0x1fu; + if (stage != Stage::Compute) userCount |= ((resources >> 27u) & 1u) << 5u; + if (userCount > 32) throw std::runtime_error("AGC driver: static user SGPR count exceeds the register bank"); + if (stage == Stage::Compute) { + compute = Graphics::DecodeComputeStageInfo(state.shader, snapshot.header); + const auto special = ReadHeaderArray(snapshot, header.specials, 1).front(); + wave = (special.dispatch_modifier & 0x8000u) != 0 ? 32u : 64u; + } else if (stage == Stage::Fragment) { + const auto count = RegisterValue(state.context, 0x1b6) & 0x3fu; + if (registration && count != 0) return {}; + pixel = Graphics::DecodePixelStageInfo(state.context, {}); + wave = pixel->wave32 ? 32u : 64u; + } else { + const auto routing = RegisterValue(state.context, 0x2d5); + wave = (routing & 0x00400000u) != 0 ? 32u : 64u; + if ((routing & 0x20u) != 0 || stage == Stage::Mesh || (routing & 4u) != 0) { + if (registration) return {}; + auto stageState = state; + stageState.context[0x1b6] = 0; + const auto stages = Graphics::DecodeShaderStages(stageState); + graphics = ShaderRecompiler::GraphicsCompileContext{0, {}, stages.mesh, stages.tessellation, {}}; + if (stages.mesh) { stage = Stage::Mesh; firstUser = 0; } + if (stages.tessellation && snapshot.type == 2) stage = Stage::TessellationEvaluation; + } + if (firstUser == 0) userCount += 8; + } + std::vector userData(userCount); + if (stage != Stage::Compute && stage != Stage::Fragment && snapshot.type != 6) vertex = Graphics::DecodeVertexStageInfo(snapshot.header, snapshot.headerAddress, userData, nullptr, true); + const ShaderRecompiler::SwappcInfo swappc{vertex.has_value(), firstUser, userCount}; + auto graph = ShaderRecompiler::GraphBuilder{}.Build(decoded, &swappc); + ShaderRecompiler::Structurizer{}.Structurize(graph); + const std::array memory{{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}}; + ShaderRecompiler::RecompileRequest request{{stage, address, code, snapshot.headerAddress, snapshot.header}, {wave, firstUser, userData, compute, pixel, vertex, memory, RegisteredFloatMode(snapshot)}, stage == Stage::Compute ? device.ComputeTarget(wave) : device.Target(), {0, 0, 0, 128}, graphics}; + if (graphics && graphics->mesh) request.layout.pushConstantSizeBytes = ShaderRecompiler::MeshDrawPushOffsetBytes; + std::vector entries; + const auto append = [&] { + PerformanceTimer timing("Shader.PrepareArtifact"); + entries.push_back({codeOffset, ShaderRecompiler::PrepareShader(request)}); + }; + append(); + if (compute) { + request.context.compute->partialThreads = {1, 1, 1}; + append(); + } else if (pixel) { + request.layout.pushConstantSizeBytes = 0; + append(); + } + return entries; +} + +} + +std::vector PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target) { + ShaderPreparationTransaction transaction; + require(decoded.programs.size() == decoded.roles.size(), "graphics ABI program roles are incomplete"); + for (const auto& program : decoded.programs) require(program.snapshot != nullptr, "graphics ABI has no registered shader snapshot"); + std::vector expected; + if (decoded.state.stages.path == Graphics::ShaderPath::Tessellation) expected = {ShaderRecompiler::ProgramRole::Local, ShaderRecompiler::ProgramRole::Hull, ShaderRecompiler::ProgramRole::Domain}; + else { + expected.push_back(ShaderRecompiler::ProgramRole::Main); + if (decoded.state.stages.path == Graphics::ShaderPath::Geometry && !decoded.programs.empty() && decoded.programs.front().snapshot->type == 4) expected.push_back(ShaderRecompiler::ProgramRole::GeometryBack); + } + if (!decoded.roles.empty() && decoded.roles.back() == ShaderRecompiler::ProgramRole::Fragment) expected.push_back(ShaderRecompiler::ProgramRole::Fragment); + require(expected == decoded.roles, "graphics ABI linked programs do not match the stage routing"); + std::vector linked; + std::vector memory; + for (std::size_t index = 0; index < decoded.programs.size(); ++index) { + const auto& program = decoded.programs[index]; + require(program.snapshot != nullptr && program.codeOffset < program.snapshot->code.size(), "graphics ABI has an invalid registered entry point"); + const auto code = std::span(program.snapshot->code).subspan(program.codeOffset); + require(program.binary.code.data() == code.data() && program.binary.code.size() == code.size() && program.binary.codeAddress == program.snapshot->codeAddress + program.codeOffset * sizeof(std::uint32_t), "graphics ABI entry point differs from its snapshot"); + linked.push_back({decoded.roles[index], program.binary, program.userDataBase, program.firstUserSgpr, program.userData}); + memory.insert(memory.end(), program.memory.begin(), program.memory.end()); + } + std::vector prepared; + std::uint32_t pushOffset = 0; + const auto capacity = decoded.state.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : Graphics::PipelinePushConstantBytes; + for (std::size_t index = 0; index < decoded.programs.size(); ++index) { + if (decoded.roles[index] == ShaderRecompiler::ProgramRole::GeometryBack) continue; + const auto& program = decoded.programs[index]; + const bool fragment = program.binary.stage == ShaderRecompiler::ShaderStage::Fragment; + const auto wave = fragment ? decoded.state.stages.fragmentWaveSize : decoded.state.stages.vertexWaveSize; + std::optional vertex; + if (!fragment) vertex = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, nullptr, true); + ShaderRecompiler::RecompileRequest request{program.binary, {wave, program.firstUserSgpr, program.userData, {}, fragment ? std::optional(decoded.pixel) : std::nullopt, vertex, memory, RegisteredFloatMode(*program.snapshot)}, target, {0, 0, pushOffset, capacity - pushOffset}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, decoded.state.stages.mesh, decoded.state.stages.tessellation, {}}}; + std::vector key; + ShaderRecompiler::BuildPreparedShaderKey(request, key); + std::shared_ptr handle; + for (const auto& entry : transaction.Read(*program.snapshot).entries) { + if (entry.codeOffset == program.codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) { + handle = entry.handle; + break; + } + } + if (handle == nullptr) { + PerformanceTimer timing("Shader.PrepareArtifact"); + handle = ShaderRecompiler::PrepareShader(request); + } + const auto bytes = handle->artifact->bindings.pushConstantSizeBytes; + require(bytes <= capacity - pushOffset, "prepared graphics stages exceed the push constant block"); + prepared.push_back({program.snapshot, {program.codeOffset, handle}}); + pushOffset += bytes; + } + transaction.Commit(); + return prepared; +} + +void Driver::ResolveGraphicsStagesAbi(std::span stages, std::span context, std::span primitive) { + PerformanceContext timingContext(FrameTiming::Preparation()); + PerformanceTimer timing("Shader.ResolveGraphicsStagesAbi"); + ShaderPreparationTransaction transaction; + CheckFailure(); + require(!stages.empty(), "graphics ABI has no shader headers"); + QueueState state{}; + std::shared_ptr registry; + std::shared_ptr owner; { - std::lock_guard lock(memos.mutex); - for (const auto& entry : memos.entries) { - if (entry.key != key || (entry.handle == nullptr && entry.failure == nullptr)) continue; - if (entry.handle == nullptr) { - if (verify) { - bool resolved = true; - try { - static_cast(ShaderRecompiler::ResolveSource(request)); - } catch (const std::exception&) { - resolved = false; - } - if (resolved) throw std::runtime_error("AGC driver: source handle memo poisoned but the source resolved"); - } - ShaderMemory::CountHandleMemo(true); - if (poisoned == nullptr) throw std::runtime_error(*entry.failure); - *poisoned = entry.failure.get(); - return nullptr; - } - if (verify && ShaderRecompiler::ResolveSource(request)->source != entry.handle->source) throw std::runtime_error("AGC driver: source handle memo answered a different source"); - ShaderMemory::CountHandleMemo(true); - return entry.handle; + std::lock_guard lock(mutex); + registry = shaders; + for (const auto* shader : stages) { + GuestMemory::CheckRange(shader, sizeof(Shader), alignof(Shader)); + const auto address = reinterpret_cast(const_cast(shader->code)); + require(registry != nullptr && registry->contains(address), "graphics ABI refers to an unregistered shader"); + if (owner == nullptr) owner = registry->at(address); + const auto& snapshot = *registry->at(address); + require(snapshot.headerAddress == reinterpret_cast(shader), "graphics ABI refers to a replaced shader header"); + require(snapshot.registeredState != nullptr, "registered shader state is missing"); + const auto& registered = *snapshot.registeredState; + for (const auto& [offset, value] : registered.shader) state.shader.insert_or_assign(offset, value); + for (const auto& [offset, value] : registered.context) state.context.insert_or_assign(offset, value); + for (const auto& [offset, value] : registered.userConfig) state.userConfig.insert_or_assign(offset, value); } } - std::shared_ptr handle; - try { - handle = ShaderRecompiler::ResolveSource(request); - } catch (const std::exception& error) { - if (FailureMemo()) { - std::lock_guard lock(memos.mutex); - memos.entries[memos.next] = {key, nullptr, std::make_shared(error.what())}; - memos.next = (memos.next + 1) % memos.entries.size(); - memos.poisoned.fetch_add(1, std::memory_order_relaxed); + for (const auto reg : context) state.context.insert_or_assign(reg.offset, reg.value); + for (const auto reg : primitive) state.userConfig.insert_or_assign(reg.offset, reg.value); + const auto localDevice = device.Load(); + require(localDevice != nullptr, "graphics ABI device is missing"); + struct GraphicsAbiKeyStorage {}; + auto& key = HostThreadLocal, GraphicsAbiKeyStorage>(); + BuildRegisteredAbiKey(state, *localDevice, key); + const auto primitiveType = state.userConfig.find(0x242u); + for (const auto& abi : transaction.Read(*owner).graphicsAbis) { + if (abi.key != key || abi.registry.lock() != registry) continue; + for (const auto& stage : abi.stages) { + const auto snapshot = stage.lock(); + require(snapshot != nullptr, "prepared graphics ABI lost a registered stage"); + if (snapshot->type != 1) ResolvePreparedGraphics(*snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target()); } - throw; + transaction.Commit(); + return; } - ShaderMemory::CountHandleMemo(false); - if (handle == nullptr) return nullptr; - std::lock_guard lock(memos.mutex); - memos.entries[memos.next] = {key, handle, nullptr}; - memos.next = (memos.next + 1) % memos.entries.size(); - return handle; + DrawDecode decoded{}; + decoded.state.stages = Graphics::DecodeShaderStages(state); + DecodeGraphicsPrograms(decoded, state, *registry, true, false); + auto prepared = PrepareGraphicsStages(decoded, localDevice->Target()); + for (auto& stage : prepared) { + const auto& entries = transaction.Read(*stage.snapshot).entries; + const auto duplicate = std::ranges::any_of(entries, [&](const auto& existing) { return existing.codeOffset == stage.entry.codeOffset && existing.handle->artifact == stage.entry.handle->artifact; }); + if (!duplicate) transaction.Edit(*stage.snapshot).entries.push_back(std::move(stage.entry)); + } + for (const auto& stage : prepared) { + if (stage.snapshot->type == 1) continue; + ResolvePreparedGraphics(*stage.snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target()); + } + PreparedShaderState::GraphicsAbi abi{key, registry, {}}; + for (const auto& stage : prepared) abi.stages.push_back(stage.snapshot); + auto& graphicsAbis = transaction.Edit(*owner).graphicsAbis; + std::erase_if(graphicsAbis, [](const auto& entry) { return entry.registry.expired(); }); + graphicsAbis.push_back(std::move(abi)); + transaction.Commit(); +} + +void Driver::ResolveShaderAbi(const Shader* shader, std::span context, std::span primitive) { + PerformanceContext timingContext(FrameTiming::Preparation()); + PerformanceTimer timing("Shader.ResolveAbi"); + ShaderPreparationTransaction transaction; + CheckFailure(); + GuestMemory::CheckRange(shader, sizeof(Shader), alignof(Shader)); + if (shader->type != 0 && shader->type != 1) { + const std::array stages{shader}; + ResolveGraphicsStagesAbi(stages, context, primitive); + transaction.Commit(); + return; + } + std::shared_ptr snapshot; + { + std::lock_guard lock(mutex); + const auto address = reinterpret_cast(const_cast(shader->code)); + require(shaders != nullptr && shaders->contains(address), "static ABI refers to an unregistered shader"); + snapshot = shaders->at(address); + require(snapshot->headerAddress == reinterpret_cast(shader), "static ABI refers to a replaced shader header"); + } + require(snapshot->registeredState != nullptr, "registered shader state is missing"); + QueueState state{}; + state.shader = snapshot->registeredState->shader; + state.context = snapshot->registeredState->context; + state.userConfig = snapshot->registeredState->userConfig; + for (const auto reg : context) state.context[reg.offset] = reg.value; + for (const auto reg : primitive) state.userConfig[reg.offset] = reg.value; + const auto localDevice = device.Load(); + require(localDevice != nullptr, "shader registration device is missing"); + struct RegisteredAbiKeyStorage {}; + auto& key = HostThreadLocal, RegisteredAbiKeyStorage>(); + BuildRegisteredAbiKey(state, *localDevice, key); + const auto& registeredAbis = transaction.Read(*snapshot).registeredAbis; + if (std::ranges::find(registeredAbis, key) != registeredAbis.end()) { + transaction.Commit(); + return; + } + auto entries = PrepareRegistered(*snapshot, *localDevice, state, false); + auto& prepared = transaction.Edit(*snapshot); + for (auto& entry : entries) { + const auto duplicate = std::ranges::any_of(prepared.entries, [&](const auto& existing) { return existing.handle->artifact == entry.handle->artifact; }); + if (!duplicate) prepared.entries.push_back(std::move(entry)); + } + prepared.registeredAbis.push_back(key); + transaction.Commit(); } alignas(256) static const std::uint32_t NullPixelCode[64] = {0xbf810000u}; @@ -161,7 +572,97 @@ std::uint64_t NullPixelProgramAddress() { return reinterpret_cast(NullPixelCode); } +void Driver::ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType) { + PerformanceContext timingContext(FrameTiming::Preparation()); + PerformanceTimer timing("Shader.ResolveGraphicsAbi"); + CheckFailure(); + if (primitiveType != 0 && primitiveType != 7 && primitiveType != 17) return; + ShaderPreparationTransaction transaction; + require(vertex != nullptr && pixel != nullptr, "rectangle ABI requires vertex and fragment shaders"); + std::shared_ptr front; + std::shared_ptr fragment; + { + std::lock_guard lock(mutex); + GuestMemory::CheckRange(vertex, sizeof(Shader), alignof(Shader)); + GuestMemory::CheckRange(pixel, sizeof(Shader), alignof(Shader)); + const auto lookup = [&](const Shader* shader) { + const auto address = reinterpret_cast(const_cast(shader->code)); + require(shaders != nullptr && shaders->contains(address), "rectangle ABI refers to an unregistered shader"); + const auto snapshot = shaders->at(address); + require(snapshot->headerAddress == reinterpret_cast(shader), "rectangle ABI refers to a replaced shader header"); + return snapshot; + }; + front = lookup(vertex); + fragment = lookup(pixel); + } + require(front != fragment, "rectangle stages refer to the same shader"); + const auto localDevice = device.Load(); + require(localDevice != nullptr, "shader registration device is missing"); + ResolvePreparedGraphics(*front, fragment, primitiveType, localDevice->Target()); + transaction.Commit(); +} + +void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target) { + ShaderPreparationTransaction transaction; + require(fragment.get() != &front, "rectangle stages refer to the same shader"); + std::vector> fragments; + { + const auto& current = transaction.Read(front); + const bool newFragment = fragment && !std::ranges::any_of(current.fragments, [&](const auto& entry) { return entry.lock() == fragment; }); + const bool newRequest = !current.rectangleRequested && (primitiveType == 7 || primitiveType == 17); + const bool expiredFragment = std::ranges::any_of(current.fragments, [](const auto& entry) { return entry.expired(); }); + if (newFragment || newRequest || expiredFragment) { + auto& prepared = transaction.Edit(front); + std::erase_if(prepared.fragments, [](const auto& entry) { return entry.expired(); }); + std::erase_if(prepared.rectangleProgress, [](const auto& entry) { return entry.fragment.expired(); }); + if (newFragment) prepared.fragments.push_back(fragment); + prepared.rectangleRequested |= newRequest; + } + const auto& prepared = transaction.Read(front); + if (!prepared.rectangleRequested) { + transaction.Commit(); + return; + } + for (const auto& entry : prepared.fragments) { + if (auto snapshot = entry.lock()) fragments.push_back(std::move(snapshot)); + } + } + for (const auto& pixel : fragments) { + const auto& current = transaction.Read(front); + const auto& pixelPrepared = transaction.Read(*pixel); + require(!current.entries.empty() && !pixelPrepared.entries.empty(), "rectangle ABI has missing stage artifacts"); + const auto progress = std::ranges::find_if(current.rectangleProgress, [&](const auto& entry) { return entry.fragment.lock() == pixel; }); + const auto vertexCount = progress == current.rectangleProgress.end() ? 0u : progress->vertexCount; + const auto fragmentCount = progress == current.rectangleProgress.end() ? 0u : progress->fragmentCount; + if (vertexCount == current.entries.size() && fragmentCount == pixelPrepared.entries.size()) continue; + auto& prepared = transaction.Edit(front); + std::vector rectangles; + for (std::size_t vertexIndex = 0; vertexIndex < prepared.entries.size(); ++vertexIndex) { + const auto& vertexEntry = prepared.entries[vertexIndex]; + for (std::size_t fragmentIndex = vertexIndex < vertexCount ? fragmentCount : 0u; fragmentIndex < pixelPrepared.entries.size(); ++fragmentIndex) { + const auto& fragmentEntry = pixelPrepared.entries[fragmentIndex]; + const auto& vertexArtifact = ShaderRecompiler::GetPreparedArtifact(*vertexEntry.handle); + const auto& fragmentArtifact = ShaderRecompiler::GetPreparedArtifact(*fragmentEntry.handle); + const auto exists = [&](const auto& entry) { return entry.vertexId == vertexArtifact.variantId && entry.fragmentId == fragmentArtifact.variantId; }; + if (std::ranges::any_of(prepared.rectangles, exists) || std::ranges::any_of(rectangles, exists)) continue; + ShaderRecompiler::RecompileResult vertexResult; + ShaderRecompiler::RecompileResult fragmentResult; + static_cast(vertexResult) = vertexArtifact; + static_cast(fragmentResult) = fragmentArtifact; + rectangles.push_back({vertexArtifact.variantId, fragmentArtifact.variantId, std::make_shared(ShaderRecompiler::BuildRectListShaders(vertexResult, fragmentResult, target))}); + } + } + prepared.rectangles.insert(prepared.rectangles.end(), std::make_move_iterator(rectangles.begin()), std::make_move_iterator(rectangles.end())); + std::erase_if(prepared.rectangleProgress, [&](const auto& entry) { return entry.fragment.expired() || entry.fragment.lock() == pixel; }); + prepared.rectangleProgress.push_back({pixel, prepared.entries.size(), pixelPrepared.entries.size()}); + } + transaction.Commit(); +} + void Driver::RegisterShader(const Shader* shader) { + PerformanceContext timingContext(FrameTiming::Preparation()); + PerformanceTimer timing("Shader.Register"); + ShaderPreparationTransaction transaction; CheckFailure(); GuestMemory::CheckRange(shader, sizeof(Shader), 1); Shader fields; @@ -178,6 +679,38 @@ void Driver::RegisterShader(const Shader* shader) { snapshot.header.resize(fields.header_size); std::memcpy(snapshot.header.data(), static_cast(shader), fields.header_size); + { + std::lock_guard lock(mutex); + rethrowFailure(); + if (shaders != nullptr) { + const auto found = shaders->find(snapshot.codeAddress); + if (found != shaders->end()) { + const auto& current = *found->second; + if (current.headerAddress == snapshot.headerAddress && current.type == snapshot.type && current.code == snapshot.code && current.header == snapshot.header) { + transaction.Commit(); + return; + } + } + } + } + + std::shared_ptr localDevice = device.Load(); + if (localDevice == nullptr) { + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + if (device == nullptr) device = std::make_shared(); + localDevice = device; + } + snapshot.registeredState = std::make_shared(DecodeRegisteredState(snapshot)); + QueueState registered{}; + registered.shader = snapshot.registeredState->shader; + registered.context = snapshot.registeredState->context; + registered.userConfig = snapshot.registeredState->userConfig; + snapshot.prepared->entries = PrepareRegistered(snapshot, *localDevice, registered, true); + if ((snapshot.type == 0 || snapshot.type == 1) && !snapshot.prepared->entries.empty()) { + std::vector key; + BuildRegisteredAbiKey(registered, *localDevice, key); + snapshot.prepared->registeredAbis.push_back(std::move(key)); + } static const char* traceRegs = std::getenv("APS5_TRACE_SHADER_REGS"); if (traceRegs != nullptr && (std::string(traceRegs) == "all" || std::strtoull(traceRegs, nullptr, 16) == snapshot.codeAddress)) { const auto print = [](const ShaderRegister* registers, std::uint32_t count) { @@ -195,18 +728,39 @@ void Driver::RegisterShader(const Shader* shader) { } std::lock_guard lock(mutex); rethrowFailure(); - const auto address = snapshot.codeAddress; - - if (shaders == nullptr) shaders = std::make_shared(); - else if (shaders.use_count() != 1) shaders = std::make_shared(*shaders); - shaders->insert_or_assign(address, std::make_shared(std::move(snapshot))); + PublishRegisteredShader(shaders, std::make_shared(std::move(snapshot))); if (shaders->find(NullPixelProgramAddress()) == shaders->end()) { ShaderSnapshot null{NullPixelProgramAddress(), reinterpret_cast(&NullPixelShader), NullPixelShader.type, {}, {}}; null.code.assign(std::begin(NullPixelCode), std::end(NullPixelCode)); null.header.resize(sizeof(Shader)); std::memcpy(null.header.data(), &NullPixelShader, sizeof(Shader)); - shaders->insert_or_assign(NullPixelProgramAddress(), std::make_shared(std::move(null))); + auto nullRegisteredState = DecodeRegisteredState(null); + nullRegisteredState.shader.emplace(0x008u, static_cast(null.codeAddress >> 8u)); + nullRegisteredState.shader.emplace(0x009u, static_cast(null.codeAddress >> 40u)); + nullRegisteredState.shader.emplace(0x00bu, 0u); + nullRegisteredState.context.emplace(0x1b3u, 0x2u); + nullRegisteredState.context.emplace(0x1b4u, 0x2u); + null.registeredState = std::make_shared(std::move(nullRegisteredState)); + QueueState nullState{}; + nullState.shader = null.registeredState->shader; + nullState.context = null.registeredState->context; + nullState.userConfig = null.registeredState->userConfig; + null.prepared->entries = PrepareRegistered(null, *localDevice, nullState, true); + PublishRegisteredShader(shaders, std::make_shared(std::move(null))); } + transaction.Commit(); } } + +extern "C" AgcDriver::DriverDetail::ShaderPreparationTransaction* AgcDriverBeginShaderPreparation_nid_postfix() { + return new AgcDriver::DriverDetail::ShaderPreparationTransaction(); +} + +extern "C" void AgcDriverCommitShaderPreparation_nid_postfix(AgcDriver::DriverDetail::ShaderPreparationTransaction* transaction) { + transaction->Commit(); +} + +extern "C" void AgcDriverEndShaderPreparation_nid_postfix(AgcDriver::DriverDetail::ShaderPreparationTransaction* transaction) noexcept { + delete transaction; +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp index 72a41535e..9a6ee823d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp @@ -5,6 +5,8 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" #include +#include +#include #ifdef _WIN32 #include #endif @@ -287,10 +289,14 @@ void Driver::waitMemory(std::span packet, std::uint32_t que if (!warned && std::chrono::steady_clock::now() - start > std::chrono::milliseconds(waitTimeoutMs())) { warned = true; ++outcomes.timedOut; - static std::set reported; - static std::uint64_t timeouts = 0; - if (++timeouts % 20 == 0) std::fprintf(stderr, "[gpu] %llu GPU waits have timed out\n", static_cast(timeouts)); - if (!reported.insert(packet[2] | (static_cast(packet[3]) << 32u)).second) return; + { + static std::mutex reportedMutex; + static std::set reported; + static std::uint64_t timeouts = 0; + const std::lock_guard lock(reportedMutex); + if (++timeouts % 20 == 0) std::fprintf(stderr, "[gpu] %llu GPU waits have timed out\n", static_cast(timeouts)); + if (!reported.insert(packet[2] | (static_cast(packet[3]) << 32u)).second) return; + } const bool wide = ((packet[0] >> 8u) & 0xffu) == 0x93u; const std::uint64_t reference = wide ? packet[4] | (static_cast(packet[5]) << 32u) : packet[4]; const std::uint64_t mask = wide ? packet[6] | (static_cast(packet[7]) << 32u) : packet[5]; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index a9427ddc0..b944d6438 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/WriteWatchCoverage.hpp" #include "ThreadOwned.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestArena.hpp" @@ -26,9 +27,43 @@ #endif #include #else +#include +#include +#include #include #include #include +#include +#include +#if __has_include() +#include +#endif +#if !defined(PROCMAP_QUERY) +struct procmap_query { + std::uint64_t size; + std::uint64_t query_flags; + std::uint64_t query_addr; + std::uint64_t vma_start; + std::uint64_t vma_end; + std::uint64_t vma_flags; + std::uint64_t vma_page_size; + std::uint64_t vma_offset; + std::uint64_t inode; + std::uint32_t dev_major; + std::uint32_t dev_minor; + std::uint32_t vma_name_size; + std::uint32_t build_id_size; + std::uint64_t vma_name_addr; + std::uint64_t build_id_addr; +}; +static_assert(sizeof(procmap_query) == 104); +enum : std::uint64_t { + PROCMAP_QUERY_VMA_READABLE = 0x01, + PROCMAP_QUERY_VMA_WRITABLE = 0x02, + PROCMAP_QUERY_COVERING_OR_NEXT_VMA = 0x10, +}; +#define PROCMAP_QUERY _IOWR('f', 17, struct procmap_query) +#endif #endif namespace AgcDriver::GuestMemory { @@ -478,6 +513,23 @@ struct PageRun { bool writable; }; +#if !defined(_WIN32) +int ProcMapsQueryFd() { + static const int fd = [] { + if (std::getenv("APS5_NO_PROCMAP_QUERY") != nullptr) return -1; + const int opened = open("/proc/self/maps", O_RDONLY | O_CLOEXEC); + if (opened < 0) return -1; + procmap_query probe{}; + probe.size = sizeof(probe); + probe.query_flags = PROCMAP_QUERY_COVERING_OR_NEXT_VMA; + if (ioctl(opened, PROCMAP_QUERY, &probe) == 0 || errno == ENOENT) return opened; + close(opened); + return -1; + }(); + return fd; +} +#endif + // Calls `emit` with consecutive runs of uniform accessibility covering [address, address + bytes) in // order, stopping early when it returns false. Returns false when the address space cannot be queried. template @@ -535,6 +587,31 @@ bool describePages(std::uintptr_t address, std::size_t bytes, Emit&& emit) { if (!emit(PageRun{cursor, next, readable, writable})) return true; cursor = next; #else + if (const int fd = ProcMapsQueryFd(); fd >= 0) { + procmap_query query{}; + query.size = sizeof(query); + query.query_flags = PROCMAP_QUERY_COVERING_OR_NEXT_VMA; + query.query_addr = cursor; + if (ioctl(fd, PROCMAP_QUERY, &query) == 0) { + if (query.vma_end <= cursor || query.vma_start >= query.vma_end) return false; + if (query.vma_start > cursor) { + const auto gapEnd = std::min(end, query.vma_start); + if (!emit(PageRun{cursor, gapEnd, false, false})) return true; + cursor = gapEnd; + continue; + } + const bool readable = (query.vma_flags & PROCMAP_QUERY_VMA_READABLE) != 0; + const auto next = std::min(end, query.vma_end); + if (!emit(PageRun{cursor, next, readable, readable && (query.vma_flags & PROCMAP_QUERY_VMA_WRITABLE) != 0})) return true; + cursor = next; + continue; + } + if (errno == ENOENT) { + static_cast(emit(PageRun{cursor, end, false, false})); + return true; + } + return false; + } std::ifstream maps("/proc/self/maps"); if (!maps.is_open()) return false; std::string line; @@ -667,9 +744,15 @@ std::vector> CommittedRanges(std::uint64 namespace { constexpr std::size_t WriteBlockBytes = 65536; +constexpr std::size_t WritePageBytes = 4096; +constexpr std::size_t WritePagesPerBlock = WriteBlockBytes / WritePageBytes; enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow }; +#ifdef _WIN32 +void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_t generation); +#endif + struct WriteTracker { std::mutex mutex; bool initialized = false; @@ -684,6 +767,7 @@ struct WriteTracker { // records in the same block (the title's per-job slots are 0x20 apart) must not count. std::vector cpuBlocks; std::vector writtenBlocks; + WriteWatchCoverage coverage; #else static constexpr std::size_t LeafBlocks = std::size_t{1} << 16; static constexpr std::size_t LeafCount = std::size_t{1} << 15; @@ -727,7 +811,9 @@ struct WriteTracker { blocks.assign(size / WriteBlockBytes + 1, 0); cpuBlocks.assign(size / WriteBlockBytes + 1, 0); writtenBlocks.assign(size / WriteBlockBytes + 1, 0); + coverage.Initialize(base, size); pages.resize(1u << 16); + GuestArena::GuestArenaSetPrivateMappingObserver_nid_postfix(&watchPrivateMapping); #else watched = GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix(); if (watched) leaves.resize(LeafCount); @@ -736,7 +822,7 @@ struct WriteTracker { bool covers(std::uint64_t address, std::size_t bytes) const { #ifdef _WIN32 - return address >= base && address - base <= size - bytes; + return coverage.Covers(address, bytes); #else return address + bytes <= LeafCount * LeafBlocks * WriteBlockBytes && GuestWriteWatch::GuestWriteWatchCovers_nid_postfix(static_cast(address), bytes); #endif @@ -771,6 +857,24 @@ struct WriteTracker { }; std::unordered_map driverPieces; + struct CpuPages { + std::array pages{}; + }; + std::unordered_map cpuPages; + + void noteCpuStore(std::uint64_t begin, std::uint64_t end, std::uint32_t stampGeneration) { + for (auto block = blockOf(begin); block <= blockOf(end - 1); ++block) { + const auto blockStart = blockBegin(block); + const auto from = std::max(begin, blockStart) - blockStart; + const auto to = std::min(end, blockStart + WriteBlockBytes) - blockStart; + auto& entry = cpuPages[block]; + for (auto page = from / WritePageBytes; page <= (to - 1) / WritePageBytes; ++page) { + auto& stamp = entry.pages[page]; + stamp = std::max(stamp, stampGeneration); + } + } + } + void noteDriverStore(std::uint64_t block, std::uint64_t address, std::uint64_t end, std::uint32_t stampGeneration) { const auto begin = blockBegin(block); const auto from = static_cast(std::max(address, begin) - begin); @@ -851,6 +955,21 @@ std::unique_lock lockTracker(WriteTracker& tracker) { return lock; } +#ifdef _WIN32 +void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_t generation) { + auto& tracker = Tracker(); + const auto lock = lockTracker(tracker); + if (!tracker.watched || bytes == 0 || bytes > tracker.size || address < tracker.base || address - tracker.base > tracker.size - bytes) return; + tracker.coverage.Restore(address, bytes, generation); + ++tracker.generation; + for (auto block = tracker.blockOf(address); block <= tracker.blockOf(address + bytes - 1); ++block) { + tracker.driverPieces.erase(block); + tracker.stamp(block, tracker.generation, StampKind::Cpu); + } + unwatchSerial.fetch_add(1, std::memory_order_release); +} +#endif + // Collect epochs are per thread and globally unique: every bump takes a fresh value from one counter, // so a memo entry (stamped with the epoch of its walk) can only match the thread that made it, and // only until that thread's next ordering point (the queue workers bump at theirs, see the header). A @@ -894,6 +1013,7 @@ void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) { auto& [tracker, kind] = *static_cast(context); if (end <= begin) return; const auto generation = tracker.generation.load(std::memory_order_relaxed); + if (kind != StampKind::Driver) tracker.noteCpuStore(begin, end, generation); for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, kind); collectDirtyRuns.fetch_add(1, std::memory_order_relaxed); } @@ -918,7 +1038,11 @@ bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, return false; } if (count != 0) dirty = true; - for (ULONG_PTR i = 0; i < count; ++i) tracker.stamp(tracker.blockOf(reinterpret_cast(tracker.pages[i])), tracker.generation, kind); + for (ULONG_PTR i = 0; i < count; ++i) { + const auto page = reinterpret_cast(tracker.pages[i]); + tracker.stamp(tracker.blockOf(page), tracker.generation, kind); + if (kind != StampKind::Driver) tracker.noteCpuStore(page, page + WritePageBytes, tracker.generation); + } if (count < tracker.pages.size()) break; cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); } @@ -930,7 +1054,11 @@ bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, std::size_t count = tracker.pages.size(); DWORD granularity = 4096; if (!GuestArena::GuestArenaCollectWrites_nid_postfix(cursor, static_cast(stop - cursor), tracker.pages.data(), &count, true)) return false; - for (ULONG_PTR i = 0; i < count; ++i) tracker.stamp(tracker.blockOf(reinterpret_cast(tracker.pages[i])), tracker.generation, kind); + for (ULONG_PTR i = 0; i < count; ++i) { + const auto page = reinterpret_cast(tracker.pages[i]); + tracker.stamp(tracker.blockOf(page), tracker.generation, kind); + if (kind != StampKind::Driver) tracker.noteCpuStore(page, page + WritePageBytes, tracker.generation); + } if (count < tracker.pages.size()) break; cursor = reinterpret_cast(tracker.pages[count - 1]) + (granularity != 0 ? granularity : page); } @@ -1011,13 +1139,16 @@ std::uint64_t CollectEpochBumps() { return collectEpochBumps.load(std::memory_order_relaxed); } +std::uint64_t CollectEpoch() { + return collectMemoEnabled() ? threadCollectEpoch : 0; +} + namespace { -void unwatchLocked(const WriteTracker& tracker, std::uint64_t address, std::size_t bytes) { +void unwatchLocked(WriteTracker& tracker, std::uint64_t address, std::size_t bytes) { if (!tracker.watched) return; #ifdef _WIN32 - static_cast(address); - static_cast(bytes); + if (tracker.coverage.Exclude(address, bytes, GuestArena::GuestArenaCommitGeneration_nid_postfix())) unwatchSerial.fetch_add(1, std::memory_order_release); #else if (GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(address), bytes)) unwatchSerial.fetch_add(1, std::memory_order_release); #endif @@ -1098,8 +1229,9 @@ bool UnchangedSinceAll(std::span queries) { std::uint64_t storeOwn(std::uint64_t address, std::size_t bytes, const std::function()>& store) { if (bytes == 0) return 0; const auto stampStored = [](WriteTracker& tracker, std::pair stored) -> std::uint64_t { - if (stored.second <= stored.first || !tracker.watched || !tracker.covers(stored.first, static_cast(stored.second - stored.first))) return 0; + if (stored.second <= stored.first) return 0; ++tracker.generation; + if (!tracker.watched || !tracker.covers(stored.first, static_cast(stored.second - stored.first))) return 0; for (auto block = tracker.blockOf(stored.first); block <= tracker.blockOf(stored.second - 1); ++block) { tracker.stamp(block, tracker.generation, StampKind::Driver); tracker.noteDriverStore(block, stored.first, stored.second, tracker.generation); @@ -1133,10 +1265,11 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) { auto& tracker = Tracker(); const auto lock = lockTracker(tracker); tracker.initialize(); - if (!tracker.watched || bytes == 0 || !tracker.covers(address, bytes)) return 0; + if (bytes == 0) return 0; + ++tracker.generation; + if (!tracker.watched || !tracker.covers(address, bytes)) return 0; const auto first = tracker.blockOf(address); const auto last = tracker.blockOf(address + bytes - 1); - ++tracker.generation; for (auto block = first; block <= last; ++block) { tracker.stamp(block, tracker.generation, StampKind::Driver); tracker.noteDriverStore(block, address, address + bytes, tracker.generation); @@ -1154,7 +1287,16 @@ bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generati const auto last = tracker.blockOf(end - 1); for (auto block = first; block <= last; ++block) { if (tracker.writtenStampOf(block) <= generation) continue; - if (tracker.cpuStampOf(block) > generation) return true; + if (tracker.cpuStampOf(block) > generation) { + const auto begin = tracker.blockBegin(block); + const auto from = std::max(address, begin) - begin; + const auto to = std::min(end, begin + WriteBlockBytes) - begin; + const auto found = tracker.cpuPages.find(block); + if (found == tracker.cpuPages.end()) return true; + for (auto page = from / WritePageBytes; page <= (to - 1) / WritePageBytes; ++page) { + if (found->second.pages[page] > generation) return true; + } + } const auto found = tracker.driverPieces.find(block); if (found == tracker.driverPieces.end()) return true; const auto& entry = found->second; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp index 4292b133f..220e685f1 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp @@ -130,6 +130,22 @@ void copyMemory(std::uint64_t source, std::uint64_t destination, std::size_t byt GuestMemory::Write(destination, copySource(source, bytes, immediate)); } +constexpr std::uint32_t CopyDataGpuClockSource = 18; +constexpr std::uint32_t CopyDataCachePolicy = (3u << 13u) | (3u << 25u); + +std::uint64_t gpuClockCount() { + return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() / 10); +} + +std::vector copyDataSource(std::span packet, std::size_t bytes) { + const auto source = ((packet[1] & 0xfu) << 1u) | ((packet[1] >> 30u) & 1u); + if (source != CopyDataGpuClockSource) return copySource(address(packet[2], packet[3]), bytes, source >= 10); + const auto value = gpuClockCount(); + std::vector data(bytes); + for (std::size_t i = 0; i < bytes; ++i) data[i] = static_cast(value >> (i * 8)); + return data; +} + } std::string Name(std::uint32_t header) { @@ -417,12 +433,12 @@ void Validate(std::span packet, std::uint32_t queue) { } case 0x40: { size(6); - require((packet[1] & ~0x40110f0fu) == 0, "COPY_DATA engine, cache or reserved fields are not implemented"); + require((packet[1] & ~(0x40110f0fu | CopyDataCachePolicy)) == 0, "COPY_DATA engine or reserved fields are not implemented"); const auto source = ((packet[1] & 0xfu) << 1u) | ((packet[1] >> 30u) & 1u); const auto destination = ((packet[1] >> 8u) & 0xfu) << 1u; require(destination == 2 || destination == 4, "COPY_DATA register or GDS destination is not implemented"); - require(source == 2 || source == 4 || source == 5 || source == 10 || source == 11, "COPY_DATA register, GDS or reference-clock source is not implemented"); - require(source < 10 || ((packet[1] & 0x10000u) == 0 && packet[3] == 0), "64-bit immediate COPY_DATA is not implemented"); + require(source == 2 || source == 4 || source == 5 || source == 10 || source == 11 || source == CopyDataGpuClockSource, "COPY_DATA register, GDS or reference-clock source is not implemented"); + require(source < 10 || source == CopyDataGpuClockSource || ((packet[1] & 0x10000u) == 0 && packet[3] == 0), "64-bit immediate COPY_DATA is not implemented"); break; } case 0x50: @@ -550,7 +566,7 @@ std::optional ResolveStore(std::span packet, co const std::size_t bytes = (packet[1] & 0x10000u) != 0 ? 8 : 4; const auto destination = address(packet[4], packet[5]); if (!fits(destination, bytes)) return std::nullopt; - return StoreWrite{destination, {}, copySource(address(packet[2], packet[3]), bytes, source >= 10)}; + return StoreWrite{destination, {}, copyDataSource(packet, bytes)}; } case 0x50: { if (packet.size() < 7 || dmaDestination(packet) == DmaSelectGds) return std::nullopt; @@ -716,6 +732,28 @@ DrawParameters ResolveDraw(std::span packet, const QueueSta return {address, count, indexSize, queue.instanceCount, packet.back(), true, indexOffset->second, 0}; } +std::vector ReadIndirectRegisters(std::span packet) { + require(packet.size() == 5 && IndirectRegisterOpcode((packet[0] >> 8u) & 0xffu), "expected indirect register packet"); + std::vector pairs(static_cast(packet[4]) * 2); + // Named for the [hooksync] attribution (the read goes through the flush hook). + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Registers); + GuestMemory::Read(address(packet[1], packet[2]), std::as_writable_bytes(std::span(pairs)), 4); + return pairs; +} + +void ExecuteIndirectRegisters(std::span packet, std::span pairs, QueueState& queue) { + const auto opcode = (packet[0] >> 8u) & 0xffu; + require(packet.size() == 5 && IndirectRegisterOpcode(opcode), "expected indirect register packet"); + require(pairs.size() == static_cast(packet[4]) * 2, "indirect register list does not match its packet"); + for (std::size_t i = 0; i < pairs.size(); i += 2) registerOffset(pairs[i]); + for (std::size_t i = 0; i < pairs.size(); i += 2) writeRegister(queue, opcode, registerOffset(pairs[i]), pairs[i + 1]); + if (queue.savedContext.has_value() && TraceContextState()) { + std::fprintf(stderr, "[context] indirect 0x%x:", opcode); + for (std::size_t i = 0; i < pairs.size(); i += 2) std::fprintf(stderr, " %x", registerOffset(pairs[i])); + std::fprintf(stderr, "\n"); + } +} + void Execute(std::span packet, QueueState& queue) { const auto opcode = (packet[0] >> 8u) & 0xffu; switch (opcode) { @@ -764,20 +802,9 @@ void Execute(std::span packet, QueueState& queue) { case 0x26: queue.indexBase = address(packet[1], packet[2]); return; case 0x2a: queue.indexType = packet[1]; return; case 0x2f: queue.instanceCount = packet[1]; return; - case 0x63: case 0x64: case 0x9f: { - std::vector pairs(static_cast(packet[4]) * 2); - // Named for the [hooksync] attribution (the read goes through the flush hook). - const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Registers); - GuestMemory::Read(address(packet[1], packet[2]), std::as_writable_bytes(std::span(pairs)), 4); - for (std::size_t i = 0; i < pairs.size(); i += 2) registerOffset(pairs[i]); - for (std::size_t i = 0; i < pairs.size(); i += 2) writeRegister(queue, opcode, registerOffset(pairs[i]), pairs[i + 1]); - if (queue.savedContext.has_value() && TraceContextState()) { - std::fprintf(stderr, "[context] indirect 0x%x:", opcode); - for (std::size_t i = 0; i < pairs.size(); i += 2) std::fprintf(stderr, " %x", registerOffset(pairs[i])); - std::fprintf(stderr, "\n"); - } + case 0x63: case 0x64: case 0x9f: + ExecuteIndirectRegisters(packet, ReadIndirectRegisters(packet), queue); return; - } case 0x59: break; case 0x3c: case 0x93: { // The waited-on value is written by the CPU or another queue; poll it like the CP would. @@ -835,8 +862,10 @@ void Execute(std::span packet, QueueState& queue) { return; } case 0x40: { - const auto source = ((packet[1] & 0xfu) << 1u) | ((packet[1] >> 30u) & 1u); - copyMemory(address(packet[2], packet[3]), address(packet[4], packet[5]), (packet[1] & 0x10000u) != 0 ? 8 : 4, source >= 10); + const auto destination = address(packet[4], packet[5]); + const auto data = copyDataSource(packet, (packet[1] & 0x10000u) != 0 ? 8 : 4); + GuestMemory::CheckRange(reinterpret_cast(destination), data.size(), 1, true); + GuestMemory::Write(destination, data); return; } case 0x50: { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderDeviceProfile.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderDeviceProfile.cpp new file mode 100644 index 000000000..5eec16c82 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderDeviceProfile.cpp @@ -0,0 +1,80 @@ +#include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp" +#include "BdaAbi.hpp" +#include +#include + +namespace AgcDriver { +namespace { + +template +const TFeatures* findFeatures(const void* chain, VkStructureType type) { + auto* current = static_cast(chain); + while (current != nullptr) { + if (current->sType == type) return reinterpret_cast(current); + current = current->pNext; + } + return nullptr; +} + +} + +ShaderDeviceProfile::ShaderDeviceProfile(const ShaderRecompiler::SpirvTarget& target, const VkDeviceCreateInfo& deviceInfo, const VkPhysicalDeviceLimits& limits) : target(target), limits(limits), capabilities(target.supportedCapabilities.begin(), target.supportedCapabilities.end()) { + using Graphics::Require; + Require(deviceInfo.pEnabledFeatures != nullptr, "shader device profile requires enabled core features"); + const auto& core = *deviceInfo.pEnabledFeatures; + const auto* robustness = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT); + nullDescriptors = robustness != nullptr && robustness->nullDescriptor == VK_TRUE; + const auto* bda = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES); + const auto* bytes = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_8BIT_STORAGE_FEATURES); + const auto* indexing = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES); + const auto* barycentric = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_BARYCENTRIC_FEATURES_KHR); + const auto* mesh = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MESH_SHADER_FEATURES_EXT); + const auto* clock = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR); + const auto* maintenance8 = findFeatures(deviceInfo.pNext, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_8_FEATURES_KHR); + Require(target.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version, "shader device profile has an incompatible BDA ABI"); + Require(bda != nullptr && bda->bufferDeviceAddress == VK_TRUE && core.shaderInt64 == VK_TRUE, "shader runtime requires enabled bufferDeviceAddress and shaderInt64"); + Require(bytes != nullptr && bytes->storageBuffer8BitAccess == VK_TRUE, "shader runtime requires enabled storageBuffer8BitAccess"); + Require(nullDescriptors, "shader runtime requires enabled nullDescriptor"); + Require(core.vertexPipelineStoresAndAtomics == VK_TRUE && core.fragmentStoresAndAtomics == VK_TRUE, "shader runtime requires enabled graphics stores and atomics"); + Require(limits.maxPushConstantsSize >= ShaderRecompiler::RuntimeAbi::PushConstantDwords * sizeof(std::uint32_t) && limits.maxBoundDescriptorSets > ShaderRecompiler::RuntimeAbi::DescriptorSet, "shader runtime ABI exceeds device limits"); + Require(limits.maxStorageBufferRange >= sizeof(ShaderRecompiler::RuntimeAbi::ShaderData), "shader runtime ShaderData exceeds device limits"); + std::ranges::sort(capabilities); + capabilities.erase(std::unique(capabilities.begin(), capabilities.end()), capabilities.end()); + const auto hasCapability = [&](spv::Capability capability) { return std::ranges::binary_search(capabilities, static_cast(capability)); }; + Require(hasCapability(spv::CapabilityShader) && hasCapability(spv::CapabilityInt64) && hasCapability(spv::CapabilityPhysicalStorageBufferAddresses) && hasCapability(spv::CapabilityStorageBuffer8BitAccess), "shader device profile is missing runtime capabilities"); + const auto checkFeature = [&](spv::Capability capability, bool enabled, const char* name) { + Require(!hasCapability(capability) || enabled, std::string("shader device profile advertises a disabled feature: ") + name); + }; + checkFeature(spv::CapabilityFloat64, core.shaderFloat64 == VK_TRUE, "shaderFloat64"); + checkFeature(spv::CapabilityTessellation, core.tessellationShader == VK_TRUE, "tessellationShader"); + checkFeature(spv::CapabilityImageGatherExtended, core.shaderImageGatherExtended == VK_TRUE, "shaderImageGatherExtended"); + checkFeature(spv::CapabilityMinLod, core.shaderResourceMinLod == VK_TRUE, "shaderResourceMinLod"); + checkFeature(spv::CapabilityStorageImageMultisample, core.shaderStorageImageMultisample == VK_TRUE, "shaderStorageImageMultisample"); + checkFeature(spv::CapabilityStorageImageReadWithoutFormat, core.shaderStorageImageReadWithoutFormat == VK_TRUE, "shaderStorageImageReadWithoutFormat"); + checkFeature(spv::CapabilityStorageImageWriteWithoutFormat, core.shaderStorageImageWriteWithoutFormat == VK_TRUE, "shaderStorageImageWriteWithoutFormat"); + checkFeature(spv::CapabilitySampledImageArrayDynamicIndexing, core.shaderSampledImageArrayDynamicIndexing == VK_TRUE, "shaderSampledImageArrayDynamicIndexing"); + checkFeature(spv::CapabilityStorageImageArrayDynamicIndexing, core.shaderStorageImageArrayDynamicIndexing == VK_TRUE, "shaderStorageImageArrayDynamicIndexing"); + const bool sampledNonUniform = indexing != nullptr && indexing->shaderSampledImageArrayNonUniformIndexing == VK_TRUE; + const bool storageNonUniform = indexing != nullptr && indexing->shaderStorageImageArrayNonUniformIndexing == VK_TRUE; + checkFeature(spv::CapabilitySampledImageArrayNonUniformIndexing, sampledNonUniform, "shaderSampledImageArrayNonUniformIndexing"); + checkFeature(spv::CapabilityStorageImageArrayNonUniformIndexing, storageNonUniform, "shaderStorageImageArrayNonUniformIndexing"); + checkFeature(spv::CapabilityShaderNonUniform, sampledNonUniform || storageNonUniform, "descriptor indexing"); + const bool barycentricEnabled = barycentric != nullptr && barycentric->fragmentShaderBarycentric == VK_TRUE; + const bool meshEnabled = mesh != nullptr && mesh->meshShader == VK_TRUE; + checkFeature(spv::CapabilityFragmentBarycentricKHR, barycentricEnabled, "fragmentShaderBarycentric"); + checkFeature(spv::CapabilityMeshShadingEXT, meshEnabled, "meshShader"); + checkFeature(spv::CapabilityShaderClockKHR, clock != nullptr && clock->shaderSubgroupClock == VK_TRUE && clock->shaderDeviceClock == VK_TRUE, "shader clock"); + Require(!target.fragmentShaderBarycentricEnabled || barycentricEnabled, "shader device profile exposes disabled barycentrics"); + Require(!target.mesh || meshEnabled, "shader device profile exposes disabled mesh shaders"); + Require(!target.nonConstantImageOffsets || (maintenance8 != nullptr && maintenance8->maintenance8 == VK_TRUE), "shader device profile exposes disabled maintenance8"); + Require(!target.tessellation || core.tessellationShader == VK_TRUE, "shader device profile exposes disabled tessellation"); + for (const auto extension : target.supportedExtensions) extensions.emplace_back(extension); + std::ranges::sort(extensions); + extensions.erase(std::unique(extensions.begin(), extensions.end()), extensions.end()); + for (const auto& extension : extensions) extensionViews.emplace_back(extension); + Require(std::ranges::find(extensionViews, "SPV_KHR_physical_storage_buffer") != extensionViews.end() && std::ranges::find(extensionViews, "SPV_KHR_8bit_storage") != extensionViews.end(), "shader device profile is missing runtime extensions"); + this->target.supportedCapabilities = capabilities; + this->target.supportedExtensions = extensionViews; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp index f624288af..7344a3418 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp @@ -201,6 +201,28 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val return true; } +bool ShaderMemory::accessible(void* context, std::uint64_t address, std::uint64_t bytes) { + auto& self = *static_cast(context); + if (bytes == 0 || address > std::numeric_limits::max() - bytes) return false; + const auto end = address + bytes; + while (address < end) { + const auto next = self.initial.upper_bound(address); + if (next != self.initial.begin()) { + const auto previous = std::prev(next); + if (address - previous->first < previous->second.size()) { + address = previous->first + previous->second.size(); + continue; + } + } + const auto base = address & ~static_cast(PageBytes - 1); + const auto pageEnd = std::min(base + PageBytes, end); + const auto& page = self.page(base); + if (!page.wordwise && !page.valid.all() && !GuestMemory::Accessible(reinterpret_cast(address), static_cast(pageEnd - address))) return false; + address = pageEnd; + } + return true; +} + ShaderMemory::KnownValueCounts ShaderMemory::KnownValues() { auto& totals = CaptureTotals(); return {totals.wordsKnown.load(std::memory_order_relaxed), totals.wordsKnownVerified.load(std::memory_order_relaxed), totals.wordsKnownMismatches.load(std::memory_order_relaxed)}; @@ -212,6 +234,14 @@ void ShaderMemory::CountHandleMemo(bool hit) { } std::shared_ptr ShaderMemory::Capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle) { + return capture(request, handle, nullptr); +} + +std::shared_ptr ShaderMemory::Capture(const ShaderRecompiler::PreparedShaderInvocation& invocation) { + return capture(invocation.Request(), nullptr, &invocation); +} + +std::shared_ptr ShaderMemory::capture(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle* handle, const ShaderRecompiler::PreparedShaderInvocation* invocation) { // The capture's word and page reads (through `read`) are attributed to it ([hooksync], [guestmem]). const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Capture); const bool profile = CaptureProfiled(); @@ -225,7 +255,8 @@ std::shared_ptr ShaderMemory::Capture(c runtime.userContext = this; runtime.readMemory = &read; runtime.readSpecializationMemory = &read; - auto capture = handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime); + runtime.accessible = &accessible; + auto capture = invocation != nullptr ? invocation->Capture(runtime) : handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime); if (profile) { totals.captureNanoseconds += NanosecondsSince(started); totals.resolveNanoseconds += capture->sourceNanoseconds; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/SubgroupClock.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/SubgroupClock.cpp new file mode 100644 index 000000000..ac24b51d0 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/SubgroupClock.cpp @@ -0,0 +1,27 @@ +#include "prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp" +#include +#include + +namespace AgcDriver { + +namespace { + +constexpr std::array NarrowClockFamilies{ + "NAVI21", "NAVI22", "VANGOGH", "NAVI23", "NAVI24", "REMBRANDT", "RAPHAEL_MENDOCINO", + "NAVI31", "NAVI32", "NAVI33", "PHOENIX", "PHOENIX2", + "STRIX1", "STRIX_HALO", "KRACKAN1", "GFX1153", "GFX1156", + "GFX1170", "GFX1171", +}; + +} + +bool NarrowSubgroupClock(VkDriverId driver, std::string_view deviceName) { + if (driver != VK_DRIVER_ID_MESA_RADV) return false; + constexpr std::string_view prefix = "(RADV "; + const auto start = deviceName.rfind(prefix); + if (start == std::string_view::npos || deviceName.empty() || deviceName.back() != ')') return false; + const auto family = deviceName.substr(start + prefix.size(), deviceName.size() - start - prefix.size() - 1u); + return std::find(NarrowClockFamilies.begin(), NarrowClockFamilies.end(), family) != NarrowClockFamilies.end(); +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index e6ce5908f..96c366d80 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1,10 +1,13 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" #include "BdaAbi.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp" +#include "prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp" #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" @@ -187,12 +190,14 @@ struct VulkanDevice::State { VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}; std::vector capabilities{1}; std::vector spirvExtensions; + std::unique_ptr shaderProfile; bool tessellationShader = false; bool meshShader = false; bool fragmentShaderBarycentric = false; bool geometryShader = false; bool sampleRateShading = false; bool shaderClock = false; + bool narrowSubgroupClock = false; // VK_EXT_descriptor_indexing with non-uniform image array indexing (bindless image tables in // graphics stages, and compute workgroups wider than a wave). bool descriptorIndexing = false; @@ -209,13 +214,17 @@ struct VulkanDevice::State { bool occlusionQueryPrecise = false; VkDeviceSize hostImportAlignment = 0; bool dmaBufImport = false; + bool memoryBudget = false; bool depthRangeUnrestricted = false; bool samplerAnisotropy = false; bool textureCompressionBC = false; bool samplerFilterMinmax = false; + bool fragmentShaderPixelInterlock = false; + bool conservativeRasterization = false; // VK_KHR_timeline_semaphore enabled: the recorder's unlocked waits are available. bool timelineSemaphores = false; bool computeWave32 = false; + std::uint32_t maxComputeSubgroupSize = 0; // Indirect draw features enabled (see Graphics::Context). bool drawIndirectFirstInstance = false; bool multiDrawIndirect = false; @@ -524,6 +533,7 @@ struct VulkanDevice::State { resourceCache.Clear(); Graphics::ClearCachedTextures(device); Graphics::ClearImageMirrors(device); + Graphics::ClearHostImports(device); patternBuffers.clear(); descriptorCache.reset(); emptyBuffer.reset(); @@ -714,6 +724,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->capabilities.push_back(spv::CapabilityGroupNonUniform); if ((state->subgroup.supportedOperations & VK_SUBGROUP_FEATURE_BALLOT_BIT) != 0) state->capabilities.push_back(spv::CapabilityGroupNonUniformBallot); if ((state->subgroup.supportedOperations & VK_SUBGROUP_FEATURE_SHUFFLE_BIT) != 0) state->capabilities.push_back(spv::CapabilityGroupNonUniformShuffle); + if ((state->subgroup.supportedOperations & VK_SUBGROUP_FEATURE_ARITHMETIC_BIT) != 0) state->capabilities.push_back(spv::CapabilityGroupNonUniformArithmetic); } APS5_LOG_OUT("Selected GPU name=%s vendor=0x%x device=0x%x subgroup=%u", state->properties.deviceName, state->properties.vendorID, state->properties.deviceID, state->subgroup.subgroupSize); state->InstanceFunction("vkGetPhysicalDeviceMemoryProperties")(selected, &state->memoryProperties); @@ -750,12 +761,27 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); state->fragmentShaderBarycentric = barycentricFeatures.fragmentShaderBarycentric == VK_TRUE; } + VkPhysicalDeviceFragmentShaderInterlockFeaturesEXT interlockFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT}; + if (hasExtension(VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME)) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &interlockFeatures}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + state->fragmentShaderPixelInterlock = interlockFeatures.fragmentShaderPixelInterlock == VK_TRUE; + } + interlockFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT}; + interlockFeatures.fragmentShaderPixelInterlock = VK_TRUE; VkPhysicalDeviceShaderClockFeaturesKHR clockFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR}; if (hasExtension(VK_KHR_SHADER_CLOCK_EXTENSION_NAME)) { VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &clockFeatures}; state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); state->shaderClock = clockFeatures.shaderSubgroupClock == VK_TRUE && clockFeatures.shaderDeviceClock == VK_TRUE; } + if (state->shaderClock && hasExtension(VK_KHR_DRIVER_PROPERTIES_EXTENSION_NAME)) { + VkPhysicalDeviceDriverProperties driver{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; + VkPhysicalDeviceProperties2 driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &driver}; + state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &driverProperties); + state->narrowSubgroupClock = NarrowSubgroupClock(driver.driverID, state->properties.deviceName); + APS5_LOG_OUT("Shader clock driver=%d narrowSubgroupClock=%d", static_cast(driver.driverID), state->narrowSubgroupClock ? 1 : 0); + } std::vector deviceExtensions; if (window != nullptr) deviceExtensions.assign(presentationExtensions.begin(), presentationExtensions.end()); if (state->fragmentShaderBarycentric) { @@ -763,6 +789,11 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->capabilities.push_back(spv::CapabilityFragmentBarycentricKHR); state->spirvExtensions.push_back("SPV_KHR_fragment_shader_barycentric"); } + if (state->fragmentShaderPixelInterlock) { + deviceExtensions.push_back(VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME); + state->capabilities.push_back(spv::CapabilityFragmentShaderPixelInterlockEXT); + state->spirvExtensions.push_back("SPV_EXT_fragment_shader_interlock"); + } if (state->shaderClock) { deviceExtensions.push_back(VK_KHR_SHADER_CLOCK_EXTENSION_NAME); state->capabilities.push_back(spv::CapabilityShaderClockKHR); @@ -774,6 +805,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); } const bool bufferInt64Atomics = atomicInt64Features.shaderBufferInt64Atomics == VK_TRUE; + if (bufferInt64Atomics) state->capabilities.push_back(spv::CapabilityInt64Atomics); atomicInt64Features = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES_KHR}; atomicInt64Features.shaderBufferInt64Atomics = VK_TRUE; if (bufferInt64Atomics) deviceExtensions.push_back(VK_KHR_SHADER_ATOMIC_INT64_EXTENSION_NAME); @@ -814,14 +846,23 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->samplerFilterMinmax = minmaxProperties.filterMinmaxSingleComponentFormats == VK_TRUE && minmaxProperties.filterMinmaxImageComponentMapping == VK_TRUE; if (state->samplerFilterMinmax) deviceExtensions.push_back(VK_EXT_SAMPLER_FILTER_MINMAX_EXTENSION_NAME); } + if (hasExtension(VK_EXT_CONSERVATIVE_RASTERIZATION_EXTENSION_NAME)) { + VkPhysicalDeviceConservativeRasterizationPropertiesEXT conservativeProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CONSERVATIVE_RASTERIZATION_PROPERTIES_EXT}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &conservativeProperties}; + state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); + state->conservativeRasterization = conservativeProperties.primitiveOverestimationSize <= 1.0f / 256.0f && conservativeProperties.degenerateTrianglesRasterized == VK_TRUE; + if (state->conservativeRasterization) deviceExtensions.push_back(VK_EXT_CONSERVATIVE_RASTERIZATION_EXTENSION_NAME); + } // Indirect draws with a GPU-side count (DRAW_INDIRECT_MULTI with count_indirect); a device // without it resolves such draws on the CPU. state->drawIndirectCount = hasExtension(VK_KHR_DRAW_INDIRECT_COUNT_EXTENSION_NAME); if (state->drawIndirectCount) deviceExtensions.push_back(VK_KHR_DRAW_INDIRECT_COUNT_EXTENSION_NAME); + state->memoryBudget = hasExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME); // Guest memory is host memory: importing it lets address-based shaders use it in place instead of // copying every registered allocation per draw. if (hasExtension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME) && std::getenv("APS5_NO_HOST_IMPORT") == nullptr) { - VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT}; + VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; + VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT, &driverProperties}; VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &hostProperties}; state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); state->hostImportAlignment = hostProperties.minImportedHostPointerAlignment; @@ -834,7 +875,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u #endif deviceExtensions.push_back(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME); #ifndef _WIN32 - state->dmaBufImport = hasExtension(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME) && hasExtension(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME); + state->dmaBufImport = driverProperties.driverID != VK_DRIVER_ID_NVIDIA_PROPRIETARY && hasExtension(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME) && hasExtension(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME); if (state->dmaBufImport) { deviceExtensions.push_back(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME); deviceExtensions.push_back(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME); @@ -922,6 +963,9 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u enabled.occlusionQueryPrecise = available.occlusionQueryPrecise; state->occlusionQueryPrecise = enabled.occlusionQueryPrecise == VK_TRUE; // Recompiled storage-image access declares no format (the guest descriptor decides it). + enabled.shaderStorageImageMultisample = available.shaderStorageImageMultisample; + if (enabled.shaderStorageImageMultisample) state->capabilities.push_back(spv::CapabilityStorageImageMultisample); + state->capabilities.push_back(spv::CapabilityImageMSArray); enabled.shaderStorageImageWriteWithoutFormat = available.shaderStorageImageWriteWithoutFormat; enabled.shaderStorageImageReadWithoutFormat = available.shaderStorageImageReadWithoutFormat; // Gathers with non-constant offsets (ImageGatherExtended). @@ -988,6 +1032,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u barycentricFeatures.pNext = byteFeatures.pNext; byteFeatures.pNext = &barycentricFeatures; } + if (state->fragmentShaderPixelInterlock) { + interlockFeatures.pNext = byteFeatures.pNext; + byteFeatures.pNext = &interlockFeatures; + } if (state->shaderClock) { clockFeatures.pNext = byteFeatures.pNext; byteFeatures.pNext = &clockFeatures; @@ -1019,7 +1067,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->timelineSemaphores = timelineFeatures.timelineSemaphore == VK_TRUE; } VkPhysicalDeviceSubgroupSizeControlFeaturesEXT subgroupSizeFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT}; - if (hasExtension(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME) && state->subgroup.subgroupSize > 32u) { + if (hasExtension(VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME) && state->subgroup.subgroupSize >= 32u) { VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &subgroupSizeFeatures}; state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); VkPhysicalDeviceSubgroupSizeControlPropertiesEXT subgroupSize{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES_EXT}; @@ -1027,6 +1075,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &sizeProperties); state->computeWave32 = subgroupSizeFeatures.subgroupSizeControl == VK_TRUE && subgroupSize.minSubgroupSize <= 32u && subgroupSize.maxSubgroupSize >= 32u && (subgroupSize.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0; + state->maxComputeSubgroupSize = subgroupSize.maxSubgroupSize; } if (state->computeWave32) { subgroupSizeFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_FEATURES_EXT}; @@ -1064,14 +1113,33 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u byteFeatures.pNext = &pipelineFeatures; } bdaFeatures.pNext = &byteFeatures; + VkPhysicalDeviceRobustness2FeaturesEXT robustness2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT}; + Graphics::Require(hasExtension(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME), "shader runtime requires VK_EXT_robustness2"); + VkPhysicalDeviceFeatures2 queried{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &robustness2}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &queried); + Graphics::Require(robustness2.nullDescriptor == VK_TRUE, "shader runtime requires nullDescriptor"); + robustness2 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT}; + robustness2.nullDescriptor = VK_TRUE; + robustness2.pNext = bdaFeatures.pNext; + bdaFeatures.pNext = &robustness2; + deviceExtensions.push_back(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); + deviceInfo.enabledExtensionCount = static_cast(deviceExtensions.size()); + deviceInfo.ppEnabledExtensionNames = deviceExtensions.data(); deviceInfo.pNext = &bdaFeatures; + auto shaderProfile = std::make_unique(buildTarget(), deviceInfo, state->properties.limits); check(state->InstanceFunction("vkCreateDevice")(selected, &deviceInfo, nullptr, &state->device), "vkCreateDevice"); + state->shaderProfile = std::move(shaderProfile); state->DeviceFunction("vkGetDeviceQueue")(state->device, family, 0, &state->queue); APS5_LOG_OUT("Vulkan device ready device=%p queue=%p family=%u", reinterpret_cast(state->device), reinterpret_cast(state->queue), family); VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; poolInfo.queueFamilyIndex = family; check(state->DeviceFunction("vkCreateCommandPool")(state->device, &poolInfo, nullptr, &state->pool), "vkCreateCommandPool"); + static const bool procTable = std::getenv("APS5_NO_PROC_TABLE") == nullptr; + if (procTable) { + Graphics::FillDeviceFunctions(graphicsContext(), state->deviceFunctions); + state->functionsReady = true; + } Graphics::PrepareImportWatch(graphicsContext()); state->bufferPool = std::make_shared(graphicsContext()); state->emptyBuffer = std::make_unique(graphicsContext(), Graphics::EmptyBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); @@ -1083,14 +1151,6 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->samplerCache = std::make_unique(); state->recorder = std::make_unique(graphicsContext(), state->timelineSemaphores); state->recorder->Activate(); - // The entry points every record site uses, resolved once (APS5_NO_PROC_TABLE=1: per call, as - // before), and the context copy graphicsContext() hands out from here on: built after the - // recorder and the descriptor cache exist, so it carries them. - static const bool procTable = std::getenv("APS5_NO_PROC_TABLE") == nullptr; - if (procTable) { - Graphics::FillDeviceFunctions(graphicsContext(), state->deviceFunctions); - state->functionsReady = true; - } state->context = buildContext(); state->contextReady = true; if (window != nullptr) { @@ -1914,37 +1974,30 @@ std::shared_ptr PresentableResident(const Graphics::Co return resident; } -Graphics::DccKeys DisplayKeys(const DisplayBuffer& buffer, std::size_t bytes) { - const auto keys = Graphics::CurrentDccKeys(buffer.dccAddress, bytes); - if (keys != Graphics::DccKeys::Uncompressed && !Graphics::IsDccClear(keys)) { - char message[256]; - std::snprintf(message, sizeof(message), "VideoOut: display buffer 0x%llx reads %s DCC keys at 0x%llx: presenting DCC metadata that is not uniformly uncompressed or fast-cleared is not implemented", static_cast(buffer.address), Graphics::DccKeysName(keys), static_cast(buffer.dccAddress)); - throw std::runtime_error(message); - } - return keys; +Graphics::DccKeys ResidentKeys(const Graphics::StorageTexture& resident) { + const auto& own = resident.Descriptor(); + GuestMemory::CollectWritesUncached(own.dccAddress, Graphics::DccKeyBytes(resident.GuestBytes())); + return Graphics::ProvedClearKeys(own, resident.GuestBytes(), resident.KeyProof()); } -bool ResidentServesDisplay(const Graphics::StorageTexture& resident, const DisplayBuffer& buffer, std::size_t bytes) { - const auto keys = DisplayKeys(buffer, bytes); +bool ResidentServesDisplay(const Graphics::StorageTexture& resident, const DisplayBuffer& buffer) { + const auto keys = DisplayBufferKeys(buffer); if (keys == Graphics::DccKeys::ClearRegister) { char message[320]; std::snprintf(message, sizeof(message), "VideoOut: display buffer 0x%llx reads register-clear DCC keys at 0x%llx over the pending image 0x%llx (DCC 0x%llx, filled keys %s): whether its results precede the clear is not modeled", static_cast(buffer.address), static_cast(buffer.dccAddress), static_cast(resident.Descriptor().baseAddress), static_cast(resident.Descriptor().dccAddress), Graphics::DccKeysName(resident.FilledKeys())); throw std::runtime_error(message); } if (Graphics::IsDccClear(resident.FilledKeys()) && resident.FilledKeys() == keys) return false; - return Graphics::StorageImageServesKeys(resident, buffer.dccAddress); + return Graphics::KeysServeSurface(resident.Descriptor().dccAddress, resident.UploadedKeys(), resident.FilledKeys(), buffer.dccAddress, [&] { return ResidentKeys(resident); }, [&] { return keys; }); } bool ResidentKeysMoved(const Graphics::StorageTexture& resident) { - const auto& own = resident.Descriptor(); - if (own.dccAddress == 0) return false; - GuestMemory::CollectWritesUncached(own.dccAddress, Graphics::DccKeyBytes(resident.GuestBytes())); - return Graphics::ProvedClearKeys(own, resident.GuestBytes(), resident.KeyProof()) != resident.UploadedKeys(); + return resident.Descriptor().dccAddress != 0 && ResidentKeys(resident) != resident.UploadedKeys(); } std::optional> CompressedClearPixel(const DisplayBuffer& buffer, std::size_t bytes) { GuestMemory::FlushGpuWrites(buffer.address, bytes); - const auto keys = DisplayKeys(buffer, bytes); + const auto keys = DisplayBufferKeys(buffer); if (keys == Graphics::DccKeys::Uncompressed) return std::nullopt; return DisplayBufferClearPixel(buffer, keys); } @@ -1998,7 +2051,7 @@ bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { if (!NoResidentPresent()) { bool pending = false; resident = PresentableResident(graphicsContext(), buffer, filter, pending, convert); - if (resident != nullptr && buffer.dccAddress != 0 && !ResidentServesDisplay(*resident, buffer, bytes)) { + if (resident != nullptr && buffer.dccAddress != 0 && !ResidentServesDisplay(*resident, buffer)) { resident.reset(); convert = false; } @@ -2398,10 +2451,16 @@ std::string VulkanDevice::DeviceName() const { } ShaderRecompiler::SpirvTarget VulkanDevice::Target() const { + require(state->shaderProfile != nullptr, "shader device profile is unavailable"); + return state->shaderProfile->Target(); +} + +ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const { const auto& limits = state->properties.limits; ShaderRecompiler::SpirvTarget target{VK_API_VERSION_1_1, state->meshShader ? 0x00010400u : 0x00010300u, state->subgroup.subgroupSize, ShaderRecompiler::BdaAbi::Version, state->capabilities, state->spirvExtensions, false, {limits.maxComputeWorkGroupSize[0], limits.maxComputeWorkGroupSize[1], limits.maxComputeWorkGroupSize[2]}, limits.maxComputeWorkGroupInvocations, limits.maxComputeSharedMemorySize, {}, {}}; target.fragmentShaderBarycentricEnabled = state->fragmentShaderBarycentric; target.nonConstantImageOffsets = state->maintenance8; + target.narrowSubgroupClock = state->narrowSubgroupClock; target.srgbDecodeFormats = state->srgbDecodeFormats; if (state->meshShader) { const auto& mesh = state->meshLimits; @@ -2419,6 +2478,10 @@ bool VulkanDevice::SamplerFilterMinmax() const { return state->samplerFilterMinmax; } +bool VulkanDevice::ConservativeRasterization() const { + return state->conservativeRasterization; +} + Graphics::Context VulkanDevice::graphicsContext() const { static const bool noCache = std::getenv("APS5_NO_CONTEXT_CACHE") != nullptr; if (state->contextReady && !noCache) return state->context; @@ -2463,6 +2526,7 @@ Graphics::Context VulkanDevice::buildContext() const { context.depthBounds = state->depthBounds; context.depthBiasClamp = state->depthBiasClamp; context.samplerFilterMinmax = state->samplerFilterMinmax; + context.conservativeRasterization = state->conservativeRasterization; context.drawIndirectCount = state->drawIndirectCount; context.occlusionQueryPrecise = state->occlusionQueryPrecise; context.emptyBuffer = state->emptyBuffer ? state->emptyBuffer->Handle() : VK_NULL_HANDLE; @@ -2472,10 +2536,12 @@ Graphics::Context VulkanDevice::buildContext() const { context.imageInt64Atomics = state->imageInt64Atomics; context.geometryShader = state->geometryShader; context.sampleRateShading = state->sampleRateShading; + context.nullDescriptors = state->shaderProfile != nullptr && state->shaderProfile->NullDescriptors(); context.primitiveListRestart = state->primitiveListRestart; context.imageViewMinLod = state->imageViewMinLod; context.pipelineExecutableInfo = state->pipelineExecutableInfo; context.srgbDecodeFormats = state->srgbDecodeFormats; + context.memoryProperties2 = state->memoryBudget ? state->InstanceFunction("vkGetPhysicalDeviceMemoryProperties2") : nullptr; return context; } @@ -2492,6 +2558,10 @@ void VulkanDevice::ColorMetadataPass(const Graphics::ColorMetadataPass& pass) { } void VulkanDevice::Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots, std::shared_ptr* recipe) { + for (const auto& shader : shaders) { + require(shader.program != nullptr, "missing compiled shader"); + ShaderRecompiler::RuntimeAbi::RequireVersion(shader.program->runtimeAbiVersion); + } // Two stdout lines per draw cost ~1.3 ms per frame of the queue-0 worker (part of it under the // GPU mutex); APS5_TRACE_DRAWS=1 restores them. static const bool trace = std::getenv("APS5_TRACE_DRAWS") != nullptr; @@ -2898,6 +2968,7 @@ struct RecordedDispatch { DispatchTimer* timer; // Whether the data refresh recorded anything. bool refreshed = false; + bool resourceKeyMatched = false; }; VkDevice VulkanDevice::Device() const { @@ -3001,6 +3072,7 @@ std::uint64_t VulkanDevice::presync(std::span VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots) { + ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion); // APS5_LOCKED_BUILD=1: the whole build under the mutex, as before the split. static const bool lockedBuild = std::getenv("APS5_LOCKED_BUILD") != nullptr; if (lockedBuild || shader.spirv.size() < 5 || shader.spirv[0] != 0x07230203u) return nullptr; @@ -3020,7 +3092,7 @@ std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderReco prepared->phaseMs[which] += std::chrono::duration(now - phaseStart).count(); phaseStart = now; }; - if (ResourceCacheEnabled() && shader.variantId != 0) { + if (ResourceCacheEnabled() && shader.PipelineVariantId() != 0) { prepared->key = DispatchContentKey(compute, context.device); phase(PreparedDispatch::PrepareKey); cached = state->resourceCache.Find(prepared->key); @@ -3216,7 +3288,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { // after an earlier dispatch's reads of the buffers by that dispatch's trailing barrier. A // recipe compares the two 64-bit hashes first: equal hashes mean the buffers hold the words. const bool differs = record.dataRefresh == RecordedDispatch::DataRefresh::Words || resources.DataWordsHash() != record.dataWordsHash; - if (differs && resources.RefreshData(commands, *record.shader, &recorder)) { + if (differs && (record.resourceKeyMatched ? resources.refreshData(commands, *record.shader, &recorder) : resources.RefreshData(commands, *record.shader, &recorder))) { covered = 0; ++d.templateRefreshed; d.templateRevalidateMs += record.revalidateMs; @@ -3293,6 +3365,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { } VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::span snapshots, std::uint64_t programAddress, std::shared_ptr prepared, std::shared_ptr* recipeOut) { + ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion); PerformanceTimer timing("Vulkan.Dispatch"); if (recipeOut != nullptr) *recipeOut = nullptr; // Group counts for the trace lines; an indirect dispatch does not know them. @@ -3316,7 +3389,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec } // Pipelines are shared by dispatches of one compiled variant; descriptor set layouts built from the // same bindings are compatible, so the pipeline layout of the first dispatch serves them all. - const std::uint64_t pipelineKey = shader.variantId != 0 ? (shader.variantId << 1u) | (pushStages != 0 ? 1u : 0u) : 0u; + const std::uint64_t pipelineKey = shader.PipelineVariantId() != 0 ? (shader.PipelineVariantId() << 1u) | (pushStages != 0 ? 1u : 0u) : 0u; std::shared_ptr objects; if (pipelineKey != 0) { std::lock_guard pipelines(state->computePipelinesMutex); @@ -3332,7 +3405,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec const auto lookupsBefore = Graphics::DeviceProcLookups(); std::shared_ptr resources; ResourceCache::Key contentKey; - const bool cacheable = ResourceCacheEnabled() && shader.variantId != 0; + const bool cacheable = ResourceCacheEnabled() && shader.PipelineVariantId() != 0; // The object came from the resource cache: a compute template whose data buffers take this // dispatch's words at the record (ShaderResources::RefreshData). bool fromCache = false; @@ -3485,9 +3558,11 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec pipelineInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; pipelineInfo.stage.module = objects->module; pipelineInfo.stage.pName = "main"; + const Graphics::PipelineSpecialization specialization(shader); + pipelineInfo.stage.pSpecializationInfo = specialization.Info(); VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT requiredSubgroup{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT}; - requiredSubgroup.requiredSubgroupSize = shader.hostSubgroupSize; - if (state->computeWave32 && shader.hostSubgroupSize == 32u) pipelineInfo.stage.pNext = &requiredSubgroup; + requiredSubgroup.requiredSubgroupSize = std::min(shader.hostSubgroupSize, state->maxComputeSubgroupSize); + if (state->computeWave32 && (shader.hostSubgroupSize == 32u || shader.hostSubgroupSize > state->maxComputeSubgroupSize)) pipelineInfo.stage.pNext = &requiredSubgroup; pipelineInfo.layout = objects->layout; if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] creating a pipeline for %zu SPIR-V words (program 0x%llx)\n", shader.spirv.size(), static_cast(programAddress)); check(state->DeviceFunction("vkCreateComputePipelines")(state->device, context.pipelineCache, 1, &pipelineInfo, nullptr, &objects->pipeline), "vkCreateComputePipelines"); @@ -3506,6 +3581,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec auto& recorder = *state->recorder; IndirectOutcome outcome{0, 0}; RecordedDispatch record{&context, &shaders[0], resources, objects, pushStages, &pushBytes, x, y, z, arguments, nullptr, programAddress, fromCache && TemplateDataRefresh() ? RecordedDispatch::DataRefresh::Words : RecordedDispatch::DataRefresh::None, 0, revalidateMs, &timer}; + record.resourceKeyMatched = fromCache; if (arguments != 0) decideIndirect(record, outcome, groupsText); // The indirect hold total below still covers the whole call of a CPU-resolved one. const bool indirect = record.arguments != 0 || outcome.cpuReason != 0; @@ -3533,7 +3609,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec static auto lastReport = std::chrono::steady_clock::now(); const auto now = std::chrono::steady_clock::now(); const auto ms = std::chrono::duration(now - syncStart).count(); - auto& waits = byProgram[programAddress != 0 ? programAddress : shader.variantId]; + auto& waits = byProgram[programAddress != 0 ? programAddress : shader.PipelineVariantId()]; ++waits.count; waits.ms += ms; ++syncs; @@ -3581,6 +3657,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec } RecipeOutcome VulkanDevice::DispatchRecipe(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::uint64_t programAddress, const std::shared_ptr& hit, IndirectOutcome& outcome, const std::shared_ptr& verify, bool refreshByWords) { + ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion); PerformanceTimer timing("Vulkan.DispatchRecipe"); outcome = {0, 0}; char groupsText[40]; @@ -3630,7 +3707,7 @@ RecipeOutcome VulkanDevice::DispatchRecipe(const ShaderRecompiler::RecompileResu return RecipeOutcome::Rebuild; } std::shared_ptr mapped; - const auto pipelineKey = (shader.variantId << 1u) | (recipe.pushes ? 1u : 0u); + const auto pipelineKey = (shader.PipelineVariantId() << 1u) | (recipe.pushes ? 1u : 0u); { std::lock_guard pipelines(state->computePipelinesMutex); if (const auto found = state->computePipelines.find(pipelineKey); found != state->computePipelines.end()) mapped = found->second; @@ -3675,6 +3752,7 @@ RecipeOutcome VulkanDevice::DispatchRecipe(const ShaderRecompiler::RecompileResu const auto dataRefresh = !TemplateDataRefresh() ? RecordedDispatch::DataRefresh::None : refreshByWords ? RecordedDispatch::DataRefresh::Words : RecordedDispatch::DataRefresh::Hash; if (dataRefresh == RecordedDispatch::DataRefresh::Words) counters.refreshByWords.fetch_add(1, std::memory_order_relaxed); RecordedDispatch record{&context, &shaders[0], hit->resources, hit->objects, recipe.pushes ? VkShaderStageFlags{VK_SHADER_STAGE_COMPUTE_BIT} : VkShaderStageFlags{0}, &recipe.pushBytes, x, y, z, arguments, nullptr, programAddress, dataRefresh, recipe.dataWordsHash, 0, &timer}; + record.resourceKeyMatched = verify != nullptr; if (arguments != 0) decideIndirect(record, outcome, groupsText); const bool indirect = record.arguments != 0 || outcome.cpuReason != 0; const auto recordStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index 8cdc11cb3..f73ee10c2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -55,7 +55,8 @@ public: BufferPool(const BufferPool&) = delete; BufferPool& operator=(const BufferPool&) = delete; // The size a buffer for `bytes` is created with: its size class, or `bytes` itself when large. - static std::size_t Capacity(std::size_t bytes); + static std::size_t Capacity(std::size_t bytes, VkMemoryPropertyFlags properties); + static VkBufferUsageFlags Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties); std::optional Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); void Put(const BufferAllocation& allocation) noexcept; @@ -92,6 +93,8 @@ private: Tier& tierFor(std::size_t capacity, VkMemoryPropertyFlags properties); // The device tier's budget (APS5_STAGING_POOL_MIB), read once. static VkDeviceSize DeviceBudget(); + static bool DeviceTiered(VkMemoryPropertyFlags properties); + static std::size_t NextCapacity(std::size_t capacity); void destroy(const BufferAllocation& allocation) noexcept; // Moves the tier's least recently used slot (the oldest front of its lists) to `evicted`; the // caller destroys those after releasing the mutex, so builds taking buffers on other threads @@ -116,6 +119,7 @@ private: static constexpr VkDeviceSize smallBudget = 64ull * 1024 * 1024; // Requests of this size and more keep their exact size and go to the large tier. static constexpr std::size_t classLimit = std::size_t{1} << 20u; + static constexpr unsigned deviceClassBits = 3; static constexpr std::size_t defaultSlots = 4096; }; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp index 6431cd075..78112575a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp @@ -12,11 +12,12 @@ namespace AgcDriver::Graphics { enum class ColorTileMode : std::uint32_t { Linear = 0, Standard4KB = 5, + Standard64KB = 9, RenderTarget = 0x1b }; constexpr TextureTileMode ColorTextureTileMode(ColorTileMode mode) { - return mode == ColorTileMode::Linear ? TextureTileMode::kLinear : mode == ColorTileMode::Standard4KB ? TextureTileMode::kStandard4KB : TextureTileMode::kR64KBX; + return mode == ColorTileMode::Linear ? TextureTileMode::kLinear : mode == ColorTileMode::Standard4KB ? TextureTileMode::kStandard4KB : mode == ColorTileMode::Standard64KB ? TextureTileMode::kStandard64KB : TextureTileMode::kR64KBX; } ColorTileMode DecodeColorTileMode(std::uint32_t attrib3); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index 32aa6cde0..26c2b821f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -107,6 +107,7 @@ struct Context { // Nonzero when VK_EXT_external_memory_host is enabled: the required host pointer alignment. VkDeviceSize hostImportAlignment = 0; bool dmaBufImport = false; + PFN_vkGetPhysicalDeviceMemoryProperties2 memoryProperties2 = nullptr; RenderCache* renderCache = nullptr; DrawQueue* drawQueue = nullptr; GraphicsPipelineCache* graphicsPipelines = nullptr; @@ -125,6 +126,7 @@ struct Context { bool depthBounds = false; bool depthBiasClamp = false; bool samplerFilterMinmax = false; + bool conservativeRasterization = false; VkBuffer emptyBuffer = VK_NULL_HANDLE; // The device's list of recorded dispatches whose copied written buffers await a CPU write-back // (VulkanDevice's State::copiedWriters; the draw counterpart is DrawCopiedWriters): an indirect @@ -138,6 +140,7 @@ struct Context { bool imageInt64Atomics = false; bool geometryShader = false; bool sampleRateShading = false; + bool nullDescriptors = false; bool primitiveListRestart = false; bool imageViewMinLod = false; bool pipelineExecutableInfo = false; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp index 8808fb712..109a02b39 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp @@ -23,6 +23,8 @@ enum class DccKeys { Uncompressed, Clear0000, Clear0001, Clear1110, Clear1111, C const char* DccKeysName(DccKeys keys); std::size_t DccKeyBytes(std::uint64_t surfaceBytes); +std::size_t DccKeyCount(TextureTileMode tileMode, std::uint32_t elementBytes, std::uint32_t width, std::uint32_t height, std::uint64_t surfaceBytes); +std::size_t DccKeyCount(const GuestTextureResource& surface, std::uint64_t surfaceBytes); template bool KeysServeSurface(std::uint64_t followedDcc, DccKeys uploaded, DccKeys filled, std::uint64_t namedDcc, ReadFollowed&& readFollowed, ReadNamed&& readNamed) { @@ -35,15 +37,18 @@ bool KeysServeSurface(std::uint64_t followedDcc, DccKeys uploaded, DccKeys fille DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes); bool IsDccClear(DccKeys keys); DccKeys CurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes); +DccKeys CurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes, std::size_t keyCount); // Stores "uncompressed" keys over the surface's metadata on the CPU (a guest memory write: it waits // for recorded GPU work that writes the keys first). void MarkDccUncompressed(std::uint64_t metaAddress, std::uint64_t surfaceBytes); +void MarkDccUncompressed(std::uint64_t metaAddress, std::uint64_t surfaceBytes, std::size_t keyCount); // The same after a write-back recorded under GuestMemory::GpuMutex: when the keys are in host-imported // memory and a recorder is active, the store is a fill recorded into the open batch (ordered behind // the title's key-writing kernels like the write-back itself, nothing waits on the CPU) and the range // reads as uncompressed from ReadDccKeys while that batch is pending; otherwise the CPU store above. // APS5_CPU_DCC_KEYS=1 always stores on the CPU. void MarkDccUncompressed(const Context& context, std::uint64_t metaAddress, std::uint64_t surfaceBytes); +void MarkDccUncompressed(const Context& context, std::uint64_t metaAddress, std::uint64_t surfaceBytes, std::size_t keyCount); // Fills `bytes` with the texel a 0000/0001/1110/1111 clear code stands for ("1" is 1.0 or the integer // maximum; the alpha channel is the last one in memory when alphaOnMsb, else the first, and 3-channel // formats have none). False when the format has no encoding here. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp index e8fc9a7c2..078e05bb2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp @@ -28,7 +28,7 @@ struct DrawInputCopy { DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uint64_t address, std::size_t bytes, std::size_t alignment, Recorder::SnapshotUse use); void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCopy& copy, Recorder::SnapshotUse use, std::uint32_t derived); -std::array MeshIndexBufferDescriptor(const Pm4::DrawParameters& draw, std::uint64_t unreadAddress); +std::array MeshIndexBufferDescriptor(const Pm4::DrawParameters& draw); struct MeshArguments { std::uint32_t groups; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index 14988113a..f9adbd7e6 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -150,9 +150,12 @@ struct MirrorStats { std::uint64_t rebuilds = 0; std::uint64_t blocksCopied = 0; std::uint64_t heapRefills = 0; + std::uint64_t sweeps = 0; + std::uint64_t heapChecks = 0; }; MirrorStats MirrorCounters(); void ClearImageMirrors(VkDevice device); +void ClearHostImports(VkDevice device); struct AddressCopy { std::uint64_t begin; @@ -203,6 +206,7 @@ public: void UploadPrepare(bool addressable); void UploadFinish(bool addressable); VkDescriptorBufferInfo Descriptor(std::uint64_t address, std::size_t bytes, std::uint32_t& adjustment) const; + static std::uint64_t ViewBytes(std::uint64_t bytes, std::uint32_t adjustment); std::vector AddressRanges() const; // The BDA table of the cached address space when it serves this upload alone (an address-based // build with no region outside it): its ranges, immutable while the space lives, and the @@ -344,6 +348,8 @@ private: // The cached address space this build maps through (its lease pins the ranges); `regions` then // holds only the regions outside it (V#s, snapshots, ranges copied per build). std::shared_ptr space; + mutable std::uint64_t writeTableSerial = 0; + mutable std::shared_ptr> writeTableRanges; // Import registry epoch when `direct` pointers were taken at acquire time; they are reused while // no import was destroyed since. std::uint64_t importsEpoch = 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp index 6397574f3..aa0765d12 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp @@ -81,6 +81,7 @@ struct GuestTextureResource { // DCC metadata of a compressed surface, or 0 (see DccMetadata.hpp). std::uint64_t dccAddress = 0; bool dccAlphaOnMsb = false; + bool dccPipeAligned = false; std::uint32_t minLod = 0; std::uint32_t allocatedMipCount = 0; }; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp new file mode 100644 index 000000000..0d53ce5be --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp @@ -0,0 +1,76 @@ +#ifndef CORE_LIBS_AGCDRIVER_GRAPHICS_PIPELINESPECIALIZATION_HPP +#define CORE_LIBS_AGCDRIVER_GRAPHICS_PIPELINESPECIALIZATION_HPP + +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +class PipelineSpecialization { +public: + static void Validate(const ShaderRecompiler::RecompileResult& shader) { + const auto& words = shader.spirv; + Require(words.size() >= 5 && words[0] == spv::MagicNumber && words[3] != 0, "invalid specialization module header"); + std::set unsignedTypes; + std::map constants; + std::map decorations; + std::set constantIds; + std::set supplied; + for (const auto& constant : shader.specialization) Require(supplied.insert(constant.id).second, "duplicate pipeline specialization ID"); + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + const auto op = static_cast(words[cursor] & 0xffffu); + Require(count != 0 && count <= words.size() - cursor, "truncated specialization module instruction"); + const auto validId = [&](std::uint32_t id) { Require(id != 0 && id < words[3], "specialization ID exceeds module bound"); }; + if (op == spv::OpTypeInt) { + Require(count == 4, "malformed integer type"); + if (words[cursor + 2] == 32u && words[cursor + 3] == 0u) unsignedTypes.insert(words[cursor + 1]); + } else if (op == spv::OpSpecConstant) { + Require(count == 4, "specialization constant must have a 32-bit literal"); + validId(words[cursor + 2]); + Require(constants.emplace(words[cursor + 2], words[cursor + 1]).second, "duplicate specialization result ID"); + } else if (op == spv::OpDecorate && count >= 3 && words[cursor + 2] == spv::DecorationSpecId) { + Require(count == 4, "malformed SpecId decoration"); + validId(words[cursor + 1]); + Require(decorations.emplace(words[cursor + 1], words[cursor + 3]).second && constantIds.insert(words[cursor + 3]).second, "duplicate SpecId decoration"); + } else if (op == spv::OpSpecConstantTrue || op == spv::OpSpecConstantFalse || op == spv::OpSpecConstantComposite || op == spv::OpSpecConstantOp) { + throw std::runtime_error("AGC graphics: unsupported specialization constant representation"); + } + cursor += count; + } + Require(constants.size() == decorations.size(), "specialization declarations and SpecId decorations disagree"); + for (const auto& [id, type] : constants) { + Require(unsignedTypes.contains(type), "specialization constant must be uint32"); + const auto decoration = decorations.find(id); + Require(decoration != decorations.end() && supplied.contains(decoration->second), "required pipeline specialization value is missing"); + } + } + + explicit PipelineSpecialization(const ShaderRecompiler::RecompileResult& shader) { + Validate(shader); + entries.reserve(shader.specialization.size()); + for (std::size_t index = 0; index < shader.specialization.size(); ++index) { + const auto offset = index * sizeof(ShaderRecompiler::PipelineSpecializationConstant) + offsetof(ShaderRecompiler::PipelineSpecializationConstant, value); + Require(offset <= std::numeric_limits::max(), "pipeline specialization offset exceeds Vulkan limits"); + entries.push_back({shader.specialization[index].id, static_cast(offset), sizeof(std::uint32_t)}); + } + info.mapEntryCount = static_cast(entries.size()); + info.pMapEntries = entries.data(); + info.dataSize = shader.specialization.size() * sizeof(ShaderRecompiler::PipelineSpecializationConstant); + info.pData = shader.specialization.data(); + } + + [[nodiscard]] const VkSpecializationInfo* Info() const { return entries.empty() ? nullptr : &info; } + +private: + std::vector entries; + VkSpecializationInfo info{}; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp index dcf383735..cea65de45 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -67,7 +67,7 @@ public: // continuing draw the command buffer without ending it. bool ContinuesRenderPass(std::uint64_t key) const; VkCommandBuffer CommandsInRenderPass(); - void LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool continuable); + void LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool continuable, std::function afterPass = {}); // DCC "uncompressed" key stores (DccMetadata.cpp StoreUncompressedOnGpu): queued on the open // batch and recorded as one run (one barrier pair for every queued fill) at Submit, before a // label store (RecordStore), or before a command that writes or reads a queued range (the @@ -86,14 +86,10 @@ public: bool Idle() const { return open == nullptr && inFlight.empty(); } // Whether recorded work still has completion actions (write-backs the CPU must see) to run. bool HasCompletions() const; - // The object lives at least until the batch open now completed; it is then destroyed AFTER the - // finishing thread released GuestMemory::GpuMutex, on a release thread of its own (never under - // a hold, see Recorder.cpp ReleaseDeferredKeeps; the finishing thread destroys it itself when - // the thread's queue is full, APS5_RELEASE_QUEUE_MAX batches, or with APS5_RELEASE_ON_UNLOCK=1; - // APS5_RELEASE_UNDER_LOCK=1 destroys it in the reap as before), so its destructor must need - // neither the mutex nor the recorder nor a particular thread. ~Recorder joins the release - // thread and waits for every release in progress before the device goes. - void Keep(std::shared_ptr object); + void Keep(std::shared_ptr object, std::size_t bytes = 0); + static constexpr std::size_t KeptBytesBudget = std::size_t{512} << 20u; + void BoundKeptBytes(); + std::size_t InFlightKeptBytes() const { return inFlightKeptBytes; } enum class SnapshotUse : std::uint8_t { Storage, Vertex, Index16, Index32 }; static constexpr std::size_t DrawSnapshotBudget = std::size_t{256} << 20u; static constexpr std::size_t DrawSnapshotEntries = 1024; @@ -499,6 +495,7 @@ private: VkCommandBuffer commands = VK_NULL_HANDLE; VkFence fence = VK_NULL_HANDLE; std::vector> kept; + std::size_t keptBytes = 0; std::vector> completions; std::vector> writes; std::vector writeNotes; @@ -590,6 +587,7 @@ private: bool continuable = false; std::uint64_t key = 0; std::uint32_t timing = NoTiming; + std::function afterPass; } renderPass; // A pass ended in this batch: Submit records the host-read barrier its draws left out. bool hostReadOwed = false; @@ -685,7 +683,7 @@ private: // Returns false when the locked path must run instead (no timeline, nested acquisition, off). bool syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int source, const void* site); // `source` is the CountSync source the wait is attributed to. - void finish(std::unique_ptr batch, bool wait, int source); + void finish(std::unique_ptr batch, bool wait, int source, bool releaseNow = false); void release(Batch& batch) noexcept; static bool overlaps(const Batch& batch, std::uint64_t address, std::uint64_t end); // The first noted read of `batch` overlapping [address, end), or null. @@ -740,6 +738,7 @@ private: // snapshot (a rebuild from inside a completion must not drop them) until finish returns. std::vector finishing; std::uint64_t submissions = 0; + std::size_t inFlightKeptBytes = 0; std::uint64_t writeNoteCount = 0; // Command buffers and fences of completed batches, reused by later ones (hundreds of batches per // frame would otherwise allocate and free their objects each time). diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp index d54fba676..671937306 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp @@ -11,14 +11,14 @@ namespace AgcDriver::Graphics { ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& context, const std::array& exportMappings, bool nullProgram = false); -ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& shader); +ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& shader, std::span header); // A guest range DecodeVertexStageInfo read (an attribute word, a vertex V#) with the bytes as read: // the draw cache validates them by value with the stage's capture (design_cpu_final rule RD). struct DecodeRead { std::uint64_t address; std::vector bytes; }; -ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span header, std::uint64_t headerAddress, std::span userData, std::vector* reads = nullptr); +ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span header, std::uint64_t headerAddress, std::span userData, std::vector* reads = nullptr, bool staticAbi = false); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index 29e48af83..c4f06ee68 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -18,12 +18,25 @@ #include #include +namespace AgcDriver { class VulkanDevice; } + namespace AgcDriver::Graphics { class Recorder; void FlushCachedTextures(VkDevice device); void ClearCachedTextures(VkDevice device); +std::uint64_t TextureCacheBudget(const VkPhysicalDeviceMemoryProperties& memory); +std::uint64_t SampledTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t textureBytes); +bool SampledBudgetReportDue(std::uint64_t reported, std::uint64_t budget, std::chrono::steady_clock::duration sinceReport); +struct TextureCacheUse { + std::size_t sampledEntries = 0; + std::uint64_t sampledBytes = 0; + std::uint64_t storageBytes = 0; +}; +TextureCacheUse TextureCacheUsage(); +std::uint64_t SampledTextureCacheBudget(const Context& context); +std::shared_ptr CachedSampledTexture(const Context& context, std::span words); // The cached storage image of a surface (render targets use it as their resident image); brought up // to date with guest memory before it is returned. @@ -60,9 +73,6 @@ public: VkDescriptorSet set = VK_NULL_HANDLE; VkDescriptorPool pool = VK_NULL_HANDLE; }; - // A set of `layout` needing `sizes` descriptors from the pool chain, opening a pool when no pool - // has room; a null set when the needs exceed what one chain pool holds (the caller then makes a - // dedicated pool, as before). SetAllocation Allocate(VkDescriptorSetLayout layout, std::span sizes); void Free(const SetAllocation& allocation) noexcept; // APS5_PROFILE_DRAW counters: layouts served from the map / created, sets allocated, pools opened. @@ -71,6 +81,7 @@ public: std::uint64_t layoutMisses = 0; std::uint64_t sets = 0; std::uint64_t pools = 0; + std::uint64_t dedicatedPools = 0; }; Stats Counters() const; @@ -82,6 +93,7 @@ private: mutable std::mutex mutex; std::map, VkDescriptorSetLayout> layouts; std::vector pools; + std::vector dedicated; Stats stats; }; @@ -228,6 +240,8 @@ public: std::vector> PresyncSurfaces() const; private: + friend class AgcDriver::VulkanDevice; + bool refreshData(VkCommandBuffer commands, const CompiledShader& shader, Recorder* recorder); struct DescribedRange { const char* kind; std::uint64_t address; @@ -396,7 +410,6 @@ private: // Whether the layout is this object's own (no cache) and destroyed with it. bool ownsLayout = false; VkDescriptorSet _set = VK_NULL_HANDLE; - // Dedicated pool of this object's set (no cache, or a set too large for a cache pool). VkDescriptorPool pool = VK_NULL_HANDLE; // The cache pool the set was allocated from, freed back to it on release. VkDescriptorPool cachePool = VK_NULL_HANDLE; @@ -414,6 +427,7 @@ private: std::vector storageAtomic; std::vector storageAtomic64; std::vector> samplers; + std::shared_ptr paddingSampler; bool reusable = false; std::vector directRegions; std::vector validatedTextures; @@ -434,6 +448,7 @@ private: // still to look up (index into `bindings`; the DescriptorBinding lives in the compiled shader), // the descriptor counts the set was sized for, and the compute stage of a deferred build. std::vector bindings; + std::vector refreshResourceKey; struct DeferredImages { const ShaderRecompiler::DescriptorBinding* binding; std::size_t index; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp index 8367436d9..123dbe52a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp @@ -41,6 +41,9 @@ struct ColorTarget { // DCC metadata of a compressed target (CB_COLOR_INFO DCC_ENABLE), or 0 (see DccMetadata.hpp). std::uint64_t dccAddress = 0; bool dccAlphaOnMsb = false; + bool dccPipeAligned = false; + std::uint64_t cmaskAddress = 0; + std::size_t cmaskBytes = 0; std::uint64_t surfaceAddress = 0; VkExtent2D surfaceExtent{}; std::uint32_t mipCount = 1; @@ -89,6 +92,7 @@ struct State { VkViewport viewport; bool negativeOneToOne; bool depthClamp = false; + VkConservativeRasterizationModeEXT conservativeRasterization = VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT; VkRect2D scissor; VkCullModeFlags cullMode; VkFrontFace frontFace; @@ -100,6 +104,8 @@ ShaderStages DecodeShaderStages(const QueueState& queue); State DecodeState(const QueueState& queue); std::array ExportMappings(const State& state); ColorTarget DecodeColorBuffer(const Registers& context, std::uint32_t slot); +std::size_t CmaskBytes(std::uint32_t width, std::uint32_t height); +std::uint32_t ColorWriteMask(const Registers& context); struct ColorMetadataPass { enum class Mode { EliminateFastClear, DccDecompress }; @@ -107,11 +113,12 @@ struct ColorMetadataPass { std::vector targets; }; std::optional DecodeColorMetadataPass(const QueueState& queue); +std::string DepthMaintenanceRejection(const QueueState& queue); // The message DecodeState (or the pixel stage decode after it) would throw for the register rules // this precheck covers, evaluated without exceptions before the draw is decoded; empty when they // pass (DecodeState still checks everything). A register a rule needs that is absent is no verdict. std::string DrawRejection(const QueueState& queue, bool indexed); -bool PixelProgramUnset(const QueueState& queue); +bool PixelProgramSkipped(const QueueState& queue); std::string NullPixelProgramRejection(const QueueState& queue); // The recording facade of the draw decoders (design_cpu_final M8, step 8a): every register read diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index 31916bbd9..fb2bb627b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -34,6 +34,7 @@ bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat); // Whether a storage image of the guest format takes DCC clear `keys` as a GPU clear (see // StorageTexture::upload); false for integer formats and non-clear keys. bool StorageClearAvailable(const Context& context, std::uint32_t guestFormat, DccKeys keys); +std::uint64_t SampledTextureMemory(); // A sampled texture's own VkImage with its memory, shared with the recorder while a recorded upload // still writes it (see the snapshot constructor), so the texture may go before the batch completes. @@ -101,12 +102,15 @@ private: ViewRange firstLayerRange{}; VkImageLayout layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; VkDeviceSize allocationBytes = 0; + VkDeviceSize countedBytes = 0; VkFormat viewFormat = VK_FORMAT_UNDEFINED; std::shared_ptr source; std::shared_ptr storageSource; std::unique_ptr upload; }; +VkFormat AttachmentProxyFormat(const Context& context, VkFormat format); + // A guest texture a shader writes through a storage image. It is uploaded like a sampled texture; // after the GPU work completes its results are stored to guest memory (retiled, changed bytes only), // either at once (WriteBack) or deferred: MarkDirty keeps them on the GPU until something reads that @@ -131,6 +135,9 @@ public: // buffer's format and mark the image dirty like a storage write. bool Attachable() const { return attachable; } VkImageView AttachmentView(VkFormat format, std::uint32_t mip = 0, std::uint32_t depthSlice = 0); + VkImageView AttachmentProxyView(); + void RecordAttachmentProxyLoad(VkCommandBuffer commands, VkImageLayout attachmentLayout) const; + void RecordAttachmentProxyStore(VkCommandBuffer commands, VkImageLayout attachmentLayout) const; void WriteBack(); // Deferred write-back (APS5_EAGER_WRITEBACK=1 stores at once instead). void MarkDirty(); @@ -285,6 +292,7 @@ public: // Keeps the image current with guest memory (see GuestMemory::CollectWrites). bool Refresh(); std::uint64_t GuestBytes() const; + VkDeviceSize AllocationBytes() const { return memoryBytes; } private: // The regions of every array layer, or of the tracked layers `layers` selects. @@ -292,6 +300,8 @@ private: // Uploads the surface, or only the tracked layers `layers` selects (the direct path; the others // upload everything). void upload(const std::vector* layers = nullptr); + void captureGuestBytes(const std::vector* layers); + bool compareUntracked(std::uint64_t address, std::size_t bytes, std::span changed, bool memoize = false) const; // Stores the pending tracked layers overlapping [address, address + bytes) to guest memory; in // each, 64 KiB blocks the CPU wrote since the layer's generation keep the CPU's bytes. Block // units asked for in pieces too often are all stored at once for a while (the hysteresis: @@ -420,9 +430,10 @@ private: std::uint64_t sliceLinearBytes = 0; SurfaceGeometry geometry; std::vector original; + mutable std::array comparedGuestBytes{}; // DCC keys the image content was uploaded under: a fast-cleared surface starts as its clear value. DccKeys uploadedKeys = DccKeys::Uncompressed; - DccKeys filledKeys = DccKeys::Uncompressed; + mutable DccKeys filledKeys = DccKeys::Uncompressed; mutable DccKeyProof keyProof; struct ForeignKeyProof { std::uint64_t dccAddress = 0; @@ -452,6 +463,7 @@ private: bool lent = false; VkImage image = VK_NULL_HANDLE; VkDeviceMemory memory = VK_NULL_HANDLE; + VkDeviceSize memoryBytes = 0; VkImageView view = VK_NULL_HANDLE; std::uint32_t defaultMip = 0; std::map extraViews; @@ -460,6 +472,9 @@ private: std::map, VkImageView> uintViews; bool attachable = false; std::map, VkImageView> attachmentViews; + VkImage proxyImage = VK_NULL_HANDLE; + VkDeviceMemory proxyMemory = VK_NULL_HANDLE; + VkImageView proxyView = VK_NULL_HANDLE; VkFormat storageFormat = VK_FORMAT_UNDEFINED; // Results are on the GPU only (guarded by the pending-write registry lock). bool dirty = false; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp index 883dbac0e..1964f46be 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp @@ -20,37 +20,39 @@ struct VertexFormat { inline VertexFormat DecodeVertexFormat(const ShaderRecompiler::VertexAttribute& attribute) { Require(attribute.components >= 1 && attribute.components <= 4, "invalid vertex attribute component count"); + const auto components = attribute.formatComponents == 0u ? attribute.components : attribute.formatComponents; + Require(components >= 1u && components <= 4u, "invalid vertex format component count"); const auto format = (attribute.resource.fields[3] >> 12u) & 0x7fu; switch (format) { - case 1: { const std::array formats{VK_FORMAT_R8_UNORM}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 2: { const std::array formats{VK_FORMAT_R8_SNORM}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 3: { const std::array formats{VK_FORMAT_R8_USCALED}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 4: { const std::array formats{VK_FORMAT_R8_SSCALED}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 5: { const std::array formats{VK_FORMAT_R8_UINT}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 1u, 1u, "u32"}; } - case 6: { const std::array formats{VK_FORMAT_R8_SINT}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 1u, 1u, "i32"}; } - case 7: { const std::array formats{VK_FORMAT_R16_UNORM}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 8: { const std::array formats{VK_FORMAT_R16_SNORM}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 9: { const std::array formats{VK_FORMAT_R16_USCALED}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 10: { const std::array formats{VK_FORMAT_R16_SSCALED}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 11: { const std::array formats{VK_FORMAT_R16_UINT}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 2u, 2u, "u32"}; } - case 12: { const std::array formats{VK_FORMAT_R16_SINT}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 2u, 2u, "i32"}; } - case 13: { const std::array formats{VK_FORMAT_R16_SFLOAT}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 14: { const std::array formats{VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 15: { const std::array formats{VK_FORMAT_R8_SNORM, VK_FORMAT_R8G8_SNORM}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 16: { const std::array formats{VK_FORMAT_R8_USCALED, VK_FORMAT_R8G8_USCALED}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 17: { const std::array formats{VK_FORMAT_R8_SSCALED, VK_FORMAT_R8G8_SSCALED}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 18: { const std::array formats{VK_FORMAT_R8_UINT, VK_FORMAT_R8G8_UINT}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 1u, 1u, "u32"}; } - case 19: { const std::array formats{VK_FORMAT_R8_SINT, VK_FORMAT_R8G8_SINT}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 1u, 1u, "i32"}; } - case 20: { const std::array formats{VK_FORMAT_R32_UINT}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 4u, 4u, "u32"}; } - case 21: { const std::array formats{VK_FORMAT_R32_SINT}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 4u, 4u, "i32"}; } - case 22: { const std::array formats{VK_FORMAT_R32_SFLOAT}; const auto count = std::min(attribute.components, 1u); return {formats[count - 1], count * 4u, 4u, "f32"}; } - case 23: { const std::array formats{VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 24: { const std::array formats{VK_FORMAT_R16_SNORM, VK_FORMAT_R16G16_SNORM}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 25: { const std::array formats{VK_FORMAT_R16_USCALED, VK_FORMAT_R16G16_USCALED}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 26: { const std::array formats{VK_FORMAT_R16_SSCALED, VK_FORMAT_R16G16_SSCALED}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 27: { const std::array formats{VK_FORMAT_R16_UINT, VK_FORMAT_R16G16_UINT}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 2u, 2u, "u32"}; } - case 28: { const std::array formats{VK_FORMAT_R16_SINT, VK_FORMAT_R16G16_SINT}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 2u, 2u, "i32"}; } - case 29: { const std::array formats{VK_FORMAT_R16_SFLOAT, VK_FORMAT_R16G16_SFLOAT}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 1: { const std::array formats{VK_FORMAT_R8_UNORM}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 2: { const std::array formats{VK_FORMAT_R8_SNORM}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 3: { const std::array formats{VK_FORMAT_R8_USCALED}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 4: { const std::array formats{VK_FORMAT_R8_SSCALED}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 5: { const std::array formats{VK_FORMAT_R8_UINT}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 1u, 1u, "u32"}; } + case 6: { const std::array formats{VK_FORMAT_R8_SINT}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 1u, 1u, "i32"}; } + case 7: { const std::array formats{VK_FORMAT_R16_UNORM}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 8: { const std::array formats{VK_FORMAT_R16_SNORM}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 9: { const std::array formats{VK_FORMAT_R16_USCALED}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 10: { const std::array formats{VK_FORMAT_R16_SSCALED}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 11: { const std::array formats{VK_FORMAT_R16_UINT}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 2u, 2u, "u32"}; } + case 12: { const std::array formats{VK_FORMAT_R16_SINT}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 2u, 2u, "i32"}; } + case 13: { const std::array formats{VK_FORMAT_R16_SFLOAT}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 14: { const std::array formats{VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 15: { const std::array formats{VK_FORMAT_R8_SNORM, VK_FORMAT_R8G8_SNORM}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 16: { const std::array formats{VK_FORMAT_R8_USCALED, VK_FORMAT_R8G8_USCALED}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 17: { const std::array formats{VK_FORMAT_R8_SSCALED, VK_FORMAT_R8G8_SSCALED}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 18: { const std::array formats{VK_FORMAT_R8_UINT, VK_FORMAT_R8G8_UINT}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 1u, 1u, "u32"}; } + case 19: { const std::array formats{VK_FORMAT_R8_SINT, VK_FORMAT_R8G8_SINT}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 1u, 1u, "i32"}; } + case 20: { const std::array formats{VK_FORMAT_R32_UINT}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 4u, 4u, "u32"}; } + case 21: { const std::array formats{VK_FORMAT_R32_SINT}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 4u, 4u, "i32"}; } + case 22: { const std::array formats{VK_FORMAT_R32_SFLOAT}; const auto count = std::min(components, 1u); return {formats[count - 1], count * 4u, 4u, "f32"}; } + case 23: { const std::array formats{VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 24: { const std::array formats{VK_FORMAT_R16_SNORM, VK_FORMAT_R16G16_SNORM}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 25: { const std::array formats{VK_FORMAT_R16_USCALED, VK_FORMAT_R16G16_USCALED}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 26: { const std::array formats{VK_FORMAT_R16_SSCALED, VK_FORMAT_R16G16_SSCALED}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 27: { const std::array formats{VK_FORMAT_R16_UINT, VK_FORMAT_R16G16_UINT}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 2u, 2u, "u32"}; } + case 28: { const std::array formats{VK_FORMAT_R16_SINT, VK_FORMAT_R16G16_SINT}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 2u, 2u, "i32"}; } + case 29: { const std::array formats{VK_FORMAT_R16_SFLOAT, VK_FORMAT_R16G16_SFLOAT}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 2u, 2u, "f32"}; } case 36: return {VK_FORMAT_B10G11R11_UFLOAT_PACK32, 4u, 4u, "f32"}; case 50: return {VK_FORMAT_A2B10G10R10_UNORM_PACK32, 4u, 4u, "f32"}; case 51: return {VK_FORMAT_A2B10G10R10_SNORM_PACK32, 4u, 4u, "f32"}; @@ -58,28 +60,28 @@ inline VertexFormat DecodeVertexFormat(const ShaderRecompiler::VertexAttribute& case 53: return {VK_FORMAT_A2B10G10R10_SSCALED_PACK32, 4u, 4u, "f32"}; case 54: return {VK_FORMAT_A2B10G10R10_UINT_PACK32, 4u, 4u, "u32"}; case 55: return {VK_FORMAT_A2B10G10R10_SINT_PACK32, 4u, 4u, "i32"}; - case 56: { const std::array formats{VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM, VK_FORMAT_R8G8B8_UNORM, VK_FORMAT_R8G8B8A8_UNORM}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 57: { const std::array formats{VK_FORMAT_R8_SNORM, VK_FORMAT_R8G8_SNORM, VK_FORMAT_R8G8B8_SNORM, VK_FORMAT_R8G8B8A8_SNORM}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 58: { const std::array formats{VK_FORMAT_R8_USCALED, VK_FORMAT_R8G8_USCALED, VK_FORMAT_R8G8B8_USCALED, VK_FORMAT_R8G8B8A8_USCALED}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 59: { const std::array formats{VK_FORMAT_R8_SSCALED, VK_FORMAT_R8G8_SSCALED, VK_FORMAT_R8G8B8_SSCALED, VK_FORMAT_R8G8B8A8_SSCALED}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 1u, 1u, "f32"}; } - case 60: { const std::array formats{VK_FORMAT_R8_UINT, VK_FORMAT_R8G8_UINT, VK_FORMAT_R8G8B8_UINT, VK_FORMAT_R8G8B8A8_UINT}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 1u, 1u, "u32"}; } - case 61: { const std::array formats{VK_FORMAT_R8_SINT, VK_FORMAT_R8G8_SINT, VK_FORMAT_R8G8B8_SINT, VK_FORMAT_R8G8B8A8_SINT}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 1u, 1u, "i32"}; } - case 62: { const std::array formats{VK_FORMAT_R32_UINT, VK_FORMAT_R32G32_UINT}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 4u, 4u, "u32"}; } - case 63: { const std::array formats{VK_FORMAT_R32_SINT, VK_FORMAT_R32G32_SINT}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 4u, 4u, "i32"}; } - case 64: { const std::array formats{VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32G32_SFLOAT}; const auto count = std::min(attribute.components, 2u); return {formats[count - 1], count * 4u, 4u, "f32"}; } - case 65: { const std::array formats{VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM, VK_FORMAT_R16G16B16_UNORM, VK_FORMAT_R16G16B16A16_UNORM}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 66: { const std::array formats{VK_FORMAT_R16_SNORM, VK_FORMAT_R16G16_SNORM, VK_FORMAT_R16G16B16_SNORM, VK_FORMAT_R16G16B16A16_SNORM}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 67: { const std::array formats{VK_FORMAT_R16_USCALED, VK_FORMAT_R16G16_USCALED, VK_FORMAT_R16G16B16_USCALED, VK_FORMAT_R16G16B16A16_USCALED}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 68: { const std::array formats{VK_FORMAT_R16_SSCALED, VK_FORMAT_R16G16_SSCALED, VK_FORMAT_R16G16B16_SSCALED, VK_FORMAT_R16G16B16A16_SSCALED}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 69: { const std::array formats{VK_FORMAT_R16_UINT, VK_FORMAT_R16G16_UINT, VK_FORMAT_R16G16B16_UINT, VK_FORMAT_R16G16B16A16_UINT}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 2u, 2u, "u32"}; } - case 70: { const std::array formats{VK_FORMAT_R16_SINT, VK_FORMAT_R16G16_SINT, VK_FORMAT_R16G16B16_SINT, VK_FORMAT_R16G16B16A16_SINT}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 2u, 2u, "i32"}; } - case 71: { const std::array formats{VK_FORMAT_R16_SFLOAT, VK_FORMAT_R16G16_SFLOAT, VK_FORMAT_R16G16B16_SFLOAT, VK_FORMAT_R16G16B16A16_SFLOAT}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 2u, 2u, "f32"}; } - case 72: { const std::array formats{VK_FORMAT_R32_UINT, VK_FORMAT_R32G32_UINT, VK_FORMAT_R32G32B32_UINT}; const auto count = std::min(attribute.components, 3u); return {formats[count - 1], count * 4u, 4u, "u32"}; } - case 73: { const std::array formats{VK_FORMAT_R32_SINT, VK_FORMAT_R32G32_SINT, VK_FORMAT_R32G32B32_SINT}; const auto count = std::min(attribute.components, 3u); return {formats[count - 1], count * 4u, 4u, "i32"}; } - case 74: { const std::array formats{VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32G32_SFLOAT, VK_FORMAT_R32G32B32_SFLOAT}; const auto count = std::min(attribute.components, 3u); return {formats[count - 1], count * 4u, 4u, "f32"}; } - case 75: { const std::array formats{VK_FORMAT_R32_UINT, VK_FORMAT_R32G32_UINT, VK_FORMAT_R32G32B32_UINT, VK_FORMAT_R32G32B32A32_UINT}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 4u, 4u, "u32"}; } - case 76: { const std::array formats{VK_FORMAT_R32_SINT, VK_FORMAT_R32G32_SINT, VK_FORMAT_R32G32B32_SINT, VK_FORMAT_R32G32B32A32_SINT}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 4u, 4u, "i32"}; } - case 77: { const std::array formats{VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32G32_SFLOAT, VK_FORMAT_R32G32B32_SFLOAT, VK_FORMAT_R32G32B32A32_SFLOAT}; const auto count = std::min(attribute.components, 4u); return {formats[count - 1], count * 4u, 4u, "f32"}; } + case 56: { const std::array formats{VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM, VK_FORMAT_R8G8B8_UNORM, VK_FORMAT_R8G8B8A8_UNORM}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 57: { const std::array formats{VK_FORMAT_R8_SNORM, VK_FORMAT_R8G8_SNORM, VK_FORMAT_R8G8B8_SNORM, VK_FORMAT_R8G8B8A8_SNORM}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 58: { const std::array formats{VK_FORMAT_R8_USCALED, VK_FORMAT_R8G8_USCALED, VK_FORMAT_R8G8B8_USCALED, VK_FORMAT_R8G8B8A8_USCALED}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 59: { const std::array formats{VK_FORMAT_R8_SSCALED, VK_FORMAT_R8G8_SSCALED, VK_FORMAT_R8G8B8_SSCALED, VK_FORMAT_R8G8B8A8_SSCALED}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 1u, 1u, "f32"}; } + case 60: { const std::array formats{VK_FORMAT_R8_UINT, VK_FORMAT_R8G8_UINT, VK_FORMAT_R8G8B8_UINT, VK_FORMAT_R8G8B8A8_UINT}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 1u, 1u, "u32"}; } + case 61: { const std::array formats{VK_FORMAT_R8_SINT, VK_FORMAT_R8G8_SINT, VK_FORMAT_R8G8B8_SINT, VK_FORMAT_R8G8B8A8_SINT}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 1u, 1u, "i32"}; } + case 62: { const std::array formats{VK_FORMAT_R32_UINT, VK_FORMAT_R32G32_UINT}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 4u, 4u, "u32"}; } + case 63: { const std::array formats{VK_FORMAT_R32_SINT, VK_FORMAT_R32G32_SINT}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 4u, 4u, "i32"}; } + case 64: { const std::array formats{VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32G32_SFLOAT}; const auto count = std::min(components, 2u); return {formats[count - 1], count * 4u, 4u, "f32"}; } + case 65: { const std::array formats{VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM, VK_FORMAT_R16G16B16_UNORM, VK_FORMAT_R16G16B16A16_UNORM}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 66: { const std::array formats{VK_FORMAT_R16_SNORM, VK_FORMAT_R16G16_SNORM, VK_FORMAT_R16G16B16_SNORM, VK_FORMAT_R16G16B16A16_SNORM}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 67: { const std::array formats{VK_FORMAT_R16_USCALED, VK_FORMAT_R16G16_USCALED, VK_FORMAT_R16G16B16_USCALED, VK_FORMAT_R16G16B16A16_USCALED}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 68: { const std::array formats{VK_FORMAT_R16_SSCALED, VK_FORMAT_R16G16_SSCALED, VK_FORMAT_R16G16B16_SSCALED, VK_FORMAT_R16G16B16A16_SSCALED}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 69: { const std::array formats{VK_FORMAT_R16_UINT, VK_FORMAT_R16G16_UINT, VK_FORMAT_R16G16B16_UINT, VK_FORMAT_R16G16B16A16_UINT}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 2u, 2u, "u32"}; } + case 70: { const std::array formats{VK_FORMAT_R16_SINT, VK_FORMAT_R16G16_SINT, VK_FORMAT_R16G16B16_SINT, VK_FORMAT_R16G16B16A16_SINT}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 2u, 2u, "i32"}; } + case 71: { const std::array formats{VK_FORMAT_R16_SFLOAT, VK_FORMAT_R16G16_SFLOAT, VK_FORMAT_R16G16B16_SFLOAT, VK_FORMAT_R16G16B16A16_SFLOAT}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 2u, 2u, "f32"}; } + case 72: { const std::array formats{VK_FORMAT_R32_UINT, VK_FORMAT_R32G32_UINT, VK_FORMAT_R32G32B32_UINT}; const auto count = std::min(components, 3u); return {formats[count - 1], count * 4u, 4u, "u32"}; } + case 73: { const std::array formats{VK_FORMAT_R32_SINT, VK_FORMAT_R32G32_SINT, VK_FORMAT_R32G32B32_SINT}; const auto count = std::min(components, 3u); return {formats[count - 1], count * 4u, 4u, "i32"}; } + case 74: { const std::array formats{VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32G32_SFLOAT, VK_FORMAT_R32G32B32_SFLOAT}; const auto count = std::min(components, 3u); return {formats[count - 1], count * 4u, 4u, "f32"}; } + case 75: { const std::array formats{VK_FORMAT_R32_UINT, VK_FORMAT_R32G32_UINT, VK_FORMAT_R32G32B32_UINT, VK_FORMAT_R32G32B32A32_UINT}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 4u, 4u, "u32"}; } + case 76: { const std::array formats{VK_FORMAT_R32_SINT, VK_FORMAT_R32G32_SINT, VK_FORMAT_R32G32B32_SINT, VK_FORMAT_R32G32B32A32_SINT}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 4u, 4u, "i32"}; } + case 77: { const std::array formats{VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32G32_SFLOAT, VK_FORMAT_R32G32B32_SFLOAT, VK_FORMAT_R32G32B32A32_SFLOAT}; const auto count = std::min(components, 4u); return {formats[count - 1], count * 4u, 4u, "f32"}; } default: throw std::runtime_error("AGC graphics: unsupported vertex format " + std::to_string(format)); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index bdee4b50f..d466793d3 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace AgcDriver::Graphics { @@ -16,6 +17,13 @@ bool sharedTiers() { return shared; } +bool exactSizes() { + static const bool exact = std::getenv("APS5_NO_BUFFER_CLASSES") != nullptr; + return exact; +} + +constexpr VkBufferUsageFlags deviceUsage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + } BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function("vkUnmapMemory")), destroyBuffer(context.Function("vkDestroyBuffer")), freeMemory(context.Function("vkFreeMemory")), maxSlots(std::max(1, std::min(MaxSlots(), context.limits.maxMemoryAllocationCount / 6u))) { @@ -47,21 +55,35 @@ void BufferPool::destroy(const BufferAllocation& allocation) noexcept { freeMemory(device, allocation.memory, nullptr); } -std::size_t BufferPool::Capacity(std::size_t bytes) { - static const bool exact = std::getenv("APS5_NO_BUFFER_CLASSES") != nullptr; +bool BufferPool::DeviceTiered(VkMemoryPropertyFlags properties) { + return (properties & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0 && (properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0 && DeviceBudget() != 0; +} + +std::size_t BufferPool::Capacity(std::size_t bytes, VkMemoryPropertyFlags properties) { constexpr std::size_t smallest = 256; - if (exact || bytes >= classLimit) return bytes; - return std::max(smallest, std::bit_ceil(bytes)); + if (exactSizes()) return bytes; + if (bytes < classLimit) return std::max(smallest, std::bit_ceil(bytes)); + if (!DeviceTiered(properties)) return bytes; + const auto step = std::bit_floor(bytes) >> deviceClassBits; + return (bytes + step - 1) / step * step; +} + +std::size_t BufferPool::NextCapacity(std::size_t capacity) { + return capacity < classLimit ? capacity * 2 : capacity + (std::bit_floor(capacity) >> deviceClassBits); +} + +VkBufferUsageFlags BufferPool::Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) { + return DeviceTiered(properties) ? usage | deviceUsage : usage; } BufferPool::Tier& BufferPool::tierFor(std::size_t capacity, VkMemoryPropertyFlags properties) { - if ((properties & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0 && (properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0 && DeviceBudget() != 0) return deviceTier; + if (DeviceTiered(properties)) return deviceTier; return !sharedTiers() && capacity < classLimit ? smallTier : largeTier; } std::optional BufferPool::Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - const auto capacity = Capacity(bytes); + const auto capacity = Capacity(bytes, properties); std::lock_guard lock(mutex); ++clock; if (profile) { @@ -73,7 +95,9 @@ std::optional BufferPool::Take(std::size_t bytes, VkBufferUsag } } auto& tier = tierFor(capacity, properties); - const auto found = tier.free.find({capacity, usage, properties}); + const auto largest = &tier == &deviceTier && !exactSizes() && bytes <= std::numeric_limits::max() / 2 ? std::max(capacity, bytes * 2) : capacity; + auto found = tier.free.end(); + for (auto size = capacity; size <= largest && found == tier.free.end(); size = NextCapacity(size)) found = tier.free.find({size, usage, properties}); if (found == tier.free.end() || found->second.empty()) { ++tier.misses; return std::nullopt; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp index 243402731..95cbf8470 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp @@ -27,6 +27,16 @@ std::uint32_t standardOffset(std::uint32_t x, std::uint32_t y, std::uint32_t ele } } +std::uint32_t standard64Extra(std::uint32_t x, std::uint32_t y, std::uint32_t elementBytes) { + switch (elementBytes) { + case 1u: return ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^ ((y << 6) & 0x1000u) ^ ((y << 7) & 0x4000u); + case 2u: return ((x << 7) & 0x2000u) ^ ((x << 8) & 0x8000u) ^ ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u); + case 4u: return ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^ ((y << 7) & 0x1000u) ^ ((y << 8) & 0x4000u); + case 8u: return ((x << 8) & 0x2000u) ^ ((x << 9) & 0x8000u) ^ ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u); + default: return ((x << 9) & 0x2000u) ^ ((x << 10) & 0x8000u) ^ ((y << 8) & 0x1000u) ^ ((y << 9) & 0x4000u); + } +} + } ColorTileMode DecodeColorTileMode(std::uint32_t attrib3) { @@ -35,7 +45,7 @@ ColorTileMode DecodeColorTileMode(std::uint32_t attrib3) { const auto mode = (attrib3 >> 14u) & 0x1fu; const auto fmaskMode = (attrib3 >> 19u) & 0x1fu; require(fmaskMode == 0 || fmaskMode == 0x18, "AGC graphics: unsupported color FMASK swizzle mode"); - require(mode == 0 || mode == 5 || mode == 0x1b, "AGC graphics: unsupported color tile mode"); + require(mode == 0 || mode == 5 || mode == 9 || mode == 0x1b, "AGC graphics: unsupported color tile mode"); return static_cast(mode); } @@ -70,18 +80,18 @@ const SwizzleTables& renderTargetTables(std::uint32_t bytesPerElement, std::uint return tables[index]; } -const SwizzleTables& standardTables(std::uint32_t bytesPerElement, std::uint32_t blockWidth, std::uint32_t blockHeight) { - static std::once_flag once[5]; - static SwizzleTables tables[5]; +const SwizzleTables& standardTables(std::uint32_t bytesPerElement, std::uint32_t blockWidth, std::uint32_t blockHeight, bool block64KB) { + static std::once_flag once[2][5]; + static SwizzleTables tables[2][5]; const auto index = static_cast(std::countr_zero(bytesPerElement)); - std::call_once(once[index], [&] { - auto& table = tables[index]; + std::call_once(once[block64KB][index], [&] { + auto& table = tables[block64KB][index]; table.x.resize(blockWidth); table.y.resize(blockHeight); - for (std::uint32_t x = 0; x < blockWidth; ++x) table.x[x] = standardOffset(x, 0, bytesPerElement); - for (std::uint32_t y = 0; y < blockHeight; ++y) table.y[y] = standardOffset(0, y, bytesPerElement); + for (std::uint32_t x = 0; x < blockWidth; ++x) table.x[x] = standardOffset(x, 0, bytesPerElement) ^ (block64KB ? standard64Extra(x, 0, bytesPerElement) : 0u); + for (std::uint32_t y = 0; y < blockHeight; ++y) table.y[y] = standardOffset(0, y, bytesPerElement) ^ (block64KB ? standard64Extra(0, y, bytesPerElement) : 0u); }); - return tables[index]; + return tables[block64KB][index]; } } @@ -108,13 +118,16 @@ ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height, yOffsets = tables.y.data(); break; } - case ColorTileMode::Standard4KB: { + case ColorTileMode::Standard4KB: + case ColorTileMode::Standard64KB: { const auto log2Bytes = static_cast(std::countr_zero(bytesPerElement)); - blockWidth = 1u << (6u - (log2Bytes + 1u) / 2u); - blockHeight = 1u << (6u - log2Bytes / 2u); + const bool block64KB = mode == ColorTileMode::Standard64KB; + const auto log2Side = block64KB ? 8u : 6u; + blockWidth = 1u << (log2Side - log2Bytes / 2u); + blockHeight = 1u << (log2Side - (log2Bytes + 1u) / 2u); pitch = (width + blockWidth - 1u) / blockWidth * blockWidth; paddedHeight = (height + blockHeight - 1u) / blockHeight * blockHeight; - const auto& tables = standardTables(bytesPerElement, blockWidth, blockHeight); + const auto& tables = standardTables(bytesPerElement, blockWidth, blockHeight, block64KB); xOffsets = tables.x.data(); yOffsets = tables.y.data(); break; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp index c6deb8e28..5e4729b1f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp @@ -392,13 +392,27 @@ std::size_t DccKeyBytes(std::uint64_t surfaceBytes) { return static_cast(surfaceBytes / KeyBytes); } +std::size_t DccKeyCount(TextureTileMode tileMode, std::uint32_t elementBytes, std::uint32_t width, std::uint32_t height, std::uint64_t surfaceBytes) { + const auto keys = DccKeyBytes(surfaceBytes); + if (tileMode != TextureTileMode::kR64KBX || elementBytes != 4) return keys; + constexpr std::uint64_t MetaBlockEdge = 512; + constexpr std::uint64_t MetaBlockBytes = 4096; + const auto console = (width + MetaBlockEdge - 1) / MetaBlockEdge * ((height + MetaBlockEdge - 1) / MetaBlockEdge) * MetaBlockBytes; + return std::max(keys, static_cast(console)); +} + +std::size_t DccKeyCount(const GuestTextureResource& surface, std::uint64_t surfaceBytes) { + const bool single = surface.mipCount == 1 && surface.baseArray == 0 && surface.depthOrLastArray == 0 && (surface.dimension == TextureDimension::k2D || surface.dimension == TextureDimension::k2DArray); + if (!single || !surface.dccPipeAligned || surface.tileMode != TextureTileMode::kR64KBX) return DccKeyBytes(surfaceBytes); + return DccKeyCount(surface.tileMode, BytesPerElement(surface.format), surface.width, surface.height, surfaceBytes); +} + namespace { // ReadDccKeys, saying in `memoized` whether the answer came from the pending-store memo rather // than the bytes. -DccKeys readDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes, bool& memoized) { +DccKeys readDccKeys(std::uint64_t metaAddress, std::size_t count, bool& memoized) { memoized = false; - const auto count = static_cast(surfaceBytes / KeyBytes); if (metaAddress == 0 || count == 0 || !GuestMemory::Accessible(reinterpret_cast(metaAddress), count)) return DccKeys::Unreadable; // A driver store still pending on the GPU is not in the bytes yet. if (const auto memo = MemoizedKeys(metaAddress, metaAddress + count)) { @@ -430,7 +444,7 @@ DccKeys readDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes, bool& DccKeys textureClearKeys(const GuestTextureResource& resource, std::uint64_t guestBytes, bool& memoized) { memoized = false; if (resource.dccAddress == 0) return DccKeys::Uncompressed; - const auto keys = readDccKeys(resource.dccAddress, guestBytes, memoized); + const auto keys = readDccKeys(resource.dccAddress, DccKeyBytes(guestBytes), memoized); if (keys == DccKeys::Uncompressed) return keys; if (IsConvertedTextureFormat(resource.format) && (keys == DccKeys::Clear0001 || keys == DccKeys::Clear1110)) throw std::runtime_error(std::string("AGC graphics: DCC clear code ") + DccKeysName(keys) + " of converted texture format " + std::to_string(resource.format) + " is not implemented"); std::byte probe[16]{}; @@ -444,15 +458,7 @@ DccKeys textureClearKeys(const GuestTextureResource& resource, std::uint64_t gue return keys; } -} - -DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { - bool memoized = false; - return readDccKeys(metaAddress, surfaceBytes, memoized); -} - -DccKeys CurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { - const auto count = static_cast(surfaceBytes / KeyBytes); +DccKeys currentDccKeys(std::uint64_t metaAddress, std::size_t count) { if (metaAddress != 0 && count != 0 && GuestMemory::GpuMutex().HeldByThisThread()) { if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->PendingWriteOverlaps(metaAddress, count)) { if (const auto keys = PendingStoreKeys(*recorder, metaAddress, count)) return *keys; @@ -460,7 +466,35 @@ DccKeys CurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { recorder->SyncThrough(metaAddress, count); } } - return ReadDccKeys(metaAddress, surfaceBytes); + bool memoized = false; + return readDccKeys(metaAddress, count, memoized); +} + +std::size_t StoredKeyCount(std::uint64_t metaAddress, std::uint64_t surfaceBytes, std::size_t keyCount) { + const auto keys = DccKeyBytes(surfaceBytes); + Require(keyCount >= keys, "a DCC key range covers fewer keys than one per 256 surface bytes"); + if (metaAddress == 0 || keyCount == 0) return 0; + if (GuestMemory::Accessible(reinterpret_cast(metaAddress), keyCount, true)) return keyCount; + if (keyCount == keys) return 0; + char message[256]; + std::snprintf(message, sizeof(message), "AGC graphics: DCC metadata 0x%llx is not writable over the 0x%zx key bytes of its surface (0x%zx at one per 256 bytes): metadata smaller than the console's DCC extent is not modeled", static_cast(metaAddress), keyCount, keys); + throw std::runtime_error(message); +} + +} + +DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { + bool memoized = false; + return readDccKeys(metaAddress, DccKeyBytes(surfaceBytes), memoized); +} + +DccKeys CurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { + return currentDccKeys(metaAddress, DccKeyBytes(surfaceBytes)); +} + +DccKeys CurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes, std::size_t keyCount) { + Require(keyCount >= DccKeyBytes(surfaceBytes), "a DCC key range covers fewer keys than one per 256 surface bytes"); + return currentDccKeys(metaAddress, keyCount); } bool IsDccClear(DccKeys keys) { @@ -468,8 +502,12 @@ bool IsDccClear(DccKeys keys) { } void MarkDccUncompressed(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { - const auto count = static_cast(surfaceBytes / KeyBytes); - if (metaAddress == 0 || count == 0 || !GuestMemory::Accessible(reinterpret_cast(metaAddress), count, true)) return; + MarkDccUncompressed(metaAddress, surfaceBytes, DccKeyBytes(surfaceBytes)); +} + +void MarkDccUncompressed(std::uint64_t metaAddress, std::uint64_t surfaceBytes, std::size_t keyCount) { + const auto count = StoredKeyCount(metaAddress, surfaceBytes, keyCount); + if (count == 0) return; // The bytes are trusted only when no recorded work writes them (lock-free, with or without // GuestMemory::GpuMutex); otherwise the store runs and waits through the flush hook. A pending // GPU store of the driver's own is no reason to skip: a title kernel recorded into the same @@ -479,8 +517,12 @@ void MarkDccUncompressed(std::uint64_t metaAddress, std::uint64_t surfaceBytes) } void MarkDccUncompressed(const Context& context, std::uint64_t metaAddress, std::uint64_t surfaceBytes) { - const auto count = static_cast(surfaceBytes / KeyBytes); - if (metaAddress == 0 || count == 0 || !GuestMemory::Accessible(reinterpret_cast(metaAddress), count, true)) return; + MarkDccUncompressed(context, metaAddress, surfaceBytes, DccKeyBytes(surfaceBytes)); +} + +void MarkDccUncompressed(const Context& context, std::uint64_t metaAddress, std::uint64_t surfaceBytes, std::size_t keyCount) { + const auto count = StoredKeyCount(metaAddress, surfaceBytes, keyCount); + if (count == 0) return; auto* recorder = CpuKeysOnly() || !GuestMemory::GpuMutex().HeldByThisThread() ? nullptr : Recorder::Active(); // The bytes are scanned only when no recorded work writes them: a pending clear kernel's keys are // not in memory yet, so the bytes could read as uncompressed while the kernel will store a clear diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index d346a19f2..7e94a8675 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -42,7 +42,7 @@ std::uint32_t GuestFormatFor(VkFormat format, std::uint32_t elementBytes) { // The color buffer as a single-mip 2D surface descriptor (tile mode SW_64KB_R_X). GuestTextureResource SurfaceForTarget(const ColorTarget& color) { - Require(color.tileMode == ColorTileMode::RenderTarget || color.tileMode == ColorTileMode::Standard4KB, "only 4 KiB standard and 64 KiB tiled color targets are resident"); + Require(color.tileMode != ColorTileMode::Linear, "linear color targets are not resident"); const bool chain = color.mipCount > 1; GuestTextureResource surface{}; surface.baseAddress = chain ? color.surfaceAddress : color.address; @@ -62,6 +62,7 @@ GuestTextureResource SurfaceForTarget(const ColorTarget& color) { surface.dstSelW = 7; surface.dccAddress = color.dccAddress; surface.dccAlphaOnMsb = color.dccAlphaOnMsb; + surface.dccPipeAligned = color.dccPipeAligned; return surface; } @@ -69,6 +70,11 @@ GuestTextureResource SurfaceForTarget(const ColorTarget& color) { namespace { +std::size_t colorKeyCount(const ColorTarget& color, std::size_t bytes) { + if (color.mipCount != 1 || color.depth != 1 || !color.dccPipeAligned) return DccKeyBytes(bytes); + return DccKeyCount(ColorTextureTileMode(color.tileMode), color.elementBytes, color.extent.width, color.extent.height, bytes); +} + std::array clearTexel(const ColorTarget& color, DccKeys keys) { std::array texel{}; const auto elementBytes = static_cast(color.elementBytes); @@ -90,16 +96,19 @@ bool clearToTexel(StorageTexture& image, const std::array& texel, return image.FillClear(std::span(pattern), StorageTexture::WholeImage, refusal); } +void writeTexels(const ColorTarget& color, const std::array& texel) { + const auto elementBytes = static_cast(color.elementBytes); + std::vector texels(color.bytes); + for (std::size_t offset = 0; offset + elementBytes <= texels.size(); offset += elementBytes) std::memcpy(texels.data() + offset, texel.data(), elementBytes); + GuestMemory::Write(color.address, texels); +} + void storeClearTexels(const Context& context, const ColorTarget& color, const std::array& texel) { StorageTexture::FlushPending(color.address, color.bytes, nullptr, "fast-clear materialization"); const auto keys = ReadDccKeys(color.dccAddress, color.bytes); if (!IsDccClear(keys)) return; - const auto current = keys == DccKeys::ClearRegister ? texel : clearTexel(color, keys); - const auto elementBytes = static_cast(color.elementBytes); - std::vector texels(color.bytes); - for (std::size_t offset = 0; offset + elementBytes <= texels.size(); offset += elementBytes) std::memcpy(texels.data() + offset, current.data(), elementBytes); - GuestMemory::Write(color.address, texels); - MarkDccUncompressed(context, color.dccAddress, color.bytes); + writeTexels(color, keys == DccKeys::ClearRegister ? texel : clearTexel(color, keys)); + MarkDccUncompressed(context, color.dccAddress, color.bytes, colorKeyCount(color, color.bytes)); } void materializeRegisterClear(const Context& context, const ColorTarget& color, StorageTexture& resident) { @@ -118,13 +127,31 @@ void materializeRegisterClear(const Context& context, const ColorTarget& color, } } if (cleared) { - MarkDccUncompressed(context, color.dccAddress, color.bytes); + MarkDccUncompressed(context, color.dccAddress, color.bytes, colorKeyCount(color, color.bytes)); return; } storeClearTexels(context, color, texel); resident.Refresh(); } +bool materializeCmaskClear(const Context& context, const ColorTarget& color, StorageTexture* resident) { + if (color.cmaskAddress == 0) return false; + const auto metadataBytes = static_cast(color.cmaskBytes) * 256u; + const auto state = CurrentDccKeys(color.cmaskAddress, metadataBytes); + if (state == DccKeys::Uncompressed) return false; + Require(color.dccAddress == 0, std::string("CMASK of a DCC color target that is not all expanded is not modeled (") + DccKeysName(state) + ")"); + Require(state == DccKeys::Clear0000, std::string("CMASK whose tiles are not all fast-cleared or all expanded is not modeled (") + DccKeysName(state) + ")"); + const auto texel = clearTexel(color, DccKeys::ClearRegister); + const char* refusal = nullptr; + if (resident == nullptr || !clearToTexel(*resident, texel, color.elementBytes, refusal)) { + StorageTexture::FlushPending(color.address, color.bytes, nullptr, "fast-clear materialization"); + writeTexels(color, texel); + if (resident != nullptr) resident->Refresh(); + } + MarkDccUncompressed(context, color.cmaskAddress, metadataBytes); + return true; +} + void imageBarrier(const Context& context, VkCommandBuffer commands, VkImage image, VkImageLayout oldLayout, VkImageLayout newLayout, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; barrier.srcAccessMask = sourceAccess; @@ -438,12 +465,12 @@ bool ValidationKey(const Context& context, std::span shade Require(shader.program != nullptr, "missing compiled shader"); const auto& program = *shader.program; const bool generated = state.rectList && (shader.stage == Stage::TessellationControl || shader.stage == Stage::TessellationEvaluation); - if (!generated && program.variantId == 0) { + if (!generated && program.PipelineVariantId() == 0) { keyed = false; break; } add(shader.stage); - add(generated ? std::uint64_t{0} : program.variantId); + add(generated ? std::uint64_t{0} : program.PipelineVariantId()); add(shader.pushConstantOffset); add(program.pushConstants.size()); add(program.bdaAbiVersion); @@ -452,6 +479,7 @@ bool ValidationKey(const Context& context, std::span shade add(attribute.location); add(attribute.components); add(attribute.fetchIndex); + add(attribute.formatComponents); // The data format bits of the V# decide the attribute's signature. add((attribute.resource.fields[3] >> 12u) & 0x7fu); } @@ -601,11 +629,12 @@ void CheckBufferAliases(std::span shaders, const ColorTarg } -std::array MeshIndexBufferDescriptor(const Pm4::DrawParameters& draw, std::uint64_t unreadAddress) { - const auto address = draw.indexed ? draw.indexAddress : unreadAddress; - const auto bytes = draw.indexed ? (static_cast(draw.indexCount) * draw.indexSize + 3u) & ~std::uint64_t{3} : 4u; - Require(address != 0 && bytes != 0 && bytes <= 0xffffffffu && (address >> 48u) == 0, "invalid mesh index buffer range"); +std::array MeshIndexBufferDescriptor(const Pm4::DrawParameters& draw) { constexpr std::uint32_t RawWord3 = 0x31016facu; + if (!draw.indexed) return {0u, 0u, 0u, RawWord3}; + const auto address = draw.indexAddress; + const auto bytes = (static_cast(draw.indexCount) * draw.indexSize + 3u) & ~std::uint64_t{3}; + Require(address != 0 && bytes != 0 && bytes <= 0xffffffffu && (address >> 48u) == 0, "invalid mesh index buffer range"); return {static_cast(address), static_cast(address >> 32u) & 0xffffu, static_cast(bytes), RawWord3}; } @@ -895,6 +924,7 @@ std::shared_ptr refreshResidentTarget(const Context& context, co std::lock_guard lock(reportMutex); if (reported.insert(color.address).second) std::fprintf(stderr, "[gpu] color target 0x%llx stays non-resident: %s\n", static_cast(color.address), error.what()); } + if (resident != nullptr) materializeCmaskClear(context, color, resident.get()); if (profile) { const auto lookupUs = std::chrono::duration(std::chrono::steady_clock::now() - lookupStart).count(); ++outcome.targetLookups; @@ -934,7 +964,7 @@ ResolvedResources resolveDrawResources(const Context& context, const State& stat // The render target and index ranges stay out of the key (see DrawResourceKey); a hit repeats // the alias checks instead. Debug aid: APS5_NO_DRAW_KEY_TRIM=1 keys them as before. static const bool trimKey = std::getenv("APS5_NO_DRAW_KEY_TRIM") == nullptr; - resolved.cacheable = recordable && !noDrawResourceCache && !noTextureCache && std::all_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->variantId != 0; }); + resolved.cacheable = recordable && !noDrawResourceCache && !noTextureCache && std::all_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->PipelineVariantId() != 0; }); if (resolved.cacheable) { resolved.contentKey = DrawResourceKey(context, shaders, state.color, draw.indexAddress, indexBytes, !trimKey); if (auto cached = SharedResourceCache().Find(resolved.contentKey)) { @@ -1182,6 +1212,7 @@ struct RecordedDraw { std::shared_ptr framebuffer; std::span targetViews; std::vector> targets; + std::vector> proxies; const IndirectRecord* indirect = nullptr; std::span moved; bool listed = false; @@ -1358,6 +1389,10 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam if (continued) { record.pipeline->Continue(commands, state); } else { + for (const auto& proxy : record.proxies) { + proxy->RecordAttachmentProxyLoad(commands, VK_IMAGE_LAYOUT_GENERAL); + countBarrier(2); + } const VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_INDEX_READ_BIT | VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT}; context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | inputs.shaderStages, 0, 1, &before, 0, nullptr, 0, nullptr); countBarrier(1); @@ -1391,7 +1426,16 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam // The pass stays open for the next draw of these attachments; the recorder ends it (and the // draw class range) before anything else is recorded. A draw that wrote memory owes the next // one a barrier, so its pass cannot be continued. - recorder->LeaveRenderPassOpen(passKey, drawTiming, !resources.WritesMemory()); + std::function storeProxies; + if (!continued && !record.proxies.empty()) { + storeProxies = [proxies = record.proxies](VkCommandBuffer passCommands) { + for (const auto& proxy : proxies) { + proxy->RecordAttachmentProxyStore(passCommands, VK_IMAGE_LAYOUT_GENERAL); + Recorder::CountBarriers(Recorder::CommandClass::Draw, 2); + } + }; + } + recorder->LeaveRenderPassOpen(passKey, drawTiming, !resources.WritesMemory(), std::move(storeProxies)); timer.phase(PhaseRecord); keepRecordedDraw(*recorder, record.resources, std::move(record.pipeline), std::move(record.framebuffer), inputs, std::move(record.targets), std::move(scratch), std::move(checkRecords), record.listed, record.completion, outcome); timer.phase(PhaseKeep); @@ -1480,6 +1524,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& // Resident target: the surface's cached storage image is attached directly and marked dirty // afterwards, so nothing is copied in or out per draw. std::shared_ptr resident; + bool proxied = false; // Linear pixels converted on the CPU, for targets the GPU detiler does not handle. std::unique_ptr transfer; // Guest bytes in host memory, and their device-local copies for the detiler. @@ -1508,7 +1553,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& timer.phase(PhaseSetup); // Debug aid: APS5_NO_RESIDENT_TARGETS=1 copies every target in and out again. static const bool residentTargets = std::getenv("APS5_NO_RESIDENT_TARGETS") == nullptr; - if ((binding.gpuTiling || (color.tileMode == ColorTileMode::Standard4KB && context.detiler != nullptr)) && residentTargets) { + if ((binding.gpuTiling || (color.tileMode != ColorTileMode::Linear && context.detiler != nullptr)) && residentTargets) { // The lookup refreshes the image on every draw (StorageTexture::Refresh: FlushPending, // CollectWrites over the target's pages, the DCC key scan of TextureClearKeys, then // UnchangedSince). The page walk is skipped while the worker's collect epoch lasts @@ -1524,9 +1569,11 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& } if (binding.resident != nullptr) { timer.phase(PhaseReadTarget); - targetViews.push_back(binding.resident->AttachmentView(color.format, color.mip, color.depthSlice)); + binding.proxied = AttachmentProxyFormat(context, color.format) != VK_FORMAT_UNDEFINED; + targetViews.push_back(binding.proxied ? binding.resident->AttachmentProxyView() : binding.resident->AttachmentView(color.format, color.mip, color.depthSlice)); continue; } + materializeCmaskClear(context, color, nullptr); Require(!color.mipTail, "rendering into a packed mip tail needs the resident image of its surface"); Require(color.depth == 1, "rendering into a 3D color target needs the resident image of its surface"); if (binding.gpuTiling) { @@ -1683,6 +1730,13 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& APS5_LOG_CHARS_OUT_DEBUG("Creating Pipeline"); // The state is copied only when a mask must change. auto masked = maskedState(state, inputs.fragmentOutputs); + if (std::any_of(targets.begin(), targets.end(), [](const TargetBinding& binding) { return binding.proxied; })) recipeOut = nullptr; + for (std::size_t index = 0; index < targets.size(); ++index) { + if (!targets[index].proxied) continue; + if (!masked.has_value()) masked = state; + masked->colors[index].format = AttachmentProxyFormat(context, state.colors[index].format); + masked->blends.at(state.colors[index].exportIndex).colorWriteMask &= VK_COLOR_COMPONENT_R_BIT; + } const State& pipelineState = masked.has_value() ? *masked : state; auto pipeline = CachedPipeline(context, pipelineState, inputs.vertexInput, *resources, shaders, lean ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); // Resident targets' views are stable while their storage image lives, so the framebuffer is @@ -1702,6 +1756,9 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& record.framebuffer = framebuffer; record.targetViews = targetViews; record.targets = owners; + for (const auto& binding : targets) { + if (binding.proxied) record.proxies.push_back(binding.resident); + } record.indirect = args != nullptr ? &indirect : nullptr; record.listed = listed; record.completion = completion; @@ -1800,6 +1857,11 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& rewritten = recordIndirectArguments(context, commands, recorder, recorded, indirect, scratch, argumentBuffer, argumentOffset, countBarrier); } for (auto& binding : targets) { + if (binding.proxied) { + binding.resident->RecordAttachmentProxyLoad(commands, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); + countBarrier(2); + continue; + } if (binding.resident != nullptr) { imageBarrier(context, commands, binding.resident->Image(), VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT); countBarrier(); @@ -1835,6 +1897,11 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(commands); APS5_LOG_CHARS_OUT_DEBUG("Render pass ended"); for (auto& binding : targets) { + if (binding.proxied) { + binding.resident->RecordAttachmentProxyStore(commands, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); + countBarrier(2); + continue; + } if (binding.resident != nullptr) { imageBarrier(context, commands, binding.resident->Image(), VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT); countBarrier(); @@ -1930,7 +1997,8 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& if (binding.gpuTiling) GuestMemory::WriteChanged(binding.color.address, binding.tiled->Bytes(), binding.original); else WriteColorTarget(binding.color, binding.transfer->Bytes()); // The stored texels are the whole target now, so later reads must see them rather than a fast clear. - MarkDccUncompressed(binding.color.dccAddress, ColorTargetLayout(binding.color.extent.width, binding.color.extent.height, binding.color.tileMode, binding.color.elementBytes).Bytes()); + const auto targetBytes = ColorTargetLayout(binding.color.extent.width, binding.color.extent.height, binding.color.tileMode, binding.color.elementBytes).Bytes(); + MarkDccUncompressed(binding.color.dccAddress, targetBytes, colorKeyCount(binding.color, targetBytes)); } timer.phase(PhaseWriteBack); if (profile && traceDraws) { @@ -2067,28 +2135,39 @@ DrawRecipeOutcome DrawWithRecipe(const Context& context, const State& state, con return result; } +namespace { + +std::shared_ptr metadataPassResident(const Context& context, const ColorTarget& color) { + if (color.tileMode == ColorTileMode::Linear || context.detiler == nullptr) return nullptr; + std::shared_ptr resident; + try { + resident = CachedStorageSurface(context, SurfaceForTarget(color)); + } catch (const std::exception&) { + return nullptr; + } + return resident != nullptr && resident->GuestBytes() == color.bytes ? resident : nullptr; +} + +} + void RunColorMetadataPass(const Context& context, const ColorMetadataPass& pass) { for (const auto& color : pass.targets) { + if (color.cmaskAddress != 0) { + const auto resident = metadataPassResident(context, color); + if (materializeCmaskClear(context, color, resident.get()) && resident != nullptr) resident->MarkDirty(); + } if (color.dccAddress == 0) continue; auto keys = CurrentDccKeys(color.dccAddress, color.bytes); if (keys == DccKeys::Uncompressed) continue; Require(IsDccClear(keys), std::string("CB metadata pass over DCC keys that are ") + DccKeysName(keys) + " (per-block metadata is not modeled)"); const auto texel = clearTexel(color, keys); - std::shared_ptr resident; - if ((color.tileMode == ColorTileMode::RenderTarget || color.tileMode == ColorTileMode::Standard4KB) && context.detiler != nullptr) { - try { - resident = CachedStorageSurface(context, SurfaceForTarget(color)); - } catch (const std::exception&) { - resident = nullptr; - } - if (resident != nullptr && resident->GuestBytes() != color.bytes) resident = nullptr; - } + const auto resident = metadataPassResident(context, color); if (resident != nullptr) { const char* refusal = nullptr; const bool current = keys == DccKeys::ClearRegister ? clearToTexel(*resident, texel, color.elementBytes, refusal) : StorageTexture::FindPending(color.address, color.bytes) == resident || resident->UploadedKeys() == keys; if (current) { resident->MarkDirty(); - MarkDccUncompressed(context, color.dccAddress, color.bytes); + MarkDccUncompressed(context, color.dccAddress, color.bytes, colorKeyCount(color, color.bytes)); continue; } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 46848056b..2205fd49e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -36,6 +36,8 @@ #include #include #include +#include +#include namespace AgcDriver::Graphics { @@ -83,9 +85,22 @@ struct GuestBufferMemory::AddressSpace { std::uint64_t serial = 0; GuestAllocations::Lease lease; std::vector base; + struct HeapRun { + std::uint64_t begin; + std::uint64_t bytes; + std::size_t first; + std::size_t last; + }; + bool mirrored = false; + std::vector writableMirrors; + std::vector heapMirrors; + std::vector heapRuns; + std::unique_ptr[]> verified; std::vector copied; // The BDA table entries of `base`, in its order. std::vector ranges; + mutable std::mutex tableMutex; + mutable std::map>, std::pair>>> writeTables; }; namespace { @@ -285,8 +300,9 @@ void decideImportWatch(const Context& context, HostImports& state) { if (state.watchDevice == context.device) return; #ifdef _WIN32 state.watchDevice = context.device; - state.unwatchImports = false; + state.unwatchImports = true; state.unwatchDmaBufImports = false; + if (context.hostImportAlignment != 0 && GuestMemory::WriteWatched()) std::fprintf(stderr, "[write-watch] Windows direct host imports use comparisons; separate shared aliases retain guest write tracking\n"); #else const auto request = importWatchRequest(); state.watchDevice = context.device; @@ -395,6 +411,11 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s } entry.alias = GuestArena::GuestArenaMapAlias_nid_postfix(static_cast(base), static_cast(bytes)); } +#else + if (!GuestMemory::Accessible(reinterpret_cast(base), static_cast(bytes), true)) { + state.failed.insert(base); + return nullptr; + } #endif decideImportWatch(context, state); VkResult result = VK_SUCCESS; @@ -404,6 +425,9 @@ const HostImport* importAllocation(const Context& context, HostImports& state, s return step == nullptr; }); entry.unwatched = step == nullptr && (entry.dmaBuf ? state.unwatchDmaBufImports : state.unwatchImports); +#ifdef _WIN32 + if (entry.alias != nullptr) entry.unwatched = false; +#endif if (entry.unwatched) GuestMemory::Unwatch(base, bytes); if (step != nullptr) { #ifdef _WIN32 @@ -525,6 +549,8 @@ struct ImageMirrors { std::uint64_t builds = 0; std::uint64_t heapRefills = 0; std::uint64_t heapUnwatched = 0; + std::uint64_t heapImported = 0; + std::uint64_t heapImportedBytes = 0; std::uint64_t subranges = 0; std::uint64_t rebuilds = 0; std::uint64_t refreshes = 0; @@ -533,6 +559,11 @@ struct ImageMirrors { // Refreshes that had to wait for recorded work writing the range. std::uint64_t syncs = 0; std::uint64_t serials = 0; + std::uint64_t version = 0; + std::uint64_t pinnedSpace = 0; + std::uint64_t pinnedVersion = 0; + std::uint64_t sweeps = 0; + std::uint64_t heapChecks = 0; std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); }; @@ -586,12 +617,26 @@ std::uint64_t heapMirrorBudget() { return bytes; } -[[noreturn]] void heapMirrorFatal(const GuestAllocations::Range& range, std::uint64_t held, const char* reason) { - std::fprintf(stderr, "FATAL: heap mirror of 0x%llx+0x%llx: %s; heap mirrors hold %llu MiB of %llu MiB (APS5_HEAP_MIRROR_MIB)\n", static_cast(range.address), static_cast(range.bytes), reason, static_cast(held >> 20u), static_cast(heapMirrorBudget() >> 20u)); - std::fflush(stderr); - std::abort(); +std::string heapMirrorError(const GuestAllocations::Range& range, std::uint64_t held, std::string_view reason) { + constexpr std::string_view prefix = "AGC graphics: "; + if (reason.starts_with(prefix)) reason.remove_prefix(prefix.size()); + char where[64]; + std::snprintf(where, sizeof(where), "heap mirror of 0x%llx+0x%llx: ", static_cast(range.address), static_cast(range.bytes)); + char holding[96]; + std::snprintf(holding, sizeof(holding), "; heap mirrors hold %llu MiB of %llu MiB (APS5_HEAP_MIRROR_MIB)", static_cast(held >> 20u), static_cast(heapMirrorBudget() >> 20u)); + return std::string(prefix) + where + std::string(reason) + holding; } +std::uint64_t heapMirrorsBeside(const GuestAllocations::Range& range) { + auto& state = Mirrors(); + std::lock_guard lock(state.mutex); + std::uint64_t held = state.heapBytes; + if (const auto found = state.entries.find(range.address); found != state.entries.end() && found->second->heap) held -= found->second->bytes; + return held; +} + +void sweepMirrors(); + bool sameRange(const std::weak_ptr& mirrored, const std::shared_ptr& range) { return !mirrored.owner_before(range) && !range.owner_before(mirrored); } @@ -767,11 +812,30 @@ void refreshMirror(ImageMirror& mirror, std::uint64_t address, std::uint64_t byt compareBlocks(blocks); } -void refreshHeapMirrors(std::vector& mirrors, std::vector& blocks) { +void appendChangedBlocks(std::span mirrors, std::uint64_t generation, std::vector& blocks) { constexpr std::uint64_t block = 65536; - std::sort(mirrors.begin(), mirrors.end(), [](const ImageMirror* left, const ImageMirror* right) { return left->base < right->base; }); thread_local std::vector* changedSlot = nullptr; auto& changed = ShaderRecompiler::ThreadOwned(changedSlot); + Mirrors().heapChecks += mirrors.size(); + for (auto* const pointer : mirrors) { + auto& mirror = *pointer; + changed.assign(mirror.generations.size(), 0); + GuestMemory::ChangedBlocks(mirror.base, static_cast(mirror.bytes), mirror.generations, changed); + const auto aligned = mirror.base / block * block; + const auto end = mirror.base + mirror.bytes; + bool refilled = false; + for (std::size_t at = 0; at < changed.size(); ++at) { + if (changed[at] == 0) continue; + const auto from = std::max(mirror.base, aligned + at * block); + blocks.push_back({&mirror, from, static_cast(std::min(end, aligned + (at + 1) * block) - from), generation}); + refilled = true; + } + if (refilled) ++Mirrors().heapRefills; + } +} + +void refreshHeapMirrors(std::vector& mirrors, std::vector& blocks) { + std::sort(mirrors.begin(), mirrors.end(), [](const ImageMirror* left, const ImageMirror* right) { return left->base < right->base; }); for (std::size_t first = 0; first < mirrors.size();) { auto last = first + 1; while (last < mirrors.size() && mirrors[last]->base == mirrors[last - 1]->base + mirrors[last - 1]->bytes) ++last; @@ -780,25 +844,45 @@ void refreshHeapMirrors(std::vector& mirrors, std::vector(bytes)); Require(generation != 0, "a heap mirror's range is no longer write-watched"); - for (auto index = first; index < last; ++index) { - auto& mirror = *mirrors[index]; - changed.assign(mirror.generations.size(), 0); - GuestMemory::ChangedBlocks(mirror.base, static_cast(mirror.bytes), mirror.generations, changed); - const auto aligned = mirror.base / block * block; - const auto end = mirror.base + mirror.bytes; - bool refilled = false; - for (std::size_t at = 0; at < changed.size(); ++at) { - if (changed[at] == 0) continue; - const auto from = std::max(mirror.base, aligned + at * block); - blocks.push_back({&mirror, from, static_cast(std::min(end, aligned + (at + 1) * block) - from), generation}); - refilled = true; - } - if (refilled) ++Mirrors().heapRefills; - } + appendChangedBlocks(std::span(mirrors).subspan(first, last - first), generation, blocks); first = last; } } +void planSpaceMirrors(GuestBufferMemory::AddressSpace& space) { + for (std::size_t index = 0; index < space.base.size(); ++index) { + const auto& region = space.base[index]; + if (region.mirror == nullptr) continue; + space.mirrored = true; + if (!region.mirror->heap) { + if (region.mirror->writable) space.writableMirrors.push_back(index); + continue; + } + auto* mirror = region.mirror.get(); + if (space.heapRuns.empty() || mirror->base != space.heapMirrors.back()->base + space.heapMirrors.back()->bytes) space.heapRuns.push_back({mirror->base, 0, space.heapMirrors.size(), 0}); + space.heapMirrors.push_back(mirror); + auto& run = space.heapRuns.back(); + run.bytes = mirror->base + mirror->bytes - run.begin; + run.last = space.heapMirrors.size(); + } + space.verified = std::make_unique[]>(space.heapRuns.size()); +} + +std::vector> refreshHeapRuns(const GuestBufferMemory::AddressSpace& space, std::vector& blocks) { + std::vector> checked; + checked.reserve(space.heapRuns.size()); + for (std::size_t index = 0; index < space.heapRuns.size(); ++index) { + const auto& run = space.heapRuns[index]; + prepareRange(run.begin, run.bytes); + const auto generation = GuestMemory::CollectWrites(run.begin, static_cast(run.bytes)); + Require(generation != 0, "a heap mirror's range is no longer write-watched"); + const auto since = space.verified[index].load(std::memory_order_relaxed); + if (since == 0 || !GuestMemory::UnchangedSince(run.begin, static_cast(run.bytes), since)) appendChangedBlocks(std::span(space.heapMirrors).subspan(run.first, run.last - run.first), generation, blocks); + checked.emplace_back(index, generation); + } + return checked; +} + // The mirror for a leased image range: the existing one, or a new one when none exists or the // registered Range object changed (a protection change). Null when the range cannot be mirrored. An // existing writable mirror's blocks are appended to `blocks` for the caller's one compare of every @@ -812,6 +896,7 @@ std::shared_ptr acquireMirror(const Context& context, const std::sh state.failed.clear(); state.heapBytes = 0; state.device = context.device; + ++state.version; } if (state.failed.contains(range->address)) return nullptr; const auto found = state.entries.find(range->address); @@ -833,19 +918,20 @@ std::shared_ptr acquireMirror(const Context& context, const std::sh const auto generation = GuestMemory::CollectWrites(range->address, range->bytes); constexpr std::uint64_t block = 65536; mirror->generations.assign(static_cast(((range->address + range->bytes + block - 1) / block) - range->address / block), generation); - std::lock_guard lock(state.mutex); if (generation == 0) { + std::lock_guard lock(state.mutex); ++state.heapUnwatched; return nullptr; } - std::uint64_t held = state.heapBytes; - if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) held -= found->second->bytes; - if (held + range->bytes > heapMirrorBudget()) heapMirrorFatal(*range, held, "past the budget"); + if (heapMirrorsBeside(*range) + range->bytes > heapMirrorBudget()) { + sweepMirrors(); + if (const auto held = heapMirrorsBeside(*range); held + range->bytes > heapMirrorBudget()) throw std::runtime_error(heapMirrorError(*range, held, "past the budget")); + } } try { mirror->buffer = std::make_shared(context, range->bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); } catch (const std::runtime_error& error) { - if (heap) heapMirrorFatal(*range, state.heapBytes, error.what()); + if (heap) throw std::runtime_error(heapMirrorError(*range, heapMirrorsBeside(*range), error.what())); std::fprintf(stderr, "[gpu] image mirror of 0x%llx+0x%llx failed: %s; falling back to copies\n", static_cast(range->address), static_cast(range->bytes), error.what()); std::lock_guard lock(state.mutex); state.failed.insert(range->address); @@ -865,6 +951,7 @@ std::shared_ptr acquireMirror(const Context& context, const std::sh if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) state.heapBytes -= found->second->bytes; if (heap) state.heapBytes += mirror->bytes; state.entries[range->address] = mirror; + ++state.version; return mirror; } @@ -882,15 +969,44 @@ std::shared_ptr findMirror(const Context& context, std::uint64_t be // Drops mirrors whose registered range is gone (a later build made a new one for the replacement). void sweepMirrors() { + const auto space = Spaces().current.load(); auto& state = Mirrors(); std::lock_guard lock(state.mutex); + if (space != nullptr && state.pinnedSpace == space->serial && state.pinnedVersion == state.version) return; + ++state.sweeps; + bool pinned = space != nullptr; + std::size_t leased = 0; for (auto it = state.entries.begin(); it != state.entries.end();) { if (!it->second->range.expired()) { + if (pinned) { + while (leased < space->lease.size() && space->lease[leased]->address < it->first) ++leased; + pinned = leased < space->lease.size() && sameRange(it->second->range, space->lease[leased]); + } ++it; continue; } if (it->second->heap) state.heapBytes -= it->second->bytes; it = state.entries.erase(it); + ++state.version; + } + state.pinnedSpace = pinned ? space->serial : 0; + state.pinnedVersion = state.version; +} + +void releaseImportedMirrors(const Context& context, const std::vector& imported) { + if (imported.empty()) return; + std::vector> released; + auto& state = Mirrors(); + std::lock_guard lock(state.mutex); + if (state.device != context.device || state.entries.empty()) return; + for (const auto base : imported) { + const auto found = state.entries.find(base); + if (found == state.entries.end() || !found->second->heap) continue; + state.heapBytes -= found->second->bytes; + ++state.heapImported; + state.heapImportedBytes += found->second->bytes; + released.push_back(std::move(found->second)); + state.entries.erase(found); } } @@ -916,7 +1032,7 @@ void reportMirrors() { } heapBytes = state.heapBytes; } - AgcDriver::ProfilePrint_nid_no_patch("[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable, %zu heap %.1f MiB), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs; heap refills %llu, unwatched heaps %llu\n", count, bytes / 1048576.0, writable, heaps, heapBytes / 1048576.0, static_cast(state.builds), static_cast(state.subranges), static_cast(state.rebuilds), static_cast(state.refreshes), static_cast(state.blocksCompared), static_cast(state.blocksCopied), static_cast(state.syncs), static_cast(state.heapRefills), static_cast(state.heapUnwatched)); + AgcDriver::ProfilePrint_nid_no_patch("[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable, %zu heap %.1f MiB), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs; heap refills %llu, unwatched heaps %llu, %llu heap mirrors released for imports (%.1f MiB)\n", count, bytes / 1048576.0, writable, heaps, heapBytes / 1048576.0, static_cast(state.builds), static_cast(state.subranges), static_cast(state.rebuilds), static_cast(state.refreshes), static_cast(state.blocksCompared), static_cast(state.blocksCopied), static_cast(state.syncs), static_cast(state.heapRefills), static_cast(state.heapUnwatched), static_cast(state.heapImported), state.heapImportedBytes / 1048576.0); } // Deferred lease release. The lease an address-based build takes (AcquireRegistered) is dropped by its @@ -1094,7 +1210,7 @@ std::string AddressCopyOverflow(std::vector copies, std::uint64_t l MirrorStats MirrorCounters() { auto& state = Mirrors(); std::lock_guard lock(state.mutex); - MirrorStats stats{0, state.heapBytes, state.rebuilds, state.blocksCopied, state.heapRefills}; + MirrorStats stats{0, state.heapBytes, state.rebuilds, state.blocksCopied, state.heapRefills, state.sweeps, state.heapChecks}; for (const auto& [base, mirror] : state.entries) stats.heapMirrors += mirror->heap ? 1 : 0; return stats; } @@ -1110,6 +1226,25 @@ void ClearImageMirrors(VkDevice device) { state.heapBytes = 0; state.device = VK_NULL_HANDLE; } + Spaces().current.store(nullptr); +} + +void ClearHostImports(VkDevice device) { + auto& state = Imports(); + std::lock_guard lock(state.mutex); + if (state.device != device) return; + for (const auto& [address, entry] : state.imports) { + state.destroyBuffer(state.device, entry.buffer, nullptr); + state.freeMemory(state.device, entry.memory, nullptr); +#ifdef _WIN32 + GuestArena::GuestArenaUnmapAlias_nid_postfix(entry.alias); +#endif + } + state.imports.clear(); + state.failed.clear(); + state.device = VK_NULL_HANDLE; + state.refreshedGeneration = 0; + ++state.epoch; } #ifndef _WIN32 @@ -1364,6 +1499,7 @@ void GuestBufferMemory::AcquireRegistered() { std::vector blocks; std::vector heaps; bool mirrored = false; + bool served = false; auto& spaces = Spaces(); // Read before the lease below is acquired: a mutation ending in between leaves the space // stale by its generation, never newer than it (see AddressSpace). @@ -1391,11 +1527,11 @@ void GuestBufferMemory::AcquireRegistered() { // an existing mirror. The preparation may wait for recorded work, whose completions can // retire an import the space's regions point at: UploadFinish re-checks the epoch // before any of them is dereferenced. - for (const auto& region : space->base) { - if (region.mirror == nullptr) continue; - mirrored = true; - if (region.mirror->heap) heaps.push_back(region.mirror.get()); - else if (region.mirror->writable && prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin); + served = true; + mirrored = space->mirrored; + for (const auto index : space->writableMirrors) { + const auto& region = space->base[index]; + if (prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin); } lap(timing.mirrorsUs, at); } @@ -1420,6 +1556,7 @@ void GuestBufferMemory::AcquireRegistered() { // must not happen under it. regions.reserve(lease.size()); std::vector copies; + std::vector imported; for (const auto& range : lease) { if (!range->readable) continue; validate(range->address, range->bytes); @@ -1436,6 +1573,7 @@ void GuestBufferMemory::AcquireRegistered() { // Reused by Upload while no import was dropped since (see importsEpoch). region.direct = entry; regions.push_back(std::move(region)); + imported.push_back(range->address); continue; } if (mirrorsEnabled()) { @@ -1458,6 +1596,7 @@ void GuestBufferMemory::AcquireRegistered() { const char* reason = !mirrorsEnabled() ? "mirrors disabled by APS5_NO_LEASE_MIRROR" : copied.sparse ? "unreadable pages" : !range->releasable ? "image mirror refused" : "outside the write-watched arena"; copies.push_back({copied.begin, copied.end, committed, reason}); } + releaseImportedMirrors(context, imported); regionsSorted = sortedLookup; static const std::uint64_t copyLimit = [] { const char* value = std::getenv("APS5_ADDRESS_COPY_MAX_MIB"); @@ -1507,16 +1646,20 @@ void GuestBufferMemory::AcquireRegistered() { } regions = std::move(extras); lease.clear(); + planSpaceMirrors(*built); space = std::move(built); spaces.droppedByWaiter.store(false, std::memory_order_relaxed); spaces.current.store(space); } } } - refreshHeapMirrors(heaps, blocks); + std::vector> checked; + if (served) checked = refreshHeapRuns(*space, blocks); + else refreshHeapMirrors(heaps, blocks); lap(timing.mirrorsUs, at); const auto copiedBefore = Mirrors().blocksCopied; compareBlocks(blocks); + for (const auto& [run, collected] : checked) space->verified[run].store(collected, std::memory_order_relaxed); lap(timing.compareUs, at); if (profile) { timing.blocksCompared += blocks.size(); @@ -1604,7 +1747,9 @@ const GuestBufferMemory::Region* GuestBufferMemory::owner(std::uint64_t address) void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic) { validate(address, bytes); - switch (baseOverlap(address, address + bytes, nullptr)) { + const auto begin = address & ~std::uint64_t{3}; + const auto end = begin == address ? address + bytes : (address + bytes + 3) & ~std::uint64_t{3}; + switch (baseOverlap(begin, end, nullptr)) { case BaseOverlap::Inside: return; case BaseOverlap::Partial: @@ -1616,9 +1761,9 @@ void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t b } // `writable` here means the bytes come from live guest memory (not a snapshot), whether or not // the shader stores to them; what is written back is decided by Writes() alone. - Region region{address, address + bytes, true, {}, nullptr}; + Region region{begin, end, true, {}, nullptr}; region.atomic = atomic; - auto committed = GuestMemory::DescribeCommitted(address, bytes); + auto committed = GuestMemory::DescribeCommitted(begin, static_cast(end - begin)); if (!committed.whole) { // A GPU heap bound whole while the guest commits its pages on demand, or a descriptor left region.sparse = true; @@ -1934,6 +2079,7 @@ void GuestBufferMemory::UploadPrepare(bool addressable) { previous.hostBacked = true; } mergeBacked(previous, region); + if (region.end > previous.end) previous.direct = nullptr; previous.atomic = previous.atomic || region.atomic; // A range starting before the mirror (only possible after the swap) keeps its prefix; the // earlier merged region ends at or before it, so the merged list stays sorted. @@ -2460,11 +2606,18 @@ VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std: const auto base = region.direct != nullptr ? region.direct->base : region.mirror != nullptr ? region.mirror->base : region.begin; const auto offset = address - base; Require(context.limits.minStorageBufferOffsetAlignment != 0, "no storage buffer offset alignment"); - adjustment = static_cast(offset % context.limits.minStorageBufferOffsetAlignment); - Require(adjustment % 4 == 0, "guest buffer view off the storage buffer offset alignment is not DWORD aligned"); - Require(bytes + adjustment <= context.limits.maxStorageBufferRange, "guest buffer view exceeds descriptor range limit"); + Require(base % 4 == 0, "a guest buffer view in a GPU owner that does not start at a DWORD boundary is not implemented"); + adjustment = static_cast(offset % std::max(context.limits.minStorageBufferOffsetAlignment, 4)); + const auto range = ViewBytes(bytes, adjustment); + const auto end = region.direct != nullptr ? region.direct->base + region.direct->bytes : region.end; + Require(address - adjustment + range <= end, "guest buffer view exceeds its GPU owner"); + Require(range <= context.limits.maxStorageBufferRange, "guest buffer view exceeds descriptor range limit"); const auto handle = region.direct != nullptr ? region.direct->buffer : region.mirror != nullptr ? region.mirror->buffer->Handle() : region.buffer->Handle(); - return {handle, offset - adjustment, bytes + adjustment}; + return {handle, offset - adjustment, range}; +} + +std::uint64_t GuestBufferMemory::ViewBytes(std::uint64_t bytes, std::uint32_t adjustment) { + return adjustment % 4 == 0 ? bytes + adjustment : 4 * (adjustment / 4 + bytes / 4 + 1); } ShaderRecompiler::BdaAbi::Range GuestBufferMemory::addressRange(const Region& region) { @@ -2480,19 +2633,70 @@ std::vector GuestBufferMemory::AddressRanges() std::vector result; result.reserve(regions.size() + (space != nullptr ? space->ranges.size() : 0)); for (const auto& region : regions) result.push_back(addressRange(region)); - if (space == nullptr) return result; - // The table is searched by address (the recompiler's lookup bisects it): both lists are sorted - // and disjoint, so a merge keeps it so. + if (space != nullptr) { + std::vector combined; + combined.reserve(result.size() + space->ranges.size()); + std::merge(space->ranges.begin(), space->ranges.end(), result.begin(), result.end(), std::back_inserter(combined), [](const auto& left, const auto& right) { return left.begin < right.begin; }); + result = std::move(combined); + } + if (writes.empty()) return result; + auto sortedWrites = writes; + std::sort(sortedWrites.begin(), sortedWrites.end()); + std::vector> intervals; + for (const auto& interval : sortedWrites) { + if (!intervals.empty() && interval.first <= intervals.back().second) intervals.back().second = std::max(intervals.back().second, interval.second); + else intervals.push_back(interval); + } std::vector merged; - merged.reserve(result.size() + space->ranges.size()); - std::merge(space->ranges.begin(), space->ranges.end(), result.begin(), result.end(), std::back_inserter(merged), [](const auto& left, const auto& right) { return left.begin < right.begin; }); + merged.reserve(result.size() + intervals.size() * 2u); + std::size_t firstWrite = 0; + for (const auto& range : result) { + if ((range.permissions & ShaderRecompiler::BdaAbi::Write) != 0u) { + merged.push_back(range); + continue; + } + while (firstWrite < intervals.size() && intervals[firstWrite].second <= range.begin) ++firstWrite; + auto cursor = range.begin; + const auto append = [&](std::uint64_t end, bool written) { + if (cursor == end) return; + auto part = range; + part.begin = cursor; + part.end = end; + part.deviceAddress += cursor - range.begin; + if (written) { + const auto* region = owner(cursor); + Require(region != nullptr && region->writable && (region->mirror == nullptr || region->mirror->writable), "runtime buffer write has no writable GPU owner"); + part.permissions |= ShaderRecompiler::BdaAbi::Write; + } + merged.push_back(part); + cursor = end; + }; + for (auto index = firstWrite; index < intervals.size() && intervals[index].first < range.end; ++index) { + append(std::max(cursor, intervals[index].first), false); + append(std::min(range.end, intervals[index].second), true); + } + append(range.end, false); + } return merged; } std::optional GuestBufferMemory::CachedAddressTable() const { if (space == nullptr || !regions.empty()) return std::nullopt; Require(uploaded && !committed, "guest GPU address ranges are not available"); - return CachedTable{space->serial, &space->ranges}; + if (writes.empty()) return CachedTable{space->serial, &space->ranges}; + if (writeTableRanges != nullptr) return CachedTable{writeTableSerial, writeTableRanges.get()}; + std::lock_guard lock(space->tableMutex); + const auto found = space->writeTables.find(writes); + if (found != space->writeTables.end()) { + writeTableSerial = found->second.first; + writeTableRanges = found->second.second; + } else { + writeTableRanges = std::make_shared>(AddressRanges()); + writeTableSerial = Spaces().serials.fetch_add(1, std::memory_order_relaxed) + 1; + if (space->writeTables.size() >= 64u) space->writeTables.erase(space->writeTables.begin()); + space->writeTables.emplace(writes, std::pair{writeTableSerial, writeTableRanges}); + } + return CachedTable{writeTableSerial, writeTableRanges.get()}; } bool GuestBufferMemory::HasCopiedWrites() const { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp index d3f6a96da..f97d82c06 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp @@ -71,7 +71,7 @@ GuestTextureResource DecodeTextureResource(std::span words) const auto bcSwizzle = (words[3] >> 25u) & 0x7u; const auto typeRaw = (words[3] >> 28u) & 0xfu; - const auto depth = (words[4] >> 0u) & 0x1fffu; + auto depth = (words[4] >> 0u) & 0x1fffu; const auto baseArray = (words[4] >> 16u) & 0x1fffu; const auto arrayPitch = (words[5] >> 0u) & 0xfu; @@ -115,7 +115,6 @@ GuestTextureResource DecodeTextureResource(std::span words) static_cast(maxCompBlkSize); // Surfaces are always written uncompressed here (render targets and storage images bypass DCC), so // only the fast-clear keys in DCC metadata change what a read returns (see DccMetadata.hpp). - static_cast(metaPipeAligned); static_cast(writeCompress); if (dccColorTransf) { static bool reported = false; @@ -166,7 +165,11 @@ GuestTextureResource DecodeTextureResource(std::span words) case TextureDimension::kCube: Require(width == height, "guest cube texture descriptor is not square"); Require(baseArray <= depth, "guest cube texture descriptor has a base array past its last array slice"); - Require((depth - baseArray + 1u) % 6u == 0, "guest cube texture descriptor does not contain a multiple of 6 array slices"); + if (baseArray == depth) depth += 5u; + if ((depth - baseArray + 1u) % 6u != 0) { + Require(false, "guest cube texture descriptor does not contain a multiple of 6 array slices" + + describe() + " (base array " + std::to_string(baseArray) + ")"); + } break; } @@ -189,6 +192,7 @@ GuestTextureResource DecodeTextureResource(std::span words) result.dstSelW = static_cast(dstSelW); result.dccAddress = metaCompress ? metaAddr << 8u : 0u; result.dccAlphaOnMsb = dccAlphaPos; + result.dccPipeAligned = metaPipeAligned; result.minLod = minLod; if (baseLevel > maxMip) { Require(LevelsFitAllocation(result, lastLevel + 1u), "guest texture descriptor starts past the surface's last mip level at levels that would move the surface's own" + describe()); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp index e77279db5..3f932eeaa 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp @@ -1,5 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" #include #include @@ -92,6 +94,7 @@ void ValidateViewport(const Context& context, const VkViewport& viewport) { } Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout) : context(context), _modules(shaders.size()), attachments(state.colors.size() + (state.depth ? 1u : 0u)), colorAttachments(state.colors.size()), depthBounds(state.depth.has_value() && state.depthBoundsTest), depthBias(state.depth.has_value() && state.depthBias) { + PerformanceTimer timing("Vulkan.GraphicsPipeline"); // A cached pipeline may outlive its device's teardown (see ClearCachedPipelines); it must not keep // the buffer pool, which is reset with the device, alive past it. this->context.bufferPool.reset(); @@ -101,6 +104,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput Require(!depthBounds || context.depthBounds, "device does not support the depth bounds test"); Require(!depthBias || state.depthBiasClamp == 0.0f || context.depthBiasClamp, "device does not support depth bias clamping"); Require(!state.negativeOneToOne || context.depthClipControl, "negative-one-to-one depth clipping requires VK_EXT_depth_clip_control with depthClipControl enabled"); + Require(state.conservativeRasterization == VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT || context.conservativeRasterization, "conservative rasterization requires VK_EXT_conservative_rasterization with at most 1/256 pixel of overestimation and degenerate triangles rasterized"); if (state.rectList) Require(context.tessellationShader && context.limits.maxTessellationPatchSize >= 4, "rect-list requires tessellation with four output control points"); if (state.stages.tessellation) { Require(context.tessellationShader, "device does not support tessellation shaders"); @@ -117,6 +121,8 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput Require(pushStages == 0 || context.limits.maxPushConstantsSize >= PipelinePushConstantBytes, "graphics push constant range exceeds device limit"); try { std::vector stages(shaders.size()); + std::vector specializations; + specializations.reserve(shaders.size()); for (std::uint32_t i = 0; i < shaders.size(); ++i) { const auto& shader = *shaders[i].program; VkShaderModuleCreateInfo module{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; @@ -128,6 +134,8 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput stages[i].stage = stage; stages[i].module = _modules[i]; stages[i].pName = "main"; + specializations.emplace_back(shader); + stages[i].pSpecializationInfo = specializations.back().Info(); } // A descriptor set layout with the same bindings as this one is compatible with the pipeline // layout, so later draws bind their own ShaderResources' set under it. @@ -208,6 +216,9 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput raster.frontFace = state.frontFace; raster.depthBiasEnable = depthBias; raster.lineWidth = 1; + VkPipelineRasterizationConservativeStateCreateInfoEXT conservative{VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_CONSERVATIVE_STATE_CREATE_INFO_EXT}; + conservative.conservativeRasterizationMode = state.conservativeRasterization; + if (state.conservativeRasterization != VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT) raster.pNext = &conservative; VkPipelineMultisampleStateCreateInfo samples{VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO}; samples.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; VkPipelineDepthStencilStateCreateInfo depthStencil{VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO}; @@ -241,7 +252,9 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput pipelineInfo.pDynamicState = &dynamic; pipelineInfo.layout = layout; pipelineInfo.renderPass = renderPass; + timing.Mark("modules_and_state"); Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines"); + timing.Mark("create"); LogPipelineStatistics_nid_no_patch(context, pipeline); } catch (...) { release(); @@ -367,9 +380,9 @@ std::vector pipelineKey(const Context& context, const State& state, c for (const auto& shader : shaders) { Require(shader.program != nullptr, "missing compiled shader"); const bool generated = state.rectList && (shader.stage == Stage::TessellationControl || shader.stage == Stage::TessellationEvaluation); - if (!generated && shader.program->variantId == 0) return {}; + if (!generated && shader.program->PipelineVariantId() == 0) return {}; append(key, shader.stage); - append(key, generated ? std::uint64_t{0} : shader.program->variantId); + append(key, generated ? std::uint64_t{0} : shader.program->PipelineVariantId()); // Where the stage's push constants sit in the block (AssemblePushConstants). append(key, shader.pushConstantOffset); } @@ -397,6 +410,7 @@ std::vector pipelineKey(const Context& context, const State& state, c append(key, state.frontFace); append(key, state.negativeOneToOne); append(key, state.depthClamp && context.depthClamp); + append(key, state.conservativeRasterization); append(key, state.blends.size()); for (const auto& blend : state.blends) append(key, blend); for (const auto value : state.blendConstants) append(key, value); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index 7a4809bd2..73aa04140 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -328,11 +328,15 @@ struct DeferredBatch { }; struct DeferredBatchesTag {}; auto& DeferredBatches() { return HostThreadLocal, DeferredBatchesTag>(); } +struct DeferredBytesTag {}; +auto& DeferredBytes() { return HostThreadLocal(); } +std::vector TakeDeferred() { + auto taken = std::move(DeferredBatches()); + DeferredBatches().clear(); + DeferredBytes() = 0; + return taken; +} std::atomic deferredPending{0}; -// APS5_PROFILE_DRAW, for the [recorder] line: batches and objects released after an unlock and the -// time that took, on the release thread and inline (on the unlocking thread: the kill switch, a -// full queue, a stopping thread or a failed hand-off), the batches that went inline because the -// queue was full, and the longest queue seen (in batches). std::atomic threadReleases{0}, threadObjects{0}, threadReleaseUs{0}; std::atomic inlineReleases{0}, inlineObjects{0}, inlineReleaseUs{0}, inlineOverBound{0}; std::atomic releaseQueueMax{0}; @@ -446,8 +450,7 @@ void ReleaseDeferredKeeps() { if (DeferredBatches().empty()) return; // Taken off the thread's list first: a destructor that took and released the mutex (none is // known to) would re-enter here and must find nothing. - auto releasing = std::move(DeferredBatches()); - DeferredBatches().clear(); + auto releasing = TakeDeferred(); if (ReleaseOnUnlock()) { DestroyDeferred(std::move(releasing), false); return; @@ -1134,11 +1137,7 @@ Recorder::~Recorder() { // other threads (an unlock hook that found the queue stopping or full) are waited for before // the caller destroys the device. The count is global: a newer recorder's batches (device // replacement) are waited for too, which only prolongs the spin. - if (!DeferredBatches().empty()) { - auto own = std::move(DeferredBatches()); - DeferredBatches().clear(); - DestroyDeferred(std::move(own), false); - } + if (!DeferredBatches().empty()) DestroyDeferred(TakeDeferred(), false); JoinReleaseThread(); while (deferredPending.load(std::memory_order_acquire) != 0) std::this_thread::yield(); { @@ -1370,10 +1369,13 @@ VkCommandBuffer Recorder::CommandsInRenderPass() { return open->commands; } -void Recorder::LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool continuable) { +void Recorder::LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool continuable, std::function afterPass) { Require(open != nullptr, "no batch is open for the render pass"); auto& pass = open->renderPass; - if (!pass.open) pass.timing = timing; + if (!pass.open) { + pass.timing = timing; + pass.afterPass = std::move(afterPass); + } pass.open = true; pass.key = key; pass.continuable = continuable; @@ -1386,6 +1388,7 @@ void Recorder::endOpenRenderPass() { // host sees them at the batch's fence). recordBarrier(open->commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); CountBarriers(CommandClass::Draw); + if (pass.afterPass) pass.afterPass(open->commands); EndGpuTiming(pass.timing); open->coveredAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; open->hostReadOwed = true; @@ -2186,9 +2189,21 @@ std::size_t Recorder::UnsignaledBatches() const { return static_cast(std::count_if(inFlight.begin(), inFlight.end(), [&](const auto& batch) { return !signaled(*batch); })); } -void Recorder::Keep(std::shared_ptr object) { +void Recorder::Keep(std::shared_ptr object, std::size_t bytes) { ensureOpen(); open->kept.push_back(std::move(object)); + open->keptBytes += bytes; +} + +void Recorder::BoundKeptBytes() { + if (!GuestMemory::GpuMutex().HeldByThisThread()) return; + if (open != nullptr && open->keptBytes >= KeptBytesBudget) Submit(); + if (DeferredBytes() != 0 && inFlightKeptBytes + DeferredBytes() > 2 * KeptBytesBudget) DestroyDeferred(TakeDeferred(), false); + while (!inFlight.empty() && inFlightKeptBytes > 2 * KeptBytesBudget) { + auto batch = std::move(inFlight.front()); + inFlight.pop_front(); + finish(std::move(batch), true, 4, true); + } } namespace { @@ -2747,6 +2762,7 @@ bool Recorder::writtenBackSince(std::uint64_t sequence, std::uint64_t begin, std } void Recorder::Submit() { + PerformanceTimer frameTiming("Recorder.Submit"); // The work count is cleared even when nothing is open: the driver counts a dispatch after its // call returns (outside the mutex), so a submit by another thread in between leaves a stale // count behind, and the callers that act on it would otherwise take the mutex for nothing at @@ -2807,6 +2823,7 @@ void Recorder::Submit() { newestSubmittedAt = batch->submittedAt; } inFlight.push_back(std::move(batch)); + inFlightKeptBytes += inFlight.back()->keptBytes; if (activeRecorder == this) pendingLabelSince.store(NoPendingLabel, std::memory_order_release); } @@ -2855,6 +2872,7 @@ VkResult WaitTimeline(VkDevice device, VkSemaphore timeline, PFN_vkWaitSemaphore } void Recorder::WaitSerial(std::uint64_t serial) { + PerformanceTimer frameTiming("Recorder.WaitSerial"); if (timeline == VK_NULL_HANDLE || serial == 0) return; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; @@ -2884,6 +2902,7 @@ void Recorder::FinishUpTo(std::uint64_t serial) { } void Recorder::Sync() { + PerformanceTimer frameTiming("Recorder.Sync"); const auto source = std::exchange(announcedSource, 4); const void* site = std::exchange(announcedSite, nullptr); Submit(); @@ -3156,6 +3175,7 @@ bool Recorder::syncThroughUnlocked(std::uint64_t address, std::uint64_t end, int } bool Recorder::Reap() { + PerformanceTimer frameTiming("Recorder.Reap"); const bool profile = DrawProfiled(); const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; std::uint64_t retired = 0; @@ -3175,7 +3195,8 @@ bool Recorder::Reap() { return inFlight.empty(); } -void Recorder::finish(std::unique_ptr batch, bool wait, int source) { +void Recorder::finish(std::unique_ptr batch, bool wait, int source, bool releaseNow) { + inFlightKeptBytes -= batch->keptBytes; // The batch's writes stay published until its completions ran, on every exit path. struct Finishing { Recorder& recorder; @@ -3287,7 +3308,7 @@ void Recorder::finish(std::unique_ptr batch, bool wait, int source) { const auto keptStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; // Deferred only from inside a hold: the unlock that releases the list is this thread's own, and // a caller finishing batches without the mutex (a test) might never make one. - if (ReleaseUnderLock() || !GuestMemory::GpuMutex().HeldByThisThread()) { + if (releaseNow || ReleaseUnderLock() || !GuestMemory::GpuMutex().HeldByThisThread()) { batch->completions.clear(); batch->kept.clear(); } else { @@ -3298,6 +3319,7 @@ void Recorder::finish(std::unique_ptr batch, bool wait, int source) { try { DeferredBatches().push_back({std::move(batch->kept), std::move(batch->completions)}); deferredPending.fetch_add(1, std::memory_order_acq_rel); + DeferredBytes() += batch->keptBytes; } catch (...) { } batch->completions.clear(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index d4849f846..336cbc301 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -7,24 +7,25 @@ namespace AgcDriver::Graphics { -Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(usage), properties(properties) { +Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) : context(context), size(size), capacity(BufferPool::Capacity(size, properties)), usage(BufferPool::Usage(usage, properties)), properties(properties) { Require(size != 0, "zero-sized GPU buffer"); const bool addressable = (usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0; Require(!addressable || context.bufferDeviceAddress, "buffer device address is not enabled"); cache = GetBufferPool(context); - if (const auto allocation = cache->Take(size, usage, properties)) { + if (const auto allocation = cache->Take(size, this->usage, properties)) { buffer = allocation->buffer; memory = allocation->memory; mapping = allocation->mapping; deviceAddress = allocation->address; allocationBytes = allocation->allocationBytes; + capacity = allocation->bytes; ready = true; return; } try { VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; info.size = capacity; - info.usage = usage; + info.usage = this->usage; info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; Check(context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer), "vkCreateBuffer"); VkMemoryRequirements requirements{}; @@ -88,19 +89,20 @@ void Buffer::Invalidate() { Check(context.Function("vkInvalidateMappedMemoryRanges")(context.device, 1, &range), "vkInvalidateMappedMemoryRanges"); } -DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsageFlags usage) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(usage) { +DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsageFlags usage) : context(context), size(size), capacity(BufferPool::Capacity(size, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)), usage(BufferPool::Usage(usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { Require(size != 0, "zero-sized device buffer"); cache = GetBufferPool(context); - if (const auto allocation = cache->Take(size, usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { + if (const auto allocation = cache->Take(size, this->usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { buffer = allocation->buffer; memory = allocation->memory; allocationBytes = allocation->allocationBytes; + capacity = allocation->bytes; return; } try { VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; info.size = capacity; - info.usage = usage; + info.usage = this->usage; info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; Check(context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer), "vkCreateBuffer device"); VkMemoryRequirements requirements{}; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp index 895fa359d..04601fb8e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp @@ -15,8 +15,8 @@ void ShaderResources::prepareAddressBindings(std::span sha if (binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable) ++tables; else ++faults; } - const bool rectListFault = shader.stage == ShaderRecompiler::ShaderStage::TessellationControl && tables == 0 && faults == 1; - Require((tables == faults || rectListFault) && tables <= 1 && faults <= 1, "invalid BDA table and fault descriptors"); + const bool faultOnly = tables == 0 && faults == 1 && shader.program->bdaAbiVersion == ShaderRecompiler::BdaAbi::Version; + Require((tables == faults || faultOnly) && tables <= 1 && faults <= 1, "invalid BDA table and fault descriptors"); Require(shader.program->bdaAbiVersion == (faults == 0 ? 0u : ShaderRecompiler::BdaAbi::Version), "incompatible BDA ABI version"); // Rect-list validation needs a fault buffer, but never accesses guest addresses. usesBda = usesBda || tables != 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp index 9b814bf83..2d7b65e3c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp @@ -38,12 +38,6 @@ std::uint32_t read(const Registers& registers, std::uint32_t offset, RegisterBan return it->second; } -std::uint32_t readOr(const Registers& registers, std::uint32_t offset, RegisterBank bank, std::uint32_t fallback) { - NoteRegisterRead(bank, offset); - const auto it = registers.find(offset); - return it == registers.end() ? fallback : it->second; -} - template T _readHeaderPod(std::span header, std::uint64_t headerAddress, const void* pointer) { if (pointer == nullptr) throw std::runtime_error("AGC graphics: null AGC header pointer"); const auto address = reinterpret_cast(pointer); @@ -68,7 +62,7 @@ template void _readHeaderArray(std::span header, s } -ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& shader) { +ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& shader, std::span header) { const auto numThreadX = read(shader, computeNumThreadX, RegisterBank::Shader); const auto numThreadY = read(shader, computeNumThreadY, RegisterBank::Shader); const auto numThreadZ = read(shader, computeNumThreadZ, RegisterBank::Shader); @@ -76,8 +70,13 @@ ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& throw std::runtime_error("AGC graphics: COMPUTE_NUM_THREAD_X/Y/Z must be nonzero"); } const auto rsrc2 = read(shader, computePgmRsrc2, RegisterBank::Shader); + std::uint32_t scratchDwords = 0; if ((rsrc2 & 0x1u) != 0) { - throw std::runtime_error("AGC graphics: COMPUTE_PGM_RSRC2.SCRATCH_EN is unsupported"); + if (header.size() < sizeof(Shader)) throw std::runtime_error("AGC graphics: COMPUTE_PGM_RSRC2.SCRATCH_EN without an AGC shader header"); + Shader agcShader; + std::memcpy(&agcShader, header.data(), sizeof(Shader)); + scratchDwords = agcShader.scratch_size_dw_per_thread; + if (scratchDwords == 0) throw std::runtime_error("AGC graphics: COMPUTE_PGM_RSRC2.SCRATCH_EN with a zero scratch size"); } // Debug aid: APS5_LDS_SLACK= grows every dispatch's LDS allocation by that much, to tell // whether a program depends on addresses past its declared allocation. @@ -87,18 +86,20 @@ ShaderRecompiler::ShaderComputeStageInfo DecodeComputeStageInfo(const Registers& std::min(((rsrc2 >> 15u) & 0x1FFu) * 128u + ldsSlack, 16384u), {((rsrc2 >> 7u) & 0x1u) != 0, ((rsrc2 >> 8u) & 0x1u) != 0, ((rsrc2 >> 9u) & 0x1u) != 0}, ((rsrc2 >> 10u) & 0x1u) != 0, - ((rsrc2 >> 11u) & 0x3u) + 1u + ((rsrc2 >> 11u) & 0x3u) + 1u, + {}, + scratchDwords }; } ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& context, const std::array& exportMappings, bool nullProgram) { - const auto inControl = read(context, spiPsInControl, RegisterBank::Context); + const auto inControl = nullProgram ? 0u : read(context, spiPsInControl, RegisterBank::Context); const auto inputNum = inControl & 0x3Fu; if (inputNum > 32u) { throw std::runtime_error("AGC graphics: SPI_PS_IN_CONTROL input count exceeds 32"); } - const auto ena = nullProgram ? readOr(context, spiPsInputEna, RegisterBank::Context, defaultPixelInputs) : read(context, spiPsInputEna, RegisterBank::Context); - const auto addr = nullProgram ? readOr(context, spiPsInputAddr, RegisterBank::Context, defaultPixelInputs) : read(context, spiPsInputAddr, RegisterBank::Context); + const auto ena = nullProgram ? defaultPixelInputs : read(context, spiPsInputEna, RegisterBank::Context); + const auto addr = nullProgram ? defaultPixelInputs : read(context, spiPsInputAddr, RegisterBank::Context); const auto activeInputs = ena & addr; using ShaderRecompiler::PixelInput; using ShaderRecompiler::PixelInputBit; @@ -115,14 +116,14 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con for (std::uint32_t i = 0; i < inputNum; ++i) { interpolatorSettings[i] = read(context, spiPsInputCntl0 + i, RegisterBank::Context); } - const auto shaderControl = read(context, dbShaderControl, RegisterBank::Context); + const auto shaderControl = nullProgram ? 0u : read(context, dbShaderControl, RegisterBank::Context); // Bits 9 and 11 are EXEC_ON_HIER_FAIL and ALPHA_TO_MASK_DISABLE on GFX10; neither changes what the // recompiled pixel shader computes. const auto conservativeZExport = (shaderControl >> 13u) & 0x3u; if (conservativeZExport == 3u) { throw std::runtime_error("AGC graphics: DB_SHADER_CONTROL.CONSERVATIVE_Z_EXPORT uses the reserved value 3"); } - const auto colFormat = nullProgram ? readOr(context, spiShaderColFormat, RegisterBank::Context, 0u) : read(context, spiShaderColFormat, RegisterBank::Context); + const auto colFormat = nullProgram ? 0u : read(context, spiShaderColFormat, RegisterBank::Context); std::array targetOutputMode{}; for (std::uint32_t i = 0; i < 8u; ++i) { targetOutputMode[i] = static_cast((colFormat >> (4u * i)) & 0xFu); @@ -154,12 +155,13 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con .earlyZ = zOrder == 1u && !pixelKillEnable && !depthExportEnable && !sampleMaskExportEnable, .executeOnNoop = ((shaderControl >> 10u) & 0x1u) != 0, .conservativeZExport = static_cast(conservativeZExport), + .orderedPixelShader = ((shaderControl >> 16u) & 0x1u) != 0, .targetOutputMode = targetOutputMode, .targetExportMapping = exportMappings }; } -ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span header, std::uint64_t headerAddress, std::span userData, std::vector* reads) { +ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span header, std::uint64_t headerAddress, std::span userData, std::vector* reads, bool staticAbi) { if (header.size() < sizeof(Shader)) throw std::runtime_error("AGC graphics: shader header is smaller than the fixed AGC header"); Shader shader; std::memcpy(&shader, header.data(), sizeof(Shader)); @@ -188,14 +190,21 @@ ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span(userData[static_cast(vertexAttribReg)]) | (static_cast(userData[static_cast(vertexAttribReg) + 1u]) << 32u); const auto bufferTableAddr = static_cast(userData[static_cast(vertexBufferReg)]) | (static_cast(userData[static_cast(vertexBufferReg) + 1u]) << 32u); - if (attribTableAddr == 0) throw std::runtime_error("AGC graphics: null vertex attribute table address"); - if (bufferTableAddr == 0) throw std::runtime_error("AGC graphics: null vertex buffer table address"); + if (!staticAbi && attribTableAddr == 0) throw std::runtime_error("AGC graphics: null vertex attribute table address"); + if (!staticAbi && bufferTableAddr == 0) throw std::runtime_error("AGC graphics: null vertex buffer table address"); info.fetchEmbedded = true; info.fetchAttribReg = static_cast(vertexAttribReg); info.fetchBufferReg = static_cast(vertexBufferReg); for (std::uint32_t i = 0; i < shader.num_input_semantics; ++i) { const auto& semantic = semantics[i]; if (semantic.static_vb_index == 1 || semantic.static_attribute == 1) throw std::runtime_error("AGC graphics: statically bound vertex attributes are not implemented"); + if (staticAbi) { + auto& destination = info.resourcesDst.at(info.resourcesNum++); + destination.registerStart = static_cast(semantic.hardware_mapping); + destination.registersNum = static_cast(semantic.size_in_elements); + destination.attrId = static_cast(semantic.semantic); + continue; + } std::array attribWordBytes{}; const auto attribWordAddress = attribTableAddr + static_cast(semantic.semantic) * 4u; AgcDriver::GuestMemory::Read(attribWordAddress, attribWordBytes, 4); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index e9a5deb4e..d79c2e918 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" #include "ThreadOwned.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" @@ -108,7 +109,13 @@ struct TextureCache { std::mutex mutex; std::list entries; std::unordered_map::iterator, TextureKeyHash> index; + std::unordered_multimap::iterator> views; std::uint64_t bytes = 0; + VkPhysicalDevice budgetDevice = VK_NULL_HANDLE; + std::chrono::steady_clock::time_point budgetRead{}; + std::uint64_t budget = 0; + std::uint64_t reportedBudget = 0; + std::chrono::steady_clock::time_point reportedAt{}; }; TextureCache& Textures() { @@ -116,6 +123,24 @@ TextureCache& Textures() { return cache; } +struct UncachedImages { + std::mutex mutex; + std::vector> images; + std::atomic any{false}; +}; + +UncachedImages& Uncached() { + static UncachedImages uncached; + return uncached; +} + +void noteUncached(const std::shared_ptr& image) { + auto& uncached = Uncached(); + std::lock_guard lock(uncached.mutex); + uncached.images.push_back(image); + uncached.any.store(true, std::memory_order_release); +} + bool TextureHashEnabled() { static const bool disabled = std::getenv("APS5_NO_TEXTURE_HASH") != nullptr; return !disabled; @@ -134,10 +159,34 @@ std::list::iterator findTexture(TextureCache& cache, const Textur void eraseTexture(TextureCache& cache, std::list::iterator it) { cache.bytes -= it->accounted; + if (it->source != nullptr) { + for (auto [view, last] = cache.views.equal_range(it->source.get()); view != last; ++view) { + if (view->second == it) { + cache.views.erase(view); + break; + } + } + } cache.index.erase(it->key); cache.entries.erase(it); } +void dropUncachedViews(TextureCache& cache) { + auto& uncached = Uncached(); + if (!uncached.any.load(std::memory_order_acquire)) return; + std::vector> images; + { + std::lock_guard lock(uncached.mutex); + images.swap(uncached.images); + uncached.any.store(false, std::memory_order_release); + } + for (const auto& weak : images) { + const auto image = weak.lock(); + if (image == nullptr) continue; + for (auto view = cache.views.find(image.get()); view != cache.views.end(); view = cache.views.find(image.get())) eraseTexture(cache, view->second); + } +} + // Moves an entry to the front (most recently used). void touchTexture(TextureCache& cache, std::list::iterator it) { cache.entries.splice(cache.entries.begin(), cache.entries, it); @@ -211,6 +260,10 @@ void logLookup(const LookupRecord& record) { std::shared_ptr cachedStorageTexture(const Context& context, std::span words, const GuestTextureResource& resource, std::uint32_t mip, std::uint64_t guestBytes = 0); +std::uint64_t heldBytes(const StorageTexture& image) { + return std::max(image.AllocationBytes(), image.GuestBytes()); +} + bool MetadataMoved(const StorageTexture& image, const GuestTextureResource& resource) { return resource.dccAddress != 0 && image.Descriptor().dccAddress != resource.dccAddress && !image.ServesKeysAt(resource.dccAddress); } @@ -280,9 +333,59 @@ std::shared_ptr sampledStorageSource(const Context& context, con } } +std::uint64_t storageCacheBytes(); + +std::optional largestDeviceLocalHeap(const VkPhysicalDeviceMemoryProperties& memory) { + std::optional heap; + for (std::uint32_t i = 0; i < std::min(memory.memoryHeapCount, VK_MAX_MEMORY_HEAPS); ++i) { + if ((memory.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0 && (!heap || memory.memoryHeaps[i].size > memory.memoryHeaps[*heap].size)) heap = i; + } + return heap; +} + +std::uint64_t forcedCacheBudget() { + static const std::uint64_t forced = [] { + const char* text = std::getenv("APS5_TEXTURE_CACHE_MIB"); + return text != nullptr ? std::strtoull(text, nullptr, 10) << 20u : 0u; + }(); + return forced; +} + +std::uint64_t storageBudget(const Context& context) { + const auto forced = forcedCacheBudget(); + return forced != 0 ? forced : TextureCacheBudget(context.memory); +} + +std::uint64_t sampledBudget(const Context& context, TextureCache& cache) { + if (const auto forced = forcedCacheBudget(); forced != 0) return forced; + if (context.memoryProperties2 == nullptr || context.physical == VK_NULL_HANDLE) return TextureCacheBudget(context.memory); + const auto now = std::chrono::steady_clock::now(); + if (cache.budgetDevice == context.physical && now - cache.budgetRead < std::chrono::seconds(1)) return cache.budget; + VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2}; + properties.pNext = &reported; + context.memoryProperties2(context.physical, &properties); + const auto sampled = SampledTextureMemory(); + const auto storage = storageCacheBytes(); + const auto budget = SampledTextureBudget(context.memory, &reported, sampled + storage); + const bool fresh = cache.budgetDevice != context.physical; + if (const auto heap = largestDeviceLocalHeap(context.memory); heap && (fresh || SampledBudgetReportDue(cache.reportedBudget, budget, now - cache.reportedAt))) { + char was[48] = ""; + if (!fresh) std::snprintf(was, sizeof(was), " (was %llu MiB)", static_cast(cache.reportedBudget >> 20u)); + std::fprintf(stderr, "[gpu] sampled texture cache budget %llu MiB%s: device-local heap %u has a VK_EXT_memory_budget budget of %llu MiB and uses %llu MiB, %llu MiB of it sampled textures (%llu MiB in the cache) and %llu MiB cached storage images; storage image cache budget %llu MiB\n", static_cast(budget >> 20u), was, *heap, static_cast(reported.heapBudget[*heap] >> 20u), static_cast(reported.heapUsage[*heap] >> 20u), static_cast(sampled >> 20u), static_cast(cache.bytes >> 20u), static_cast(storage >> 20u), static_cast(TextureCacheBudget(context.memory) >> 20u)); + cache.reportedBudget = budget; + cache.reportedAt = now; + } + cache.budgetDevice = context.physical; + cache.budgetRead = now; + cache.budget = budget; + return budget; +} + // `guestBytes` is the surface size when the caller described the surface already (0: described here). std::shared_ptr cachedTexture(const Context& context, std::span words, const GuestTextureResource& resource, VkComponentMapping components, std::uint64_t guestBytes = 0, bool depthCompare = false) { CaptureTrace::Log("sampled-lookup address=%llx width=%u height=%u dcc=%llx", static_cast(resource.baseAddress), resource.width, resource.height, static_cast(resource.dccAddress)); + if (auto* recorder = Recorder::Active(); recorder != nullptr) recorder->BoundKeptBytes(); if (auto depth = DepthSurfaceTexture(context, words, resource, components)) return depth; const auto depthBitsWidth = words.size() >= 4 ? ShaderRecompiler::DepthBitsTextureWidth(words[1], words[3]) : 0u; if (depthBitsWidth == 32u) { @@ -385,6 +488,7 @@ std::shared_ptr cachedTexture(const Context& context, std::spansource != nullptr) { // The view follows the storage image, whatever the GPU wrote to it since; guest memory @@ -420,11 +524,11 @@ std::shared_ptr cachedTexture(const Context& context, std::span(source != nullptr ? 0u : bytes), nullptr, *keys, generation}; - entry.accounted = guestBytes; if (source != nullptr) { entry.source = source; entry.sourceVersion = source->Version(); entry.texture = std::make_shared(context, source, resource, components); + entry.accounted = source->Cached() ? 0u : heldBytes(*source); counters.fromStorage.fetch_add(1, std::memory_order_relaxed); } else { // Snapshot before the upload so a write racing with it is caught by the next comparison. @@ -437,15 +541,17 @@ std::shared_ptr cachedTexture(const Context& context, std::span(resource.baseAddress), resource.width, resource.height, resource.format, static_cast(resource.tileMode), nonzero, entry.bytes.size() / 64); } entry.texture = std::make_shared(context, *context.detiler, resource, components, entry.bytes, depthCompare); + entry.accounted = entry.texture->AllocationBytes(); counters.snapshots.fetch_add(1, std::memory_order_relaxed); } - constexpr std::uint64_t budget = 2048ull << 20u; + const auto budget = sampledBudget(context, cache); while (!cache.entries.empty() && cache.bytes + entry.accounted > budget) eraseTexture(cache, std::prev(cache.entries.end())); cache.bytes += entry.accounted; auto texture = entry.texture; logLookup({texture.get(), resource, guestBytes, *keys, generation, source.get()}); cache.entries.push_front(std::move(entry)); cache.index[key] = cache.entries.begin(); + if (source != nullptr) cache.views.emplace(source.get(), cache.entries.begin()); reportTextureCounters(); if (profile) LookupOutcomes::Add(source != nullptr ? LookupOutcomes::SampledMadeView : LookupOutcomes::SampledMadeSnapshot, start); return texture; @@ -469,6 +575,7 @@ struct CachedStorageTexture { StorageKey key; std::uint32_t mip; std::shared_ptr texture; + std::uint64_t accounted = 0; }; // As TextureCache: use order with a hash index by key, plus one by image for StorageImageCached. @@ -477,7 +584,7 @@ struct StorageTextureCache { std::list entries; std::unordered_map::iterator, StorageKeyHash> index; std::unordered_map::iterator> byImage; - std::uint64_t bytes = 0; + std::atomic bytes{0}; }; StorageTextureCache& StorageTextures() { @@ -485,6 +592,10 @@ StorageTextureCache& StorageTextures() { return cache; } +std::uint64_t storageCacheBytes() { + return StorageTextures().bytes.load(std::memory_order_relaxed); +} + std::list::iterator findStorage(StorageTextureCache& cache, const StorageKey& key) { if (TextureHashEnabled()) { const auto found = cache.index.find(key); @@ -510,8 +621,9 @@ std::list::iterator findStorageByImage(StorageTextureCache // Evicts an entry: its pending results go to guest memory first (the image may die with the entry). void evictStorage(StorageTextureCache& cache, std::list::iterator it) { it->texture->SetCached(false); + noteUncached(it->texture); it->texture->Flush(); - cache.bytes -= it->texture->GuestBytes(); + cache.bytes -= it->accounted; cache.index.erase(it->key); cache.byImage.erase(it->texture.get()); cache.entries.erase(it); @@ -570,6 +682,7 @@ std::shared_ptr cachedStorageTexture(const Context& context, std static const bool disabled = std::getenv("APS5_NO_TEXTURE_CACHE") != nullptr; if (disabled) return std::make_shared(context, *context.detiler, viewed, mip); static_cast(words); + if (auto* recorder = Recorder::Active(); recorder != nullptr) recorder->BoundKeptBytes(); const bool profile = LookupOutcomes::Profiled(); auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; auto& counters = TextureCounts(); @@ -606,9 +719,10 @@ std::shared_ptr cachedStorageTexture(const Context& context, std // APS5_NO_KEEP_NEW_STORAGE=1 leaves the image to its cache entry alone, as before. static const bool keepNew = std::getenv("APS5_NO_KEEP_NEW_STORAGE") == nullptr; if (auto* recorder = Recorder::Active(); keepNew && recorder != nullptr && GuestMemory::GpuMutex().HeldByThisThread() && recorder->Recording()) recorder->Keep(entry.texture); - constexpr std::uint64_t budget = 2048ull << 20u; - while (!cache.entries.empty() && cache.bytes + entry.texture->GuestBytes() > budget) evictStorage(cache, std::prev(cache.entries.end())); - cache.bytes += entry.texture->GuestBytes(); + entry.accounted = heldBytes(*entry.texture); + const auto budget = storageBudget(context); + while (!cache.entries.empty() && cache.bytes + entry.accounted > budget) evictStorage(cache, std::prev(cache.entries.end())); + cache.bytes += entry.accounted; auto texture = entry.texture; cache.entries.push_front(std::move(entry)); cache.index[key] = cache.entries.begin(); @@ -621,6 +735,49 @@ std::shared_ptr cachedStorageTexture(const Context& context, std } +std::uint64_t TextureCacheBudget(const VkPhysicalDeviceMemoryProperties& memory) { + const auto heap = largestDeviceLocalHeap(memory); + return std::max(2048ull << 20u, heap ? memory.memoryHeaps[*heap].size / 4u : 0u); +} + +std::uint64_t SampledTextureBudget(const VkPhysicalDeviceMemoryProperties& memory, const VkPhysicalDeviceMemoryBudgetPropertiesEXT* reported, std::uint64_t textureBytes) { + const auto heap = largestDeviceLocalHeap(memory); + if (reported == nullptr || !heap || reported->heapBudget[*heap] == 0) return TextureCacheBudget(memory); + const auto budget = reported->heapBudget[*heap]; + const auto usage = reported->heapUsage[*heap]; + const auto reserved = usage - std::min(usage, textureBytes) + TextureCacheBudget(memory) + budget / 8u; + return std::max(2048ull << 20u, budget > reserved ? budget - reserved : 0u); +} + +bool SampledBudgetReportDue(std::uint64_t reported, std::uint64_t budget, std::chrono::steady_clock::duration sinceReport) { + const auto change = budget > reported ? budget - reported : reported - budget; + return change > reported / 10u && sinceReport >= std::chrono::seconds(10); +} + +TextureCacheUse TextureCacheUsage() { + TextureCacheUse use; + { + auto& sampled = Textures(); + std::lock_guard lock(sampled.mutex); + dropUncachedViews(sampled); + use.sampledEntries = sampled.entries.size(); + use.sampledBytes = sampled.bytes; + } + use.storageBytes = storageCacheBytes(); + return use; +} + +std::uint64_t SampledTextureCacheBudget(const Context& context) { + auto& cache = Textures(); + std::lock_guard lock(cache.mutex); + return sampledBudget(context, cache); +} + +std::shared_ptr CachedSampledTexture(const Context& context, std::span words) { + const auto resource = DecodeTextureResource(words); + return cachedTexture(context, words, resource, ViewComponents(resource)); +} + void FlushCachedTextures(VkDevice device) { Require(device != VK_NULL_HANDLE, "cannot flush textures without a Vulkan device"); GuestMemory::AssertGpuLockHeld("FlushCachedTextures"); @@ -649,7 +806,8 @@ void ClearCachedTextures(VkDevice device) { continue; } it->texture->SetCached(false); - storage.bytes -= it->texture->GuestBytes(); + noteUncached(it->texture); + storage.bytes -= it->accounted; storage.index.erase(it->key); storage.byImage.erase(it->texture.get()); it = storage.entries.erase(it); @@ -884,7 +1042,61 @@ double ShaderResources::phase(BuildPhase which) { return ms; } +namespace { + +void ValidateDescriptorLayout(const Context& context, std::span bindings) { + std::map heapTotals; + std::map, std::uint64_t> stageTotals; + std::map stageResources; + for (const auto& layout : bindings) { + heapTotals[layout.descriptorType] += layout.descriptorCount; + for (std::uint32_t bit = 0u; bit < 32u; ++bit) { + const auto stage = VkShaderStageFlags{1u} << bit; + if ((layout.stageFlags & stage) == 0u) continue; + const auto count = stageTotals[{stage, layout.descriptorType}] += layout.descriptorCount; + const auto all = stageResources[stage] += layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER ? 0u : layout.descriptorCount; + Require(all <= context.limits.maxPerStageResources, "typed heaps exceed per-stage resource capacity"); + if (layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE) Require(count <= context.limits.maxPerStageDescriptorSampledImages, "typed heaps exceed per-stage sampled image capacity"); + if (layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) Require(count <= context.limits.maxPerStageDescriptorStorageImages, "typed heaps exceed per-stage storage image capacity"); + if (layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) Require(count <= context.limits.maxPerStageDescriptorSamplers, "typed heaps exceed per-stage sampler capacity"); + } + } + Require(heapTotals[VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE] <= context.limits.maxDescriptorSetSampledImages && heapTotals[VK_DESCRIPTOR_TYPE_STORAGE_IMAGE] <= context.limits.maxDescriptorSetStorageImages && heapTotals[VK_DESCRIPTOR_TYPE_SAMPLER] <= context.limits.maxDescriptorSetSamplers, "typed heaps exceed descriptor set capacity"); +} + +void ValidateRuntimeResources(const CompiledShader& shader, std::span words) { + Require(shader.program != nullptr, "missing compiled shader"); + const auto& program = *shader.program; + const auto& descriptors = program.bindings; + constexpr auto bindingCount = static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count); + std::array heaps{}; + if (program.runtimeImageCount != 0u) { + for (const auto& binding : descriptors) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages) continue; + auto& heap = heaps[binding.binding % bindingCount]; + Require(heap == nullptr, "duplicate runtime image binding"); + heap = &binding; + } + } + Require(words.size() == program.shaderDataDwords, "invalid compact shader data size"); + constexpr auto metadataWords = sizeof(ShaderRecompiler::RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t); + Require(program.imageMetadataDword <= words.size() && program.runtimeImageCount <= (words.size() - program.imageMetadataDword) / metadataWords, "runtime image metadata exceeds shader data"); + for (const auto index : program.runtimeImageResources) { + Require(index < program.runtimeImageCount, "runtime image resource exceeds its compact layout"); + const auto offset = program.imageMetadataDword + index * metadataWords; + const auto kind = words[offset]; + const auto first = words[offset + 1u]; + const auto count = words[offset + 2u]; + Require(count != 0u, "runtime image metadata has no descriptor elements"); + Require(kind < heaps.size() && heaps[kind] != nullptr, "runtime metadata references an unbound image heap"); + Require(first < heaps[kind]->count && count <= heaps[kind]->count - first, "runtime metadata exceeds its bound heap"); + } +} + +} + void ShaderResources::buildPrepare(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes) { + PerformanceTimer frameTiming("Resources.Prepare"); const auto stageStart = std::chrono::steady_clock::now(); phaseStart = stageStart; if (BuildProfiled()) { @@ -893,9 +1105,11 @@ void ShaderResources::buildPrepare(std::span shaders, cons } try { Require(!shaders.empty() && context.limits.maxBoundDescriptorSets >= 1, "shader descriptor set exceeds device limits"); + if (shaders.size() == 1u) refreshResourceKey = ContentKey(shaders.front(), false, true); std::set occupied; for (const auto& shader : shaders) { Require(shader.program != nullptr, "missing compiled shader"); + ShaderRecompiler::RuntimeAbi::RequireVersion(shader.program->runtimeAbiVersion); const VkShaderStageFlags flags = VulkanStage(shader.stage); std::uint64_t stageStorageBuffers = 0; std::uint64_t stageResources = 0; @@ -955,6 +1169,9 @@ void ShaderResources::buildPrepare(std::span shaders, cons } else { Require(binding.count == 1, "shader data and flattened SRT descriptors must not be arrays"); Require(!binding.guestDescriptor.empty(), "empty shader data descriptor"); + if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) { + ValidateRuntimeResources(shader, binding.guestDescriptor); + } item.allocations.push_back(addDataBuffer(binding.guestDescriptor)); if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) shaderData = static_cast(item.allocations.back()); } @@ -983,6 +1200,7 @@ void ShaderResources::buildPrepare(std::span shaders, cons if (context.descriptorCache != nullptr && !noLayoutCache) { _layout = context.descriptorCache->Layout(layoutKey, description); } else { + ValidateDescriptorLayout(context, description); VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; info.bindingCount = static_cast(description.size()); info.pBindings = description.data(); @@ -1027,6 +1245,8 @@ void ShaderResources::buildPrepare(std::span shaders, cons } void ShaderResources::buildComplete() { + PerformanceTimer frameTiming("Resources.Complete"); + Require(!completed, "cannot update a published descriptor heap"); const auto stageStart = std::chrono::steady_clock::now(); phaseStart = stageStart; try { @@ -1054,7 +1274,7 @@ void ShaderResources::buildComplete() { std::size_t imageCount = 0; for (const auto& binding : bindings) { bufferCount += binding.allocations.size(); - imageCount += binding.imageAllocations.size(); + if (binding.layout.descriptorType != VK_DESCRIPTOR_TYPE_STORAGE_BUFFER) imageCount += binding.layout.descriptorCount; } std::vector buffers; std::vector images; @@ -1075,15 +1295,24 @@ void ShaderResources::buildComplete() { break; case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: write.pImageInfo = images.data() + images.size(); - for (const auto index : binding.imageAllocations) images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()}); + for (const auto index : binding.imageAllocations) { + if (index == std::numeric_limits::max()) images.push_back({VK_NULL_HANDLE, VK_NULL_HANDLE, VK_IMAGE_LAYOUT_GENERAL}); + else images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()}); + } + Require(binding.imageAllocations.size() == binding.layout.descriptorCount, "descriptor allocations disagree with compact binding"); break; case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: write.pImageInfo = images.data() + images.size(); - for (const auto index : binding.imageAllocations) images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL}); + for (const auto index : binding.imageAllocations) { + if (index == std::numeric_limits::max()) images.push_back({VK_NULL_HANDLE, VK_NULL_HANDLE, VK_IMAGE_LAYOUT_GENERAL}); + else images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index]), VK_IMAGE_LAYOUT_GENERAL}); + } + Require(binding.imageAllocations.size() == binding.layout.descriptorCount, "descriptor allocations disagree with compact binding"); break; case VK_DESCRIPTOR_TYPE_SAMPLER: write.pImageInfo = images.data() + images.size(); for (const auto index : binding.imageAllocations) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED}); + Require(binding.imageAllocations.size() == binding.layout.descriptorCount, "descriptor allocations disagree with compact binding"); break; default: throw std::runtime_error("AGC graphics: ShaderResources encountered an unknown descriptor type while writing the descriptor set"); } @@ -1184,42 +1413,41 @@ void ShaderResources::reportDescriptorCaches() const { const auto descriptors = context.descriptorCache != nullptr ? context.descriptorCache->Counters() : DescriptorCache::Stats{}; const auto samplerHits = context.samplerCache != nullptr ? context.samplerCache->Hits() : 0; const auto samplerMisses = context.samplerCache != nullptr ? context.samplerCache->Misses() : 0; - AgcDriver::ProfilePrint_nid_no_patch("[descriptors] layouts %llu hits / %llu created, sets %llu from %llu pools, samplers %llu hits / %llu created\n", static_cast(descriptors.layoutHits), static_cast(descriptors.layoutMisses), static_cast(descriptors.sets), static_cast(descriptors.pools), static_cast(samplerHits), static_cast(samplerMisses)); + AgcDriver::ProfilePrint_nid_no_patch("[descriptors] layouts %llu hits / %llu created, sets %llu from %llu pools and %llu dedicated pools, samplers %llu hits / %llu created\n", static_cast(descriptors.layoutHits), static_cast(descriptors.layoutMisses), static_cast(descriptors.sets), static_cast(descriptors.pools), static_cast(descriptors.dedicatedPools), static_cast(samplerHits), static_cast(samplerMisses)); } -std::vector ShaderResources::ContentKey(const CompiledShader& shader, bool dataWords, bool movableBuffers) { +template +void VisitContentKey(const CompiledShader& shader, bool dataWords, bool movableBuffers, const TVisitor& visit) { Require(shader.program != nullptr, "missing compiled shader"); const auto& program = *shader.program; - std::vector key; - key.reserve(8 + program.bindings.size() * 12); - key.push_back(dataWords ? 1u : 0u); - key.push_back(static_cast(shader.stage)); - key.push_back(static_cast(program.variantId)); - key.push_back(static_cast(program.variantId >> 32u)); - key.push_back(static_cast(program.bindings.size())); + visit(dataWords ? 1u : 0u); + visit(static_cast(shader.stage)); + visit(static_cast(program.PipelineVariantId())); + visit(static_cast(program.PipelineVariantId() >> 32u)); + visit(static_cast(program.bindings.size())); const auto packBits = [&](const std::vector& bits) { - key.push_back(static_cast(bits.size())); + visit(static_cast(bits.size())); std::uint32_t word = 0; for (std::size_t i = 0; i < bits.size(); ++i) { if (bits[i]) word |= 1u << (i % 32u); if (i % 32u == 31u || i + 1 == bits.size()) { - key.push_back(word); + visit(word); word = 0; } } }; for (const auto& binding : program.bindings) { - key.insert(key.end(), {static_cast(binding.kind), static_cast(binding.role), binding.descriptorSet, binding.binding, binding.count, binding.readOnly ? 1u : 0u, binding.imageShape.has_value() ? static_cast(*binding.imageShape) + 1u : 0u, static_cast(binding.guestDescriptor.size())}); + for (const auto word : {static_cast(binding.kind), static_cast(binding.role), binding.descriptorSet, binding.binding, binding.count, binding.readOnly ? 1u : 0u, binding.imageShape.has_value() ? static_cast(*binding.imageShape) + 1u : 0u, static_cast(binding.guestDescriptor.size())}) visit(word); if (movableBuffers && binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers && binding.guestDescriptor.size() == static_cast(binding.count) * 4u) { for (std::uint32_t element = 0; element < binding.count; ++element) { const auto* words = binding.guestDescriptor.data() + static_cast(element) * 4u; const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; const bool empty = words[2] == 0 || (words[0] == 0 && (words[1] & 0xffffu) == 0); - if (written || empty) key.insert(key.end(), words, words + 4); - else key.insert(key.end(), {0u, words[1] & 0xffff0000u, 0u, words[3]}); + if (written || empty) { for (std::uint32_t index = 0; index < 4u; ++index) visit(words[index]); } + else { for (const auto word : {0u, words[1] & 0xffff0000u, 0u, words[3]}) visit(word); } } } else if ((dataWords && !movableBuffers) || !DataRole(binding.role)) { - key.insert(key.end(), binding.guestDescriptor.begin(), binding.guestDescriptor.end()); + for (const auto word : binding.guestDescriptor) visit(word); } packBits(binding.imageWritten); packBits(binding.samplerDepthCompare); @@ -1230,12 +1458,29 @@ std::vector ShaderResources::ContentKey(const CompiledShader& sha // Read-only elements are bound without a write set: an object built for one written set // must not serve a build with another (the variant implies it, this makes it explicit). packBits(binding.bufferWritten); - key.push_back(static_cast(binding.imageSamplers.size())); - key.insert(key.end(), binding.imageSamplers.begin(), binding.imageSamplers.end()); + visit(static_cast(binding.imageSamplers.size())); + for (const auto sampler : binding.imageSamplers) visit(sampler); } +} + +std::vector ShaderResources::ContentKey(const CompiledShader& shader, bool dataWords, bool movableBuffers) { + Require(shader.program != nullptr, "missing compiled shader"); + std::vector key; + key.reserve(8u + shader.program->bindings.size() * 12u); + VisitContentKey(shader, dataWords, movableBuffers, [&](std::uint32_t word) { key.push_back(word); }); return key; } +bool MatchesContentKey(const CompiledShader& shader, std::span key) { + std::size_t cursor = 0; + bool equal = true; + VisitContentKey(shader, false, true, [&](std::uint32_t word) { + equal &= cursor < key.size() && key[cursor] == word; + ++cursor; + }); + return equal && cursor == key.size(); +} + namespace { // APS5_PROFILE_DRAW: Revalidate outcomes and time, printed as [rescache] every 10 s next to the @@ -1673,17 +1918,23 @@ ShaderResources::OwnRefreshFallback ShaderResources::refreshOwnObjects(std::span for (const auto& binding : shader.program->bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages) continue; if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage) { - for (std::uint32_t element = 0; element < binding.count; ++element, ++textureIndex) { + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + if (words[0] == 0u && (words[1] & 0xffu) == 0u) continue; if (textureIndex >= textures.size()) return OwnRefreshFallback::Uncached; - const auto overlap = listed(textureIndex, false); + const auto index = textureIndex++; + const auto overlap = listed(index, false); if (overlap == overlapping.end()) continue; - if (const auto fallback = refreshView(textureIndex, *overlap); fallback != OwnRefreshFallback::Count) return fallback; + if (const auto fallback = refreshView(index, *overlap); fallback != OwnRefreshFallback::Count) return fallback; } } else if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { - for (std::uint32_t element = 0; element < binding.count; ++element, ++storageIndex) { + for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + if (words[0] == 0u && (words[1] & 0xffu) == 0u) continue; if (storageIndex >= storageTextures.size()) return OwnRefreshFallback::Uncached; - if (listed(storageIndex, true) == overlapping.end()) continue; - if (const auto fallback = refreshStorage(storageIndex); fallback != OwnRefreshFallback::Count) return fallback; + const auto index = storageIndex++; + if (listed(index, true) == overlapping.end()) continue; + if (const auto fallback = refreshStorage(index); fallback != OwnRefreshFallback::Count) return fallback; } } } @@ -1722,6 +1973,7 @@ bool ShaderResources::Revalidate(std::span shaders, ProofR const auto elementWords = binding.count != 0 ? binding.guestDescriptor.size() / binding.count : 0; for (std::uint32_t element = 0; element < binding.count; ++element) { const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + if (words[0] == 0u && (words[1] & 0xffu) == 0u) continue; const auto resource = DecodeTextureResource(words); const VkComponentMapping components = ViewComponents(resource); if (textureIndex >= textures.size() || cachedTexture(context, words, resource, components, 0, !binding.imageDepthCompare.empty() && binding.imageDepthCompare.at(element)) != textures[textureIndex]) return false; @@ -1730,6 +1982,7 @@ bool ShaderResources::Revalidate(std::span shaders, ProofR } else if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { for (std::uint32_t element = 0; element < binding.count; ++element) { const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + if (words[0] == 0u && (words[1] & 0xffu) == 0u) continue; if (storageIndex >= storageTextures.size()) return false; std::shared_ptr expected; if (SameAsPreviousStorageElement(binding, element) && StorageDedupeEnabled()) expected = storageTextures[storageIndex - 1]; @@ -2101,6 +2354,7 @@ DescriptorCache::DescriptorCache(const Context& context) : context(context), des DescriptorCache::~DescriptorCache() { for (const auto pool : pools) destroyPool(context.device, pool, nullptr); + for (const auto pool : dedicated) destroyPool(context.device, pool, nullptr); for (const auto& [key, layout] : layouts) destroyLayout(context.device, layout, nullptr); } @@ -2111,6 +2365,7 @@ VkDescriptorSetLayout DescriptorCache::Layout(std::span key ++stats.layoutHits; return found->second; } + ValidateDescriptorLayout(context, bindings); ++stats.layoutMisses; VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; info.bindingCount = static_cast(bindings.size()); @@ -2122,21 +2377,54 @@ VkDescriptorSetLayout DescriptorCache::Layout(std::span key } namespace { +std::uint32_t SetDescriptorLimit(const VkPhysicalDeviceLimits& limits, VkDescriptorType type) { + switch (type) { + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: return limits.maxDescriptorSetStorageBuffers; + case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: return limits.maxDescriptorSetSampledImages; + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: return limits.maxDescriptorSetStorageImages; + case VK_DESCRIPTOR_TYPE_SAMPLER: return limits.maxDescriptorSetSamplers; + default: throw std::runtime_error("AGC graphics: descriptor set uses an unsupported descriptor type " + std::to_string(type)); + } +} + // What one chain pool holds; a set needing more of any type gets a dedicated pool. constexpr std::uint32_t ChainPoolSets = 1024; constexpr std::array ChainPoolSizes{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4096}, {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_SAMPLER, 512}}}; +constexpr std::array ChainPoolCreateSizes{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4096 + 2}, {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_SAMPLER, 512}}}; } DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout layout, std::span sizes) { + bool fits = true; for (const auto& size : sizes) { + Require(size.descriptorCount <= SetDescriptorLimit(context.limits, size.type), "descriptor set exceeds the device's per-set descriptor limit"); const auto capacity = std::find_if(ChainPoolSizes.begin(), ChainPoolSizes.end(), [&](const auto& item) { return item.type == size.type; }); - if (capacity == ChainPoolSizes.end() || size.descriptorCount > capacity->descriptorCount) return {}; + if (capacity == ChainPoolSizes.end() || size.descriptorCount > capacity->descriptorCount) fits = false; } std::lock_guard lock(mutex); const auto allocate = context.Resolved(&DeviceFunctions::allocateDescriptorSets, "vkAllocateDescriptorSets"); VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; allocation.descriptorSetCount = 1; allocation.pSetLayouts = &layout; + if (!fits) { + dedicated.reserve(dedicated.size() + 1); + VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + poolInfo.maxSets = 1; + poolInfo.poolSizeCount = static_cast(sizes.size()); + poolInfo.pPoolSizes = sizes.data(); + VkDescriptorPool pool = VK_NULL_HANDLE; + Check(context.Function("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &pool), "vkCreateDescriptorPool"); + allocation.descriptorPool = pool; + VkDescriptorSet set = VK_NULL_HANDLE; + const auto result = allocate(context.device, &allocation, &set); + if (result != VK_SUCCESS) { + destroyPool(context.device, pool, nullptr); + Check(result, "vkAllocateDescriptorSets"); + } + dedicated.push_back(pool); + ++stats.dedicatedPools; + ++stats.sets; + return {set, pool}; + } // Newest pool first: it has the most room; a full or fragmented pool is left for its sets to // drain and tried again later. for (auto it = pools.rbegin(); it != pools.rend(); ++it) { @@ -2152,8 +2440,8 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; poolInfo.flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT; poolInfo.maxSets = ChainPoolSets; - poolInfo.poolSizeCount = static_cast(ChainPoolSizes.size()); - poolInfo.pPoolSizes = ChainPoolSizes.data(); + poolInfo.poolSizeCount = static_cast(ChainPoolCreateSizes.size()); + poolInfo.pPoolSizes = ChainPoolCreateSizes.data(); VkDescriptorPool pool = VK_NULL_HANDLE; Check(context.Function("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &pool), "vkCreateDescriptorPool"); pools.push_back(pool); @@ -2168,6 +2456,11 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l void DescriptorCache::Free(const SetAllocation& allocation) noexcept { if (allocation.set == VK_NULL_HANDLE || allocation.pool == VK_NULL_HANDLE) return; std::lock_guard lock(mutex); + if (const auto found = std::find(dedicated.begin(), dedicated.end(), allocation.pool); found != dedicated.end()) { + dedicated.erase(found); + destroyPool(context.device, allocation.pool, nullptr); + return; + } static_cast(freeSets(context.device, allocation.pool, 1, &allocation.set)); } @@ -2314,6 +2607,12 @@ bool ShaderResources::DataWordsDiffer(const CompiledShader& shader) const { } bool ShaderResources::RefreshData(VkCommandBuffer commands, const CompiledShader& shader, Recorder* recorder) { + Require(shader.program != nullptr, "missing compiled shader"); + Require(!refreshResourceKey.empty() && MatchesContentKey(shader, refreshResourceKey), "runtime data refresh cannot replace bound resources"); + return refreshData(commands, shader, recorder); +} + +bool ShaderResources::refreshData(VkCommandBuffer commands, const CompiledShader& shader, Recorder* recorder) { Require(shader.program != nullptr, "missing compiled shader"); const auto& program = *shader.program; Require(program.bindings.size() == bindings.size(), "template bindings disagree with the shader"); @@ -2330,6 +2629,7 @@ bool ShaderResources::RefreshData(VkCommandBuffer commands, const CompiledShader const auto size = binding.guestDescriptor.size() * sizeof(std::uint32_t); Require(allocation.buffer != nullptr && !allocation.guest && allocation.size == size && size <= MaxRefreshBytes, "template data buffer cannot take the dispatch's words"); if (allocation.dataWords.size() == binding.guestDescriptor.size() && std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), binding.guestDescriptor.begin())) continue; + if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) ValidateRuntimeResources(shader, binding.guestDescriptor); if (recorder != nullptr && timing == Recorder::NoTiming) timing = recorder->BeginGpuTiming(Recorder::CommandClass::TemplateDataRefresh); writeDataWords(commands, bindings[index].allocations.front(), binding.guestDescriptor); allocation.dataWords.assign(binding.guestDescriptor.begin(), binding.guestDescriptor.end()); @@ -2362,13 +2662,21 @@ void ShaderResources::PrecollectSurfaces() const { void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags) { Require(binding.count != 0, "empty descriptor binding"); + const auto heap = static_cast(binding.binding % static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count)); + const auto capacity = binding.count; + Require(binding.count <= ShaderRecompiler::RuntimeAbi::HeapCapacity(heap), "typed descriptor heap capacity exceeded"); + const auto localBinding = static_cast(heap); + const bool sampledHeap = localBinding >= ShaderRecompiler::RuntimeAbi::FirstImageBinding && localBinding < ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding; + const bool storageHeap = localBinding >= ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding && localBinding < static_cast(ShaderRecompiler::RuntimeAbi::Binding::Samplers); + Require((binding.kind == ShaderRecompiler::DescriptorKind::SampledImage && sampledHeap) || (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage && storageHeap) || (binding.kind == ShaderRecompiler::DescriptorKind::Sampler && heap == ShaderRecompiler::RuntimeAbi::Binding::Samplers), "resource class disagrees with typed descriptor heap"); + Require(binding.kind == ShaderRecompiler::DescriptorKind::Sampler || context.nullDescriptors, "typed image heaps require enabled nullDescriptor"); if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { // Storage images are guest textures the shader writes (looked up in stage B). Require(binding.role == ShaderRecompiler::DescriptorRole::GuestImages, "storage image binding has a non-image role"); Require(binding.guestDescriptor.size() == static_cast(binding.count) * 8u, "guest storage image descriptors must contain 8 dwords each"); Require(context.detiler != nullptr, "device texture detiler is unavailable"); - Require(binding.count <= context.limits.maxPerStageDescriptorStorageImages, "shader storage-image descriptors exceed per-stage limits"); - bindings.push_back({{binding.binding, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, binding.count, flags, nullptr}, {}, {}}); + Require(capacity <= context.limits.maxPerStageDescriptorStorageImages, "shader storage-image descriptors exceed per-stage limits"); + bindings.push_back({{binding.binding, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, capacity, flags, nullptr}, {}, {}}); plannedStorageImages += binding.count; deferredImages.push_back({&binding, bindings.size() - 1}); return; @@ -2380,19 +2688,19 @@ void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& Require(binding.guestDescriptor.size() % binding.count == 0, "guest image descriptor size is not a multiple of the binding count"); const auto elementWords = binding.guestDescriptor.size() / binding.count; - Binding item{{binding.binding, sampledImage ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_SAMPLER, binding.count, flags, nullptr}, {}, {}}; + Binding item{{binding.binding, sampledImage ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_SAMPLER, capacity, flags, nullptr}, {}, {}}; if (sampledImage) { Require(elementWords == 8, "guest texture descriptor must contain 8 dwords"); Require(binding.imageShape.has_value(), "guest image binding is missing an image shape"); Require(context.detiler != nullptr, "device texture detiler is unavailable"); Require(context.textureCache != nullptr, "device texture cache is unavailable"); - Require(binding.count <= context.limits.maxPerStageDescriptorSampledImages, "shader sampled-image descriptors exceed per-stage limits"); + Require(capacity <= context.limits.maxPerStageDescriptorSampledImages, "shader sampled-image descriptors exceed per-stage limits"); plannedSampledImages += binding.count; Require(plannedSampledImages <= context.limits.maxDescriptorSetSampledImages, "pipeline sampled-image descriptors exceed device limits"); } else { Require(elementWords == 4, "guest sampler descriptor must contain 4 dwords"); - Require(binding.count <= context.limits.maxPerStageDescriptorSamplers, "shader sampler descriptors exceed per-stage limits"); + Require(capacity <= context.limits.maxPerStageDescriptorSamplers, "shader sampler descriptors exceed per-stage limits"); Require(binding.samplerDepthCompare.size() == binding.count, "guest sampler binding is missing depth comparison metadata"); Require(binding.samplerUnnormalized.empty() || binding.samplerUnnormalized.size() == binding.count, "guest sampler binding has unnormalized coordinate metadata of another size"); for (std::uint32_t element = 0; element < binding.count; ++element) { @@ -2459,7 +2767,7 @@ bool ShaderResources::precollectImages() { const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); ImageRecord record; record.sampled = binding.kind == ShaderRecompiler::DescriptorKind::SampledImage; - try { + if (words[0] != 0u || (words[1] & 0xffu) != 0u) { record.resource = DecodeTextureResource(words); record.guestBytes = DescribeSurface(record.resource).guestBytes; record.generation = GuestMemory::CollectWrites(record.resource.baseAddress, static_cast(record.guestBytes)); @@ -2478,8 +2786,6 @@ bool ShaderResources::precollectImages() { counters.records.fetch_add(1, std::memory_order_relaxed); } } - } catch (const std::exception&) { - record.decoded = false; } imageRecords.push_back(std::move(record)); } @@ -2552,6 +2858,11 @@ void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBind for (std::uint32_t element = 0; element < binding.count; ++element) { const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); const auto* record = nextRecord(); + if (words[0] == 0u && (words[1] & 0xffu) == 0u) { + item.imageAllocations.push_back(std::numeric_limits::max()); + --plannedSampledImages; + continue; + } const auto resource = record != nullptr && record->decoded ? record->resource : DecodeTextureResource(words); const bool firstLayer = binding.imageShape == ShaderRecompiler::DescriptorImageShape::Image2D && resource.dimension == TextureDimension::k2DArray; if (!firstLayer && !MatchesGuestDimension(*binding.imageShape, resource.dimension)) throw std::runtime_error("AGC graphics: guest texture dimension disagrees with the shader's declared image shape (shape " + std::to_string(static_cast(*binding.imageShape)) + ", dimension " + std::to_string(static_cast(resource.dimension)) + ")"); @@ -2587,6 +2898,11 @@ void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBind if (sameAsPrevious) ++mipOffset; else mipOffset = 0; const auto* record = nextRecord(); + if (words[0] == 0u && (words[1] & 0xffu) == 0u) { + item.imageAllocations.push_back(std::numeric_limits::max()); + --plannedStorageImages; + continue; + } const auto resource = record != nullptr && record->decoded ? record->resource : DecodeTextureResource(words); const bool firstLayer = binding.imageShape == ShaderRecompiler::DescriptorImageShape::Image2D && resource.dimension == TextureDimension::k2DArray; if (binding.imageShape.has_value() && !firstLayer && !MatchesGuestDimension(*binding.imageShape, resource.dimension)) throw std::runtime_error("AGC graphics: guest storage texture dimension disagrees with the shader's declared image shape (shape " + std::to_string(static_cast(*binding.imageShape)) + ", dimension " + std::to_string(static_cast(resource.dimension)) + ")"); @@ -2650,12 +2966,9 @@ std::vector> ShaderResources::PresyncSur // Stage A has not run (an address-based build, or the pass is off): decoded from the shader. if (deferredCompute.program == nullptr) return surfaces; forEachImageElement(*deferredCompute.program, [&](const ShaderRecompiler::DescriptorBinding& binding, std::uint32_t, std::span words) { - try { - const auto resource = DecodeTextureResource(words); - consider(resource.format, resource.baseAddress, DescribeSurface(resource).guestBytes, binding.kind == ShaderRecompiler::DescriptorKind::SampledImage); - } catch (const std::exception&) { - // Stage B reports the bad descriptor. - } + if (words[0] == 0u && (words[1] & 0xffu) == 0u) return; + const auto resource = DecodeTextureResource(words); + consider(resource.format, resource.baseAddress, DescribeSurface(resource).guestBytes, binding.kind == ShaderRecompiler::DescriptorKind::SampledImage); }); return surfaces; } @@ -2721,7 +3034,7 @@ std::optional> ShaderResources::MovedR const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; if (written || empty || item.written || size > context.limits.maxStorageBufferRange) return std::nullopt; const auto begin = address - item.adjustment; - const auto bytes = static_cast(size) + item.adjustment; + const auto bytes = static_cast(GuestBufferMemory::ViewBytes(size, item.adjustment)); if (!GuestMemory::Accessible(reinterpret_cast(begin), bytes)) return std::nullopt; if (guestMemory.WritesOverlap(begin, bytes) || recorder.PendingWriteOverlaps(begin, bytes) || PendingStorageOverlaps(begin, bytes, nullptr) || AnyShadowedOverlaps(begin, bytes)) return std::nullopt; moved.push_back({index, address, static_cast(size)}); @@ -2760,7 +3073,7 @@ std::shared_ptr ShaderResources::PrepareDrawBindi if (!direct || recorder.PendingWriteOverlaps(item.address, item.size)) continue; } const auto begin = address - item.adjustment; - const auto bytes = size + item.adjustment; + const auto bytes = static_cast(GuestBufferMemory::ViewBytes(size, item.adjustment)); const auto registryGeneration = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); const auto generation = GuestMemory::CollectWrites(begin, bytes); auto buffer = recorder.ReusableDrawSnapshot(begin, bytes); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp index 883c9a8ea..527ef1087 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp @@ -1,5 +1,6 @@ #include "BdaAbi.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" #include #include @@ -132,6 +133,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi using Stage = ShaderRecompiler::ShaderStage; Require(compiled.program != nullptr, "missing compiled shader"); const auto& shader = *compiled.program; + PipelineSpecialization::Validate(shader); const auto stage = compiled.stage; const bool vertex = stage == Stage::Vertex || stage == Stage::Local; const bool fragment = stage == Stage::Fragment; @@ -181,6 +183,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi case spv::CapabilityGroupNonUniform: subgroupOperations = VK_SUBGROUP_FEATURE_BASIC_BIT; break; case spv::CapabilityGroupNonUniformBallot: subgroupOperations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_BALLOT_BIT; break; case spv::CapabilityGroupNonUniformShuffle: subgroupOperations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_SHUFFLE_BIT; break; + case spv::CapabilityGroupNonUniformArithmetic: subgroupOperations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_ARITHMETIC_BIT; break; default: break; } const bool isSubgroupCapability = subgroupOperations != 0; @@ -207,6 +210,10 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi mesh && capability == spv::CapabilityMeshShadingEXT; + const bool isInterlockCapability = + fragment && + capability == spv::CapabilityFragmentShaderPixelInterlockEXT; + // Enabled unconditionally or by the device setup in VulkanDevice. const bool isFeatureCapability = capability == spv::CapabilitySampled1D || @@ -215,6 +222,8 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi capability == spv::CapabilityImageQuery || (fragment && geometryShader && capability == spv::CapabilityGeometry) || (fragment && sampleRateShading && capability == spv::CapabilitySampleRateShading) || + capability == spv::CapabilityImageMSArray || + capability == spv::CapabilityStorageImageMultisample || capability == spv::CapabilityStorageImageWriteWithoutFormat || capability == spv::CapabilityStorageImageReadWithoutFormat || capability == spv::CapabilityInt64 || @@ -241,6 +250,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi isBdaCapability || isTessellationCapability || isMeshCapability || + isInterlockCapability || isFeatureCapability || isDescriptorIndexingCapability || (imageInt64Atomics && (capability == spv::CapabilityInt64Atomics || capability == spv::CapabilityInt64ImageEXT)), @@ -256,6 +266,10 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const auto end = std::find(text, text + bytes.size(), '\0'); Require(end != text + bytes.size(), "unterminated SPIR-V extension"); const std::string_view extension(text, static_cast(end - text)); + if (extension == "SPV_EXT_fragment_shader_interlock") { + Require(fragment, "SPV_EXT_fragment_shader_interlock requires a fragment shader"); + break; + } if (extension == "SPV_KHR_fragment_shader_barycentric") { Require(fragment && fragmentShaderBarycentric, "SPV_KHR_fragment_shader_barycentric requires enabled fragmentShaderBarycentric in a fragment shader"); break; @@ -267,7 +281,6 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi case spv::OpDecorationGroup: case spv::OpGroupDecorate: case spv::OpGroupMemberDecorate: - case spv::OpSpecConstant: case spv::OpSpecConstantTrue: case spv::OpSpecConstantFalse: case spv::OpSpecConstantComposite: @@ -345,6 +358,8 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi case spv::OpConstant: if (count == 4) module.constants.emplace(instruction[2], instruction[3]); break; + case spv::OpSpecConstant: + break; default: break; } cursor += count; @@ -376,7 +391,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi mode(spv::ExecutionModeVertexOrderCw, {}); Require(module.modes.size() == 3, "unsupported tessellation-evaluation execution mode"); } else if (fragment) { - for (const auto& [name, operands] : module.modes) Require(operands.empty() && (name == spv::ExecutionModeOriginUpperLeft || name == spv::ExecutionModeEarlyFragmentTests || name == spv::ExecutionModeDepthReplacing || name == spv::ExecutionModeDepthLess || name == spv::ExecutionModeDepthGreater), "unsupported fragment execution mode"); + for (const auto& [name, operands] : module.modes) Require(operands.empty() && (name == spv::ExecutionModeOriginUpperLeft || name == spv::ExecutionModeEarlyFragmentTests || name == spv::ExecutionModeDepthReplacing || name == spv::ExecutionModeDepthLess || name == spv::ExecutionModeDepthGreater || name == spv::ExecutionModePixelInterlockOrderedEXT), "unsupported fragment execution mode"); } else Require(module.modes.empty(), "unsupported vertex execution mode"); std::set> descriptors; bool push = false; @@ -451,6 +466,11 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const auto binding = std::find_if(shader.bindings.begin(), shader.bindings.end(), [&](const auto& item) { return item.descriptorSet == key.first && item.binding == key.second; }); Require(binding != shader.bindings.end(), "SPIR-V resource is absent from recompiler binding metadata"); Require(binding->kind == ShaderRecompiler::DescriptorKind::Sampler || binding->kind == ShaderRecompiler::DescriptorKind::SampledImage || binding->kind == ShaderRecompiler::DescriptorKind::StorageImage, "SPIR-V descriptor type disagrees with recompiler binding metadata"); + Require(type.size() == 4 && (type[0] & 0xffffu) == spv::OpTypeArray, "typed heap must be declared as a descriptor array"); + const auto heap = static_cast(binding->binding % static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count)); + const auto capacity = ShaderRecompiler::RuntimeAbi::HeapCapacity(heap); + const auto count = module.constants.find(type[3]); + Require(binding->count != 0u && binding->count <= capacity && count != module.constants.end() && count->second == binding->count, "typed heap array length disagrees with compact binding"); } else { Require(variable.storage == spv::StorageClassStorageBuffer && decoration.set && decoration.binding, "unsupported or unbound shader resource"); const auto key = std::make_pair(*decoration.set, *decoration.binding); @@ -519,7 +539,12 @@ std::set ValidateShaders(std::span shaders, if (i != 0) { for (const auto& [location, signature] : current.inputs) { const auto output = previous.outputs.find(location); - Require(output != previous.outputs.end() && output->second == signature, "graphics interfaces disagree at location " + std::to_string(location)); + if (output == previous.outputs.end() || output->second != signature) { + Require(false, "graphics interfaces disagree at location " + std::to_string(location) + + " (stage " + std::to_string(i - 1) + " output " + + (output == previous.outputs.end() ? "missing" : output->second) + + ", stage " + std::to_string(i) + " input " + signature + ")"); + } } } previous = current; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp index 53288a665..dc0ab6fb9 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp @@ -56,6 +56,43 @@ std::string vteMessage(std::uint32_t viewportControl) { return message.str(); } +std::string conservativeMessage(std::uint32_t control) { + std::ostringstream message; + message << "AGC graphics: PA_SC_CONSERVATIVE_RASTERIZATION_CNTL=0x" << std::hex << control << ": only 0x6000 (off) and 0x6001 (overestimation) are supported"; + return message.str(); +} + +void requireConservativeTriangles(bool triangles, const char* primitive, const char* source, std::uint32_t value) { + if (triangles) return; + std::ostringstream message; + message << "AGC graphics: conservative rasterization of " << primitive << " is unsupported (" << source << "=0x" << std::hex << value << "): only triangles are overestimated"; + throw std::runtime_error(message.str()); +} + +VkConservativeRasterizationModeEXT decodeConservativeRasterization(const QueueState& queue) { + const auto& cx = queue.context; + const auto control = read(cx, 0x313); + if (control == 0x6000u) return VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT; + if (control != 0x6001u) throw std::runtime_error(conservativeMessage(control)); + const auto stages = read(cx, 0x2d5); + if ((stages & 0x20u) != 0) { + const auto output = read(cx, 0x29b); + const auto type = output & 0x3fu; + const bool perStream = (output & 0x80000000u) != 0; + requireConservativeTriangles(!perStream && type == 2u, perStream ? "per-stream primitive types" : type == 0u ? "points" : type == 1u ? "lines" : "other primitive types", "VGT_GS_OUT_PRIM_TYPE", output); + } else if ((stages & 4u) != 0) { + const auto parameters = read(cx, 0x2db); + const auto domain = parameters & 3u; + const auto topology = (parameters >> 5u) & 7u; + requireConservativeTriangles((domain == 1u || domain == 2u) && (topology == 2u || topology == 3u), topology == 0u ? "points" : topology == 1u || domain == 0u ? "lines" : "other primitive types", "VGT_TF_PARAM", parameters); + } else { + const auto primitive = read(queue.userConfig, 0x242, RegisterBank::UserConfig); + requireConservativeTriangles(primitive == 4u || primitive == 5u || primitive == 6u, primitive == 1u ? "points" : primitive == 2u || primitive == 3u ? "lines" : primitive == 7u || primitive == 17u ? "rectangles" : "other primitive types", "VGT_PRIMITIVE_TYPE", primitive); + } + Require((read(cx, 0x1b3) & 0x44u) == 0, "conservative rasterization with centroid interpolation is unsupported"); + return VK_CONSERVATIVE_RASTERIZATION_MODE_OVERESTIMATE_EXT; +} + // The register rules DecodeState and DrawRejection share (the precheck must reject exactly what // DecodeState would): the masks whose set bits are unsupported, and the depth-control verdict. // Render target index, viewport index and the misc export vector that carries them are accepted but @@ -64,9 +101,10 @@ constexpr std::uint32_t LayerExports = (1u << 18u) | (1u << 19u) | (1u << 21u) | constexpr std::uint32_t DepthControlMask = ~0x007007f0u; // EXEC_ON_HIER_FAIL / EXEC_ON_NOOP / EXEC_IF_OVERLAPPED (bits 9, 10, 17) only force the pixel shader // to run, which it always does here. -constexpr std::uint32_t ShaderControlMask = ~(0x0000f870u | 0x00020600u); +constexpr std::uint32_t ShaderControlMask = ~(0x0000f870u | 0x00020600u | 0x00010000u); +constexpr std::uint32_t PixelStageRunsMask = 0x00020747u; constexpr std::uint32_t AlphaToCoverageMask = ~0x0001ff00u; -constexpr std::uint32_t ScanModeMask = ~2u; +constexpr std::uint32_t ScanModeMask = ~0x22u; constexpr std::uint32_t ScanControlMask = ~0x06023fffu; constexpr std::uint32_t ScreenOffsetMask = ~0x01ff01ffu; // Bits 26/27 (ZCLIP_NEAR/FAR_DISABLE) become depth clamping; bit 19 selects the [0, 1] clip space. @@ -234,7 +272,8 @@ std::uint32_t effectiveDepthControl(std::uint32_t depthControl) { } bool colorControlSupported(std::uint32_t colorControl, bool hasColorTarget) { - return colorControl == 0xcc0010u || (!hasColorTarget && (colorControl & ~0x70u) == 0xcc0000u); + colorControl &= ~1u; + return colorControl == 0xcc0010u || ((colorControl >> 4u) & 7u) == 0u || (!hasColorTarget && (colorControl & ~0x70u) == 0xcc0000u); } std::string colorControlMessage(std::uint32_t colorControl) { @@ -292,12 +331,13 @@ DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number, }; const bool alternate = swap == 1; const auto single = [&](VkFormat vkFormat, std::uint32_t bytes) { return DecodedColorFormat{vkFormat, bytes, static_cast((0xe4u & ~3u) | swap)}; }; - if (swap > 1 && format != 1 && format != 2 && format != 4) return fail(); + if (swap > 1 && format != 1 && format != 2 && format != 4 && format != 10 && format != 12) return fail(); switch (format) { case 1: if (number == unorm) return single(VK_FORMAT_R8_UNORM, 1); if (number == snorm) return single(VK_FORMAT_R8_SNORM, 1); if (number == uint) return single(VK_FORMAT_R8_UINT, 1); + if (number == srgb && swap == 0) return single(VK_FORMAT_R8_SRGB, 1); return fail(); case 2: if (number == unorm) return single(VK_FORMAT_R16_UNORM, 2); @@ -330,23 +370,27 @@ DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number, // COLOR_2_10_10_10 keeps red in the low bits, the Vulkan A2B10G10R10 packing. if (number != unorm) return fail(); return {alternate ? VK_FORMAT_A2R10G10B10_UNORM_PACK32 : VK_FORMAT_A2B10G10R10_UNORM_PACK32, 4}; - case 10: - if (number == unorm) return {alternate ? VK_FORMAT_B8G8R8A8_UNORM : VK_FORMAT_R8G8B8A8_UNORM, 4}; - if (number == snorm) return {alternate ? VK_FORMAT_B8G8R8A8_SNORM : VK_FORMAT_R8G8B8A8_SNORM, 4}; - if (number == srgb) return {alternate ? VK_FORMAT_B8G8R8A8_SRGB : VK_FORMAT_R8G8B8A8_SRGB, 4}; + case 10: { + const auto reversed = static_cast(swap == 2 ? 0x1bu : swap == 3 ? 0x93u : 0xe4u); + if (number == unorm) return {alternate ? VK_FORMAT_B8G8R8A8_UNORM : VK_FORMAT_R8G8B8A8_UNORM, 4, reversed}; + if (number == snorm) return {alternate ? VK_FORMAT_B8G8R8A8_SNORM : VK_FORMAT_R8G8B8A8_SNORM, 4, reversed}; + if (number == srgb) return {alternate ? VK_FORMAT_B8G8R8A8_SRGB : VK_FORMAT_R8G8B8A8_SRGB, 4, reversed}; return fail(); + } case 11: if (swap != 0) return fail(); if (number == floating) return {VK_FORMAT_R32G32_SFLOAT, 8}; if (number == uint) return {VK_FORMAT_R32G32_UINT, 8}; return fail(); - case 12: - if (swap != 0) return fail(); - if (number == floating) return {VK_FORMAT_R16G16B16A16_SFLOAT, 8}; - if (number == unorm) return {VK_FORMAT_R16G16B16A16_UNORM, 8}; - if (number == snorm) return {VK_FORMAT_R16G16B16A16_SNORM, 8}; - if (number == uint) return {VK_FORMAT_R16G16B16A16_UINT, 8}; + case 12: { + if (swap == 1) return fail(); + const auto reversed = static_cast(swap == 2 ? 0x1bu : swap == 3 ? 0x93u : 0xe4u); + if (number == floating) return {VK_FORMAT_R16G16B16A16_SFLOAT, 8, reversed}; + if (number == unorm) return {VK_FORMAT_R16G16B16A16_UNORM, 8, reversed}; + if (number == snorm) return {VK_FORMAT_R16G16B16A16_SNORM, 8, reversed}; + if (number == uint) return {VK_FORMAT_R16G16B16A16_UINT, 8, reversed}; return fail(); + } case 14: if (swap != 0) return fail(); if (number == floating) return {VK_FORMAT_R32G32B32A32_SFLOAT, 16}; @@ -396,7 +440,7 @@ ShaderStages DecodeShaderStages(const QueueState& queue) { APS5_LOG_OUT_DEBUG("DecodeShaderStages value=0x%x primitive=%u path=%u vertexWave=%u", value, primitive, static_cast(result.path), result.vertexWaveSize); if (path == ShaderPath::Vertex) { validate((value & 0x2000u) != 0, "legacy vertex routing without PRIMGEN_EN is unsupported"); - validate((value & ~0x02402010u) == 0, "unsupported vertex routing, scheduling or wave-ID state"); + validate((value & ~0x0247a010u) == 0, "unsupported vertex routing, scheduling or wave-ID state"); } else if (path == ShaderPath::Tessellation) { validate((value & 0x00600020u) == 0, "wave32 tessellation or geometry amplification is unsupported"); validate((value & ~0x0007ed0du) == 0 && (value & 3u) == 1u && ((value >> 3u) & 3u) == 1u, "unsupported tessellation routing"); @@ -429,9 +473,17 @@ ShaderStages DecodeShaderStages(const QueueState& queue) { return result; } +std::string DepthMaintenanceRejection(const QueueState& queue) { + const auto control = find(queue.context, 0x000); + if (control == queue.context.end() || (control->second & ~0x2063u) == 0) return {}; + return zeroMessage(0x000, control->second, "DB_RENDER_CONTROL depth copy, resummarize or decompress"); +} + State DecodeState(const QueueState& queue) { + if (auto reason = DepthMaintenanceRejection(queue); !reason.empty()) throw std::runtime_error(reason); const auto& cx = queue.context; State result{}; + result.conservativeRasterization = decodeConservativeRasterization(queue); result.stages = DecodeShaderStages(queue); const auto primitive = read(queue.userConfig, 0x242, RegisterBank::UserConfig); APS5_LOG_OUT_DEBUG("DecodeState primitive=%u path=%u vertexWave=%u", primitive, static_cast(result.stages.path), result.stages.vertexWaveSize); @@ -492,7 +544,6 @@ State DecodeState(const QueueState& queue) { Require(read(cx, 0x83) == 0xffffu, "clip rectangles are unsupported"); Require((read(cx, 0x8c) & 0xfu) == 0xau, "nonstandard triangle edge rules are unsupported"); Require(read(cx, 0x2f9) == 0x2du, "nonstandard pixel center or vertex quantization is unsupported"); - Require(read(cx, 0x313) == 0x6000u, "conservative rasterization is unsupported"); Require(read(cx, 0x30e) == 0xffffffffu && read(cx, 0x30f) == 0xffffffffu, "sample masks are unsupported"); const auto viewportControl = read(cx, 0x206); if (viewportControl != 0x43fu) throw std::runtime_error(vteMessage(viewportControl)); @@ -512,7 +563,7 @@ State DecodeState(const QueueState& queue) { const auto shaderMask = read(cx, 0x8f); // Channels of targets the pixel shader does not export are never written, so the target mask only // matters where the shader exports. - const auto targetMask = read(cx, 0x8e) & shaderMask; + const auto targetMask = ColorWriteMask(cx); APS5_LOG_OUT_DEBUG("CB_TARGET_MASK=0x%x CB_SHADER_MASK=0x%x", targetMask, shaderMask); std::vector exportSlots; for (std::uint32_t slot = 0; slot < 8; ++slot) { @@ -523,6 +574,7 @@ State DecodeState(const QueueState& queue) { for (std::uint32_t index = 0; index < exportSlots.size(); ++index) { if (written(exportSlots[index])) exportCount = index + 1; } + if (((read(cx, 0x202) >> 4u) & 7u) == 0u) exportCount = 0; result.hasColorTarget = exportCount != 0; APS5_LOG_OUT_DEBUG("hasColorTarget=%u exports=%u", result.hasColorTarget ? 1u : 0u, exportCount); @@ -598,6 +650,7 @@ State DecodeState(const QueueState& queue) { if (((exportedMask >> ((mapping >> (2u * component)) & 3u)) & 1u) != 0) state.colorWriteMask |= 1u << component; } state.blendEnable = (blend >> 30u) & 1u; + if (state.blendEnable && (mapping == 0x1bu || mapping == 0x93u)) throw std::runtime_error("AGC graphics: blending into a color target with a reversed component order is not implemented"); if (state.blendEnable) { Require((read(cx, 0x31c + slot * 0xfu) & 0x10000u) == 0, "blend bypass conflicts with enabled blending"); state.srcColorBlendFactor = blendFactor(blend & 0x1fu); @@ -630,6 +683,24 @@ std::array ExportMappings(const State& state) { return mappings; } +std::size_t CmaskBytes(std::uint32_t width, std::uint32_t height) { + constexpr std::size_t metablockWidth = 1024; + constexpr std::size_t metablockHeight = 512; + constexpr std::size_t metablockBytes = 4096; + return ((width + metablockWidth - 1) / metablockWidth) * ((height + metablockHeight - 1) / metablockHeight) * metablockBytes; +} + +std::uint32_t ColorWriteMask(const Registers& context) { + auto mask = read(context, 0x8e) & read(context, 0x8f); + for (std::uint32_t slot = 0; slot < 8; ++slot) { + const auto channels = 0xfu << (slot * 4u); + if ((mask & channels) == 0) continue; + const auto info = find(context, 0x31c + slot * 0xfu); + if (info != context.end() && ((info->second >> 2u) & 0x1fu) == 0) mask &= ~channels; + } + return mask; +} + ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot) { const auto stride = slot * 0xfu; ColorTarget color{}; @@ -641,7 +712,7 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot) { const auto format = (info >> 2u) & 0x1fu; const auto decoded = DecodeColorFormat(format, number, swap); // ROUND_MODE (bit 18) only affects unorm rounding. With DCC_ENABLE (bit 28) the target is written - if ((info & ~(0x00039f7cu | 0x00040000u | 0x10000000u)) != 0) throw std::runtime_error("AGC graphics: color compression, DCC, endian conversion, nonstandard rounding or color optimization is unsupported (CB_COLOR_INFO 0x" + [&] { char text[16]; std::snprintf(text, sizeof(text), "%08x", info); return std::string(text); }() + ")"); + if ((info & ~(0x00039f7cu | 0x00040000u | 0x10000000u | 0x00002000u)) != 0) throw std::runtime_error("AGC graphics: color compression, DCC, endian conversion, nonstandard rounding or color optimization is unsupported (CB_COLOR_INFO 0x" + [&] { char text[16]; std::snprintf(text, sizeof(text), "%08x", info); return std::string(text); }() + ")"); Require((info & 0x8000u) != 0 || number == 7 || number == 4 || number == 5, "unclamped normalized color is unsupported"); const auto view = read(cx, 0x31b + stride); Require((view & ~0x3fffffffu) == 0, "reserved CB_COLOR_VIEW bits are set"); @@ -689,11 +760,19 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot) { const auto clear = find(cx, 0x323 + word + stride); color.clearWords[word] = clear == cx.end() ? 0u : clear->second; } + if ((info & 0x2000u) != 0) { + Require(maxMip == 0 && !volume && slice == 0, "CMASK fast clears of a mipmapped, 3D or array color target are unsupported"); + const auto cmaskHigh = find(cx, 0x398 + slot); + color.cmaskAddress = ((cmaskHigh == cx.end() ? 0ull : static_cast(cmaskHigh->second & 0xffu)) << 40u) | (static_cast(read(cx, 0x31f + stride)) << 8u); + Require(color.cmaskAddress != 0, "CMASK fast clears without a CMASK address are unsupported"); + color.cmaskBytes = CmaskBytes(color.extent.width, color.extent.height); + } if ((info & 0x10000000u) != 0) { if (maxMip == 0) { const auto dccHigh = find(cx, 0x3a8 + slot); color.dccAddress = ((dccHigh == cx.end() ? 0ull : static_cast(dccHigh->second & 0xffu)) << 40u) | (static_cast(read(cx, 0x325 + stride)) << 8u); color.dccAlphaOnMsb = DccAlphaOnMsb(color.format, swap); + color.dccPipeAligned = ((attrib3 >> 30u) & 1u) != 0; } else { static bool reported = false; if (!reported) { @@ -743,6 +822,7 @@ std::optional DecodeColorMetadataPass(const QueueState& queue } std::string DrawRejection(const QueueState& queue, bool indexed) { + if (auto reason = DepthMaintenanceRejection(queue); !reason.empty()) return reason; const auto& cx = queue.context; // A register a rule needs that is absent gives no verdict here: DecodeState reports it. const auto value = [&](const Registers& registers, std::uint32_t offset, std::uint32_t& out) { @@ -783,15 +863,15 @@ std::string DrawRejection(const QueueState& queue, bool indexed) { if (value(cx, 0x83, word) && word != 0xffffu) return require(false, "clip rectangles are unsupported"); if (value(cx, 0x8c, word) && (word & 0xfu) != 0xau) return require(false, "nonstandard triangle edge rules are unsupported"); if (value(cx, 0x2f9, word) && word != 0x2du) return require(false, "nonstandard pixel center or vertex quantization is unsupported"); - if (value(cx, 0x313, word) && word != 0x6000u) return require(false, "conservative rasterization is unsupported"); + if (value(cx, 0x313, word) && word != 0x6000u && word != 0x6001u) return conservativeMessage(word); std::uint32_t other = 0; if (value(cx, 0x30e, word) && value(cx, 0x30f, other) && (word != 0xffffffffu || other != 0xffffffffu)) return require(false, "sample masks are unsupported"); if (value(cx, 0x206, word) && word != 0x43fu) return vteMessage(word); if (auto reason = nonzero(cx, 0x204, ClipControlMask, "unsupported PA_CL_CLIP_CNTL flags"); !reason.empty()) return reason; std::uint32_t targetMask = 0, shaderMask = 0; - if (value(cx, 0x8e, targetMask) && value(cx, 0x8f, shaderMask) && value(cx, 0x202, word) && !colorControlSupported(word, (targetMask & shaderMask) != 0)) return colorControlMessage(word); + if (value(cx, 0x8e, targetMask) && value(cx, 0x8f, shaderMask) && value(cx, 0x202, word) && !colorControlSupported(word, ColorWriteMask(cx) != 0)) return colorControlMessage(word); if (auto reason = nonzero(cx, 0x1c4, zFormatSupported(zFormat) ? 0u : ~0u, "depth or sample-mask export"); !reason.empty()) return reason; - if (PixelProgramUnset(queue)) return NullPixelProgramRejection(queue); + if (PixelProgramSkipped(queue)) return NullPixelProgramRejection(queue); for (const auto offset : {0x1b3u, 0x1b4u, 0x1c5u}) { if (find(cx, offset) != cx.end()) continue; char text[64]; @@ -801,17 +881,24 @@ std::string DrawRejection(const QueueState& queue, bool indexed) { return {}; } -bool PixelProgramUnset(const QueueState& queue) { +bool PixelProgramSkipped(const QueueState& queue) { const auto low = find(queue.shader, 0x008, RegisterBank::Shader); const auto high = find(queue.shader, 0x009, RegisterBank::Shader); - return low != queue.shader.end() && high != queue.shader.end() && low->second == 0 && high->second == 0; + if (low != queue.shader.end() && high != queue.shader.end() && low->second == 0 && high->second == 0) return true; + const auto& cx = queue.context; + const auto targetMask = find(cx, 0x8e); + const auto shaderMask = find(cx, 0x8f); + const auto zFormat = find(cx, 0x1c4); + const auto shaderControl = find(cx, 0x203); + if (targetMask == cx.end() || shaderMask == cx.end() || zFormat == cx.end() || shaderControl == cx.end()) return false; + return (targetMask->second & shaderMask->second) == 0 && zFormat->second == 0 && (shaderControl->second & PixelStageRunsMask) == 0; } std::string NullPixelProgramRejection(const QueueState& queue) { const auto targetMask = find(queue.context, 0x8e); const auto shaderMask = find(queue.context, 0x8f); if (targetMask == queue.context.end() || shaderMask == queue.context.end()) return "AGC graphics: a draw without a pixel program needs CB_TARGET_MASK and CB_SHADER_MASK"; - if ((targetMask->second & shaderMask->second) != 0) return "AGC graphics: a draw without a pixel program writes color"; + if (ColorWriteMask(queue.context) != 0) return "AGC graphics: a draw without a pixel program writes color"; return {}; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 249b7f754..312921acd 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -223,6 +223,15 @@ void ChainMinLod(const Context& context, const GuestTextureResource& descriptor, viewInfo.pNext = &minLod; } +std::atomic& SampledMemoryCounter() { + static std::atomic bytes{0}; + return bytes; +} + +} + +std::uint64_t SampledTextureMemory() { + return SampledMemoryCounter().load(std::memory_order_relaxed); } Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, VkComponentMapping components, std::span snapshot, bool depthCompare) : context(context) { @@ -312,9 +321,9 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex VkCommandBuffer commands = VK_NULL_HANDLE; if (recorder != nullptr) { commands = recorder->Commands(); - recorder->Keep(staging); - recorder->Keep(tiled); - recorder->Keep(linear); + recorder->Keep(staging, static_cast(guestBytes)); + recorder->Keep(tiled, static_cast(guestBytes)); + recorder->Keep(linear, static_cast(linearBytes)); recorder->Keep(owned); } else { batch = std::make_unique(context); @@ -438,6 +447,8 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex totals.view += timer.lap(); if (++totals.count % 200 == 0) std::fprintf(stderr, "[texture] %llu textures (%llu copied from storage images, %llu uploads recorded): allocate+image %.0f ms, guest read %.0f ms, detile+copy %.0f ms, view+buffer release %.0f ms\n", static_cast(totals.count), static_cast(totals.fromStorage), static_cast(totals.recordedUploads), totals.allocate, totals.read, totals.gpu, totals.view); } + countedBytes = allocationBytes; + SampledMemoryCounter().fetch_add(countedBytes, std::memory_order_relaxed); } catch (...) { release(); throw; @@ -525,6 +536,7 @@ void Texture::release() noexcept { // The image and its memory go with the last holder: this texture, or the batch still uploading it. owned.reset(); image = VK_NULL_HANDLE; + SampledMemoryCounter().fetch_sub(std::exchange(countedBytes, 0), std::memory_order_relaxed); } VkImageView Texture::View() const { @@ -655,6 +667,28 @@ bool StorageClearAvailable(const Context& context, std::uint32_t guestFormat, Dc return IsDccClear(keys) && StorageFormatAvailable(context, guestFormat) && ClearColorFor(StorageFormatForGuest(context, guestFormat), keys, clear); } +VkFormat AttachmentProxyFormat(const Context& context, VkFormat format) { + if (format != VK_FORMAT_R8_SRGB) return VK_FORMAT_UNDEFINED; + struct Table { + std::mutex mutex; + std::unordered_map formats; + }; + static Table table; + { + std::lock_guard lock(table.mutex); + if (const auto found = table.formats.find(context.physical); found != table.formats.end()) return found->second; + } + const char* forced = std::getenv("APS5_SRGB_ATTACHMENT_PROXY"); + VkFormatProperties properties{}; + context.formatProperties(context.physical, format, &properties); + constexpr VkFormatFeatureFlags attachment = VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT | VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT; + const bool proxied = (forced != nullptr && std::strcmp(forced, "1") == 0) || (properties.optimalTilingFeatures & attachment) != attachment; + const auto proxy = proxied ? VK_FORMAT_R8G8B8A8_SRGB : VK_FORMAT_UNDEFINED; + std::lock_guard lock(table.mutex); + table.formats.emplace(context.physical, proxy); + return proxy; +} + StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, std::uint32_t mipLevel) : context(context), detiler(detiler), descriptor(descriptor) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; PhaseTimer timer; @@ -708,6 +742,7 @@ StorageTexture::StorageTexture(const Context& context, TextureDetiler& detiler, context.Function("vkGetImageMemoryRequirements")(context.device, image, &requirements); VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; + memoryBytes = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory storage texture"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory storage"); @@ -1006,6 +1041,95 @@ VkImageView StorageTexture::AttachmentView(VkFormat format, std::uint32_t mip, s return created; } +VkImageView StorageTexture::AttachmentProxyView() { + if (proxyView != VK_NULL_HANDLE) return proxyView; + Require(storageFormat == VK_FORMAT_R8_UNORM && descriptor.dimension == TextureDimension::k2D && descriptor.mipCount == 1 && geometry.imageLayers == 1, "an RGBA8_SRGB attachment proxy is only modeled for a single-mip 2D R8 surface"); + try { + VkImageCreateInfo imageInfo{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + imageInfo.flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; + imageInfo.imageType = VK_IMAGE_TYPE_2D; + imageInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + imageInfo.extent = {descriptor.width, descriptor.height, 1}; + imageInfo.mipLevels = 1; + imageInfo.arrayLayers = 1; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; + imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + Check(context.Function("vkCreateImage")(context.device, &imageInfo, nullptr, &proxyImage), "vkCreateImage attachment proxy"); + VkMemoryRequirements requirements{}; + context.Function("vkGetImageMemoryRequirements")(context.device, proxyImage, &requirements); + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &proxyMemory), "vkAllocateMemory attachment proxy"); + Check(context.Function("vkBindImageMemory")(context.device, proxyImage, proxyMemory, 0), "vkBindImageMemory attachment proxy"); + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.image = proxyImage; + viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + viewInfo.format = VK_FORMAT_R8G8B8A8_SRGB; + viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &proxyView), "vkCreateImageView attachment proxy"); + } catch (...) { + if (proxyImage) context.Function("vkDestroyImage")(context.device, proxyImage, nullptr); + if (proxyMemory) context.Function("vkFreeMemory")(context.device, proxyMemory, nullptr); + proxyImage = VK_NULL_HANDLE; + proxyMemory = VK_NULL_HANDLE; + proxyView = VK_NULL_HANDLE; + throw; + } + return proxyView; +} + +namespace { + +void blitWhole(const Context& context, VkCommandBuffer commands, VkImage source, VkImage destination, std::uint32_t width, std::uint32_t height) { + VkImageBlit region{}; + region.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.srcOffsets[1] = {static_cast(width), static_cast(height), 1}; + region.dstSubresource = region.srcSubresource; + region.dstOffsets[1] = region.srcOffsets[1]; + context.Function("vkCmdBlitImage")(commands, source, VK_IMAGE_LAYOUT_GENERAL, destination, VK_IMAGE_LAYOUT_GENERAL, 1, ®ion, VK_FILTER_NEAREST); +} + +VkImageMemoryBarrier proxyBarrier(VkImage image, VkAccessFlags source, VkAccessFlags destination, VkImageLayout from, VkImageLayout to) { + VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + barrier.srcAccessMask = source; + barrier.dstAccessMask = destination; + barrier.oldLayout = from; + barrier.newLayout = to; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + return barrier; +} + +} + +void StorageTexture::RecordAttachmentProxyLoad(VkCommandBuffer commands, VkImageLayout attachmentLayout) const { + Require(proxyView != VK_NULL_HANDLE, "the storage image has no attachment proxy"); + const auto barrier = context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier"); + const VkMemoryBarrier written{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT}; + const auto discard = proxyBarrier(proxyImage, 0, VK_ACCESS_TRANSFER_WRITE_BIT, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL); + barrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &written, 0, nullptr, 1, &discard); + blitWhole(context, commands, image, proxyImage, descriptor.width, descriptor.height); + const auto attach = proxyBarrier(proxyImage, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_IMAGE_LAYOUT_GENERAL, attachmentLayout); + barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, nullptr, 0, nullptr, 1, &attach); +} + +void StorageTexture::RecordAttachmentProxyStore(VkCommandBuffer commands, VkImageLayout attachmentLayout) const { + Require(proxyView != VK_NULL_HANDLE, "the storage image has no attachment proxy"); + const auto barrier = context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier"); + const auto detach = proxyBarrier(proxyImage, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT, attachmentLayout, VK_IMAGE_LAYOUT_GENERAL); + barrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &detach); + blitWhole(context, commands, proxyImage, image, descriptor.width, descriptor.height); + const VkMemoryBarrier stored{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT}; + barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &stored, 0, nullptr, 0, nullptr); +} + VkImageView StorageTexture::View(std::uint32_t mip) { if (mip == defaultMip) return view; const auto found = extraViews.find(mip); @@ -1128,6 +1252,7 @@ bool StorageTexture::Refresh() { bool unchanged = true; bool stamped = true; bool tracked = true; + bool compared = false; bool keysChanged = false; bool cpuWrote = false; bool direct = false; @@ -1143,6 +1268,8 @@ bool StorageTexture::Refresh() { stampedBlocks.assign(generations.size(), 0); cpuBlocks.assign(generations.size(), 0); tracked = GuestMemory::ChangedBlocks(descriptor.baseAddress, static_cast(guestBytes), generations, stampedBlocks, cpuBlocks); + compared = !tracked && compareUntracked(descriptor.baseAddress, static_cast(guestBytes), stampedBlocks, true); + if (compared) cpuBlocks = stampedBlocks; }; const auto keysStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; keys = ProvedKeys(); @@ -1178,8 +1305,16 @@ bool StorageTexture::Refresh() { } // "cpu" names a CPU store in both models; a block without a generation (untracked) says // nothing about who wrote it. + if (compared && !keysChanged) { + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + const auto first = layerBegin(layer) / 65536 - descriptor.baseAddress / 65536; + const auto last = (layerBegin(layer) + layerBytes(layer) - 1) / 65536 - descriptor.baseAddress / 65536; + changed[layer] = std::any_of(stampedBlocks.begin() + first, stampedBlocks.begin() + last + 1, [](auto value) { return value != GuestMemory::BlockUnchanged; }); + } + unchanged = std::none_of(changed.begin(), changed.end(), [](bool value) { return value; }); + } for (std::size_t k = 0; k < cpuBlocks.size(); ++k) { - if (tracked && cpuBlocks[k] != 0 && generations[k] != 0) cpuWrote = true; + if (cpuBlocks[k] != 0 && (compared || (tracked && generations[k] != 0))) cpuWrote = true; } // Unchanged bytes rescue an upload only when nothing else moved: with the keys changed // the same bytes read differently, and a changed unit holding pending results must take @@ -1249,6 +1384,7 @@ bool StorageTexture::Refresh() { // unit's stamps say nothing, so it is stored. const auto droppable = [&](std::uint32_t unit) { if (keysChanged && IsDccClear(keys)) return true; + if (compared) return stampedBlocks.at(unit) == GuestMemory::BlockWritten; if (!tracked || layerGeneration[unit] == 0 || unit >= stampedBlocks.size() || stampedBlocks[unit] != GuestMemory::BlockWritten) return false; const auto begin = layerBegin(unit); const auto bytes = layerBytes(unit); @@ -1324,13 +1460,12 @@ bool StorageTexture::Refresh() { } DccKeys StorageTexture::ProvedKeys() const { - if (descriptor.dccAddress != 0 && uploadedKeys == DccKeys::Uncompressed) { - if (const auto keys = WaitForKeyWriters(descriptor, guestBytes)) { - keyProof = {}; - return *keys; - } - } - return ProvedClearKeys(descriptor, guestBytes, keyProof); + std::optional keys; + if (descriptor.dccAddress != 0 && uploadedKeys == DccKeys::Uncompressed) keys = WaitForKeyWriters(descriptor, guestBytes); + if (keys.has_value()) keyProof = {}; + else keys = ProvedClearKeys(descriptor, guestBytes, keyProof); + if (IsDccClear(filledKeys) && *keys != filledKeys && !anyLayerPending()) filledKeys = DccKeys::Uncompressed; + return *keys; } bool StorageTexture::ServesKeysAt(std::uint64_t dccAddress) const { @@ -1362,11 +1497,60 @@ void traceKeyStore(const char* path, const GuestTextureResource& descriptor, std static const bool trace = std::getenv("APS5_TRACE_DCC_KEYS") != nullptr; if (!trace || descriptor.dccAddress == 0) return; const auto packet = GuestMemory::CurrentPacket(); - std::fprintf(stderr, "[dcc-keys] uncompressed store by %s (%s) for surface 0x%llx+0x%llx: keys 0x%llx+0x%llx (packet 0x%x queue 0x%x)\n", path, flushReason != nullptr ? flushReason : "?", static_cast(descriptor.baseAddress), static_cast(guestBytes), static_cast(descriptor.dccAddress), static_cast(guestBytes / 256), packet.opcode, packet.queue); + std::fprintf(stderr, "[dcc-keys] uncompressed store by %s (%s) for surface 0x%llx+0x%llx: keys 0x%llx+0x%llx (packet 0x%x queue 0x%x)\n", path, flushReason != nullptr ? flushReason : "?", static_cast(descriptor.baseAddress), static_cast(guestBytes), static_cast(descriptor.dccAddress), static_cast(DccKeyCount(descriptor, guestBytes)), packet.opcode, packet.queue); } } +void StorageTexture::captureGuestBytes(const std::vector* layers) { + comparedGuestBytes = {}; + const bool complete = layers == nullptr || std::all_of(layers->begin(), layers->end(), [](bool selected) { return selected; }); + for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { + if (layers != nullptr && !(*layers)[layer]) continue; + const auto offset = static_cast(layerBegin(layer) - descriptor.baseAddress); + GuestMemory::ReadCommitted(layerBegin(layer), std::span(original).subspan(offset, static_cast(layerBytes(layer)))); + } + originalValid = originalValid || complete; +} + +bool StorageTexture::compareUntracked(std::uint64_t address, std::size_t bytes, std::span changed, bool memoize) const { + if (!originalValid) return false; + struct Exempt { + const StorageTexture* previous; + ~Exempt() { refreshing = previous; } + } exempt{refreshing}; + refreshing = this; + Require(address >= descriptor.baseAddress && bytes <= guestBytes && address - descriptor.baseAddress <= guestBytes - bytes, "texture comparison exceeds its snapshot"); + Require(original.size() == guestBytes, "texture comparison has no complete snapshot"); + GuestMemory::FlushGpuWrites(address, bytes); + if (!originalValid) return false; + constexpr std::uint64_t blockBytes = 65536; + const auto end = address + bytes; + const auto first = address / blockBytes; + Require(changed.size() == (end - 1) / blockBytes - first + 1, "texture comparison block count differs"); + const auto* recorder = Recorder::Active(); + const std::array stamp{GuestMemory::CollectEpoch(), GuestMemory::TrackerGeneration(), GuestMemory::ForgetSerial(), recorder != nullptr ? recorder->NewestWriteNote(address, bytes) : 0}; + const bool cacheable = memoize && stamp[0] != 0 && (stamp[2] & 1u) == 0 && address == descriptor.baseAddress && bytes == guestBytes; + if (cacheable && comparedGuestBytes == stamp) { + std::fill(changed.begin(), changed.end(), GuestMemory::BlockUnchanged); + return true; + } + const auto saved = std::span(original).subspan(static_cast(address - descriptor.baseAddress), bytes); + if (GuestMemory::EqualsCommittedUnsynced(address, saved)) { + std::fill(changed.begin(), changed.end(), GuestMemory::BlockUnchanged); + if (cacheable) comparedGuestBytes = stamp; + return true; + } + comparedGuestBytes = {}; + for (std::size_t index = 0; index < changed.size(); ++index) { + const auto begin = std::max(address, (first + index) * blockBytes); + const auto stop = std::min(end, (first + index + 1) * blockBytes); + const auto saved = std::span(original).subspan(static_cast(begin - descriptor.baseAddress), static_cast(stop - begin)); + changed[index] = GuestMemory::EqualsCommittedUnsynced(begin, saved) ? GuestMemory::BlockUnchanged : GuestMemory::BlockWritten; + } + return true; +} + void StorageTexture::upload(const std::vector* layers) { CaptureTrace::Log("upload image=%llx bytes=%llu generation=%llu reason=%s partial=%d", static_cast(descriptor.baseAddress), static_cast(guestBytes), static_cast(generation), uploadReason, layers != nullptr); const bool profile = LookupOutcomes::Profiled(); @@ -1392,7 +1576,7 @@ void StorageTexture::upload(const std::vector* layers) { // The clear is recorded into the open batch like a direct upload (the image kept by it): a // batch of its own submitted the recorder's work first and waited for all of it, 25-40 ms // under the GPU mutex at the movie stage. APS5_NO_RECORDED_CLEAR=1 waits as before. - originalValid = false; + captureGuestBytes(nullptr); forgetBorrowed(0, trackedLayers); stampLayers(false); static const bool recordClear = std::getenv("APS5_NO_RECORDED_CLEAR") == nullptr; @@ -1440,9 +1624,8 @@ void StorageTexture::upload(const std::vector* layers) { return; } if (const auto* import = uploadedKeys == DccKeys::Uncompressed ? HostImportFor(context, descriptor.baseAddress, static_cast(guestBytes)) : nullptr) { - // The surface lives in host-imported memory: the detiler reads it in place, no guest bytes - // are copied, and write tracking alone validates the image (a change re-runs this). - originalValid = false; + if (GuestMemory::Watched(descriptor.baseAddress, static_cast(guestBytes))) originalValid = false; + else captureGuestBytes(layers); stampLayers(true); // A whole-surface upload of a block-unit image goes through the windows too when a unit // shadow holds part of the surface (the detile then reads the slabs; a new image has no @@ -1489,7 +1672,7 @@ void StorageTexture::upload(const std::vector* layers) { recorder->FlushStoresOverlapping(descriptor.baseAddress, static_cast(guestBytes)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload); - recorder->Keep(linear); + recorder->Keep(linear, linear->Size()); // The image itself must outlive the recorded copy: the cache may evict it right after. if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); // The detile reads the tiled bytes from the import when the batch runs. @@ -1818,7 +2001,7 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< for (const auto& [begin, end] : reads) recorder->FlushStoresOverlapping(begin, static_cast(end - begin)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload); - recorder->Keep(linear); + recorder->Keep(linear, linear->Size()); for (const auto& source : sources) { if (source.slab != nullptr) recorder->Keep(source.slab); } @@ -1984,8 +2167,8 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp recorder->FlushStoresOverlapping(flushBegin, static_cast(flushEnd - flushBegin)); commands = recorder->Commands(&covered); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); - recorder->Keep(linear); - recorder->Keep(tiledScratch); + recorder->Keep(linear, linear->Size()); + recorder->Keep(tiledScratch, tiledScratch->Size()); for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab); for (const auto& slab : padding.slabs) recorder->Keep(slab); if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); @@ -2105,7 +2288,7 @@ bool StorageTexture::overlaps(std::uint64_t address, std::size_t bytes) const { void StorageTexture::MarkDirty() { if (descriptor.dccAddress != 0 && IsDccClear(uploadedKeys) && !IsDccClear(filledKeys)) { traceKeyStore("first write", descriptor, guestBytes); - MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + MarkDccUncompressed(context, descriptor.dccAddress, guestBytes, DccKeyCount(descriptor, guestBytes)); uploadedKeys = DccKeys::Uncompressed; } markLayersPending(0, trackedLayers); @@ -3297,11 +3480,12 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { } std::vector changedBlocks(spanBlocks); const bool tracked = GuestMemory::ChangedBlocks(firstStored, static_cast(lastStored - firstStored), generations, changedBlocks); + const bool compared = !tracked && compareUntracked(firstStored, static_cast(lastStored - firstStored), changedBlocks); for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { if (!layers[layer]) continue; const auto begin = layerBegin(layer); const auto end = begin + layerBytes(layer); - if (!tracked || layerGeneration[layer] == 0) { + if (!compared && (!tracked || layerGeneration[layer] == 0)) { keep.emplace_back(begin, end); continue; } @@ -3310,7 +3494,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { const auto to = std::min(at + block, end); if (from >= to) continue; const bool edge = from != at || to != at + block; - if (changedBlocks[static_cast((at - spanBegin) / block)] == GuestMemory::BlockWritten && (!edge || GuestMemory::StoredOver(from, static_cast(to - from), layerGeneration[layer]))) { + if (changedBlocks[static_cast((at - spanBegin) / block)] == GuestMemory::BlockWritten && (compared || !edge || GuestMemory::StoredOver(from, static_cast(to - from), layerGeneration[layer]))) { skippedAny = true; skippedLayer[layer] = true; ++skipped; @@ -3375,7 +3559,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { if (profile) Profile().storageGpu += timer.lap(); originalValid = false; traceKeyStore("block write-back", descriptor, guestBytes); - if (!IsDccClear(filledKeys)) MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + if (!IsDccClear(filledKeys)) MarkDccUncompressed(context, descriptor.dccAddress, guestBytes, DccKeyCount(descriptor, guestBytes)); uploadedKeys = DccKeys::Uncompressed; for (const auto& [from, to] : keep) GuestMemory::MarkWritten(from, static_cast(to - from)); settle(true); @@ -3406,8 +3590,8 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { recorder->FlushStoresOverlapping(firstStored, static_cast(lastStored - firstStored)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); - recorder->Keep(linear); - recorder->Keep(tiledScratch); + recorder->Keep(linear, linear->Size()); + recorder->Keep(tiledScratch, tiledScratch->Size()); for (const auto& slab : padding.slabs) recorder->Keep(slab); // The image itself must outlive the recorded retile: the cache may evict it right after. if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); @@ -3483,7 +3667,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { // metadata is host-imported (no CPU wait for the title's key-writing kernels), else a CPU // store (APS5_CPU_DCC_KEYS=1 keeps the CPU store; see DccMetadata.hpp). traceKeyStore("layer write-back", descriptor, guestBytes); - if (!IsDccClear(filledKeys)) MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + if (!IsDccClear(filledKeys)) MarkDccUncompressed(context, descriptor.dccAddress, guestBytes, DccKeyCount(descriptor, guestBytes)); uploadedKeys = DccKeys::Uncompressed; for (const auto& [from, to] : keep) GuestMemory::MarkWritten(from, static_cast(to - from)); // The walk covers this surface's pages only: an adjacent image's later CPU write is stamped @@ -3593,7 +3777,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { // The texels now hold the whole image, so later reads must see them rather than a fast clear // (the keys may be host-imported although the texels were not: then a recorded fill, else a CPU store). traceKeyStore("cpu write-back", descriptor, guestBytes); - if (!IsDccClear(filledKeys)) MarkDccUncompressed(context, descriptor.dccAddress, guestBytes); + if (!IsDccClear(filledKeys)) MarkDccUncompressed(context, descriptor.dccAddress, guestBytes, DccKeyCount(descriptor, guestBytes)); uploadedKeys = DccKeys::Uncompressed; // The store above is the only write to these pages, so `original` is current at a fresh // generation, unless blocks were kept for the CPU: then the image is stale there. @@ -3638,6 +3822,9 @@ void StorageTexture::release() noexcept { uintViews.clear(); for (const auto& [format, attachment] : attachmentViews) context.Function("vkDestroyImageView")(context.device, attachment, nullptr); attachmentViews.clear(); + if (proxyView) context.Function("vkDestroyImageView")(context.device, proxyView, nullptr); + if (proxyImage) context.Function("vkDestroyImage")(context.device, proxyImage, nullptr); + if (proxyMemory) context.Function("vkFreeMemory")(context.device, proxyMemory, nullptr); if (view) context.Function("vkDestroyImageView")(context.device, view, nullptr); if (image) context.Function("vkDestroyImage")(context.device, image, nullptr); if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp index 04fda0eda..d5978cf76 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp @@ -96,7 +96,7 @@ constexpr auto MakeFormatLookupTable() { constexpr auto kFormatLookupTable = MakeFormatLookupTable(); std::uint32_t remapGuestFormat(std::uint32_t guestFormat) { - return guestFormat == 30 || guestFormat == 34 ? 20 : guestFormat; + return guestFormat == 30 || guestFormat == 34 || guestFormat == 43 ? 20 : guestFormat; } const FormatEntry& findFormatEntry(std::uint32_t guestFormat) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp index 0c87d3653..2a2beb77f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp @@ -298,7 +298,10 @@ ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, s Require(width != 0 && height != 0 && depth != 0, "cannot compute layout for a zero-sized 3D texture"); Require(mipCount != 0 && mipCount <= 16u, "3D texture mip count is out of range"); const auto bytesPerElement = BytesPerElement(format); - Require(BlockWidth(format) == 1u && BlockHeight(format) == 1u, "block-compressed 3D textures are not implemented"); + const auto texelWidth = BlockWidth(format); + const auto texelHeight = BlockHeight(format); + const auto elementsWidth0 = (width + texelWidth - 1u) / texelWidth; + const auto elementsHeight0 = (height + texelHeight - 1u) / texelHeight; ThickLayout result{}; result.depth = depth; result.mips.resize(mipCount); @@ -315,7 +318,7 @@ ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, s const auto effectiveLog2 = log2BlockBytes - (log2BlockBytes - 8u) / 3u; maxTailLevels = effectiveLog2 <= 11u ? 1u + (1u << (effectiveLog2 - 9u)) : effectiveLog2 - 4u; for (std::uint32_t level = 0; mipCount > 1 && level < mipCount; ++level) { - if (ShiftCeil(width, level) <= tailWidth && ShiftCeil(height, level) <= tailHeight && mipCount - level <= maxTailLevels) { + if (ShiftCeil(elementsWidth0, level) <= tailWidth && ShiftCeil(elementsHeight0, level) <= tailHeight && mipCount - level <= maxTailLevels) { firstTailLevel = level; break; } @@ -326,14 +329,14 @@ ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, s std::uint64_t linearOffset = 0; for (auto level = mipCount; level-- > 0;) { auto& mip = result.mips[level]; - mip.width = std::max(width >> level, 1u); - mip.height = std::max(height >> level, 1u); + mip.width = TexelLevelDimension(width, level, texelWidth); + mip.height = TexelLevelDimension(height, level, texelHeight); mip.tail = level >= firstTailLevel; mip.tailX = 0; mip.tailY = 0; const bool tiled = tileMode != TextureTileMode::kLinear; - const auto levelWidth = tiled ? std::max((width + (1u << level) - 1u) >> level, 1u) : mip.width; - const auto levelHeight = tiled ? std::max((height + (1u << level) - 1u) >> level, 1u) : mip.height; + const auto levelWidth = tiled ? std::max(ShiftCeil(elementsWidth0, level), 1u) : mip.width; + const auto levelHeight = tiled ? std::max(ShiftCeil(elementsHeight0, level), 1u) : mip.height; const auto paddedWidth = AlignUp(levelWidth, block[0]); mip.pitchBytes = paddedWidth * bytesPerElement; if (mip.tail) { diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaDevice.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaDevice.cpp index 484d60a7a..8d73dada8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaDevice.cpp @@ -6,7 +6,9 @@ #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include #include +#include #include +#include #include namespace AgcDriver::Graphics { @@ -42,7 +44,27 @@ public: Require(count != 0, "no Vulkan device"); std::vector devices(count); Check(enumerate(instance, &count, devices.data()), "vkEnumeratePhysicalDevices"); - context.physical = devices.front(); + const auto rankDeviceType = [](VkPhysicalDeviceType type) { + switch (type) { + case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return 3; + case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return 2; + case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return 1; + default: return 0; + } + }; + const auto physicalProperties = function("vkGetPhysicalDeviceProperties"); + int selectedRank = -1; + for (const auto physical : devices) { + VkPhysicalDeviceProperties candidate{}; + physicalProperties(physical, &candidate); + if (candidate.apiVersion < VK_API_VERSION_1_1) continue; + const int rank = rankDeviceType(candidate.deviceType); + if (rank <= selectedRank) continue; + context.physical = physical; + selectedRank = rank; + cpu = candidate.deviceType == VK_PHYSICAL_DEVICE_TYPE_CPU; + } + Require(context.physical != VK_NULL_HANDLE, "no Vulkan 1.1 device"); const auto extensions = function("vkEnumerateDeviceExtensionProperties"); Check(extensions(context.physical, nullptr, &count, nullptr), "vkEnumerateDeviceExtensionProperties"); std::vector available(count); @@ -90,6 +112,7 @@ public: ~Device() { release(); } const Context& GetContext() const { return context; } + bool RunsOnCpu() const { return cpu; } private: template @@ -111,6 +134,7 @@ private: PFN_vkGetInstanceProcAddr instanceProc = nullptr; VkInstance instance = VK_NULL_HANDLE; Context context{}; + bool cpu = false; }; } @@ -125,13 +149,24 @@ int main(int argc, char** argv) { Require(static_cast(file), "cannot save BDA test SPIR-V"); return 0; } - Device device; - Buffer buffer(device.GetContext(), 256, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + std::unique_ptr device; + try { + device = std::make_unique(); + } catch (const std::exception& error) { + if (std::getenv("ANYPS5_REQUIRE_VULKAN") != nullptr) throw; + std::cout << "skipped, no usable Vulkan device: " << error.what() << '\n'; + return 77; + } + Buffer buffer(device->GetContext(), 256, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); Require(buffer.DeviceAddress() != 0 && buffer.Bytes().size() == 256, "invalid real BDA buffer"); buffer.Bytes()[255] = std::byte{0x5a}; Require(buffer.Bytes()[255] == std::byte{0x5a}, "real BDA buffer mapping failed"); - RunBdaExecutionTests(device.GetContext()); - RunColorTransferTests(device.GetContext()); + if (device->RunsOnCpu()) { + std::cout << "CPU Vulkan device: BDA execution not tested\n"; + } else { + RunBdaExecutionTests(device->GetContext()); + } + RunColorTransferTests(device->GetContext()); std::cout << "Vulkan BDA allocation and execution tests passed\n"; return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp index 362b4944e..f256818ac 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp @@ -11,10 +11,13 @@ #endif #include #include +#include #include #include +#include #include #include +#include #include namespace { @@ -32,10 +35,113 @@ void reject(TAction action, const char* reason) { throw std::runtime_error(std::string("expected BDA rejection: ") + reason); } +void importCrossingTests(const Context& context, const BdaTestAccess& access) { + constexpr std::size_t half = 65536; + void* block = ::operator new(2 * half, std::align_val_t{half}); + auto* guest = static_cast(block); + std::memset(guest, 0x11, half); + std::memset(guest + half, 0x22, half); + const auto first = reinterpret_cast(block); + const auto second = first + half; + auto importing = context; + importing.hostImportAlignment = half; + const auto registry = [&](bool add) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + for (auto* range : {guest, guest + half}) { + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, range, half, true, false); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, range); + } + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + registry(true); + { + GuestBufferMemory leased(importing); + leased.AcquireRegistered(); + Require(HostImportCovers(importing, first, half) && HostImportCovers(importing, second, half), "the registered ranges were not imported"); + const auto secondImport = *HostImportFor(importing, second, half); + leased.AddReadable(second - 16, 32); + leased.AddReadable(second + 64, 16); + leased.Upload(true); + std::uint32_t adjustment = 0; + const auto crossing = leased.Descriptor(second - 16, 32, adjustment); + const auto crossingBytes = access.bytes(crossing.buffer); + Require(crossing.offset + crossing.range <= crossingBytes.size(), "a view crossing into the next registered range is bound past its host import"); + Require(crossingBytes[crossing.offset + adjustment + 15] == std::byte{0x11} && crossingBytes[crossing.offset + adjustment + 16] == std::byte{0x22}, "a view crossing into the next registered range misses its bytes"); + const auto inside = leased.Descriptor(second + 64, 16, adjustment); + const auto insideBytes = access.bytes(inside.buffer); + Require(inside.offset + inside.range <= insideBytes.size() && (inside.buffer == secondImport.buffer ? inside.offset + adjustment == 64 : insideBytes[inside.offset + adjustment] == std::byte{0x22}), "a view in the second registered range is bound past its buffer or misses its bytes"); + for (const auto& range : leased.AddressRanges()) { + if (range.end <= first || range.begin >= second + half) continue; + const auto mapped = access.addressBytes(range.deviceAddress); + Require(mapped.size() >= range.end - range.begin, "a BDA range runs past the end of its host import"); + if (range.begin <= second && second < range.end && range.deviceAddress != secondImport.address) Require(mapped[second - range.begin] == std::byte{0x22}, "the BDA range over the second registered range misses its bytes"); + } + leased.WriteBack(); + } + registry(false); + Require(HostImportFor(importing, first, half) == nullptr && !HostImportCovers(importing, second, half), "the host imports outlived their ranges"); + ::operator delete(block, std::align_val_t{half}); +} + ShaderRecompiler::DescriptorBinding binding(Role role, std::uint32_t slot) { return {ShaderRecompiler::DescriptorKind::StorageBuffer, role, 0, slot, 1, {}, false}; } +void importedHeapMirrorTests(const Context& context, const BdaTestAccess& access) { + if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) { + std::cout << "guest arena unavailable or not write-watched: heap mirrors of imported ranges not tested\n"; + return; + } + constexpr std::size_t bytes = 1u << 20u; + void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, bytes); +#ifdef _WIN32 + GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes); +#endif + std::memset(block, 0x11, bytes); + const auto address = reinterpret_cast(block); + auto importing = context; + importing.hostImportAlignment = bytes; + const auto registry = [&](bool add) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, bytes, true, false); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + const auto build = [&](const Context& with) { + GuestAllocations::GuestAllocationsEnd_nid_postfix(GuestAllocations::GuestAllocationsBegin_nid_postfix()); + GuestBufferMemory leased(with); + leased.AcquireRegistered(); + leased.Upload(true); + const auto ranges = leased.AddressRanges(); + const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == address && range.end == address + bytes; }); + const auto device = found != ranges.end() ? found->deviceAddress : 0; + leased.WriteBack(); + return device; + }; + const auto before = MirrorCounters(); + registry(true); + Require(build(context) != 0, "the heap range is missing from the BDA table of a build without host imports"); + const auto mirrored = MirrorCounters(); + Require(mirrored.heapMirrors == before.heapMirrors + 1 && mirrored.heapBytes == before.heapBytes + bytes, "a range built without host imports was not heap mirrored"); + Require(build(importing) != 0 && HostImportCovers(importing, address, bytes), "the heap mirrored range was not imported once imports were available"); + const auto imported = MirrorCounters(); + Require(imported.heapMirrors == before.heapMirrors && imported.heapBytes == before.heapBytes, "a heap mirror outlived the host import that serves its range"); + Require(build(importing) != 0 && MirrorCounters().heapMirrors == before.heapMirrors && MirrorCounters().rebuilds == imported.rebuilds, "an imported range was heap mirrored again"); + static_cast(block)[bytes / 2] = 0x22; + const auto device = build(context); + const auto again = MirrorCounters(); + Require(device != 0 && again.heapMirrors == before.heapMirrors + 1 && again.heapBytes == before.heapBytes + bytes && again.rebuilds == imported.rebuilds + 1, "a range no import serves was not heap mirrored again"); + Require(access.addressBytes(device)[bytes / 2] == std::byte{0x22} && access.addressBytes(device)[bytes / 2 + 1] == std::byte{0x11}, "a heap mirror made after its import missed the guest bytes"); + registry(false); + Require(HostImportFor(importing, address, bytes) == nullptr && !HostImportCovers(importing, address, bytes), "the import outlived its range"); + Require(build(context) == 0, "the unregistered heap range is still in the BDA table"); + Require(MirrorCounters().heapMirrors == before.heapMirrors && MirrorCounters().heapBytes == before.heapBytes, "the heap mirror outlived its range"); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(block, bytes); +#endif + GuestArena::GuestArenaRelease_nid_postfix(block, bytes); +} + void heapMirrorTests(const Context& context, const BdaTestAccess& access) { if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) { std::cout << "guest arena unavailable or not write-watched: heap mirrors not tested\n"; @@ -61,9 +167,21 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { if (written != 0) leased.AddWritable(written, 32); leased.Upload(true); const auto ranges = leased.AddressRanges(); - const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == address && range.end == address + bytes; }); - Require(found != ranges.end() && found->permissions == ShaderRecompiler::BdaAbi::Read, "the heap range is missing from the BDA table"); + const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == address; }); + Require(found != ranges.end() && found->deviceAddress != 0u, "the heap range is missing from the BDA table"); const auto device = found->deviceAddress; + auto cursor = address; + std::uint64_t writableBytes = 0; + for (auto part = found; cursor < address + bytes && part != ranges.end(); ++part) { + Require(part->begin == cursor && part->end > cursor && part->end <= address + bytes, "heap BDA ranges have gaps or overlap"); + Require(part->deviceAddress == device + cursor - address, "heap BDA ranges are not contiguous on the device"); + const bool writable = written != 0u && part->begin >= written && part->end <= written + 32u; + const auto permissions = ShaderRecompiler::BdaAbi::Read | (writable ? ShaderRecompiler::BdaAbi::Write : 0u); + Require(part->permissions == permissions, "heap BDA range has incorrect permissions"); + if (writable) writableBytes += part->end - part->begin; + cursor = part->end; + } + Require(cursor == address + bytes && writableBytes == (written != 0u ? 32u : 0u), "heap BDA coverage or writable extent is incorrect"); if (gpu) gpu(leased); leased.WriteBack(); return device; @@ -90,6 +208,10 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { guest[65536 + 7] = 0x44; Require(build(0, {}) == first && MirrorCounters().blocksCopied == one.blocksCopied + 2, "two written heap blocks were not read again"); Require(access.addressBytes(first)[5] == std::byte{0x33} && access.addressBytes(first)[65536 + 7] == std::byte{0x44}, "the heap mirror missed the CPU writes"); + const auto quiet = MirrorCounters(); + Require(build(0, {}) == first && MirrorCounters().heapChecks == quiet.heapChecks && MirrorCounters().sweeps == quiet.sweeps && MirrorCounters().blocksCopied == quiet.blocksCopied, "an unchanged heap range was compared block by block or the mirrors were swept again"); + guest[65536 + 9] = 0x66; + Require(build(0, {}) == first && MirrorCounters().heapChecks > quiet.heapChecks && MirrorCounters().blocksCopied == quiet.blocksCopied + 1 && access.addressBytes(first)[65536 + 9] == std::byte{0x66}, "a CPU write after an unchanged build was missed"); registry(false, false); sweep(); const auto swept = MirrorCounters(); @@ -104,6 +226,13 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { }); Require(guest[16] == 0x77 && guest[16 + 8] == 0x55, "a writable heap mirror's write-back lost the GPU's store or rolled back the CPU's"); Require(build(0, {}) == device && access.addressBytes(device)[16] == std::byte{0x77} && access.addressBytes(device)[16 + 8] == std::byte{0x55}, "the writable heap mirror missed the stores"); + Require(build(address + 40, [&](GuestBufferMemory& leased) { + std::uint32_t adjustment = 0; + const auto view = leased.Descriptor(address + 40, 8, adjustment); + access.bytes(view.buffer)[view.offset + adjustment] = std::byte{0x78}; + }) == device && guest[40] == 0x78, "a writable heap mirror's second write-back lost the GPU's store"); + guest[65536 + 1] = 0x79; + Require(build(0, {}) == device && access.addressBytes(device)[40] == std::byte{0x78} && access.addressBytes(device)[65536 + 1] == std::byte{0x79}, "the writable heap mirror missed a store after a write-back"); registry(false, true); sweep(); Require(MirrorCounters().heapMirrors == before.heapMirrors, "the writable heap mirror outlived its range"); @@ -217,6 +346,71 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { GuestArena::GuestArenaRelease_nid_postfix(raw, size); } #endif + { + constexpr std::size_t large = 36 * 65536; + constexpr std::size_t small = 17 * 65536; + constexpr std::size_t total = 2 * large + small; + void* raw = GuestArena::GuestArenaAllocate_nid_postfix(total, 65536); +#ifdef _WIN32 + GuestArena::GuestArenaCommit_nid_postfix(raw, total, PAGE_READWRITE, total); +#endif + auto* const heap = static_cast(raw); + const std::array heaps{heap, heap + large, heap + 2 * large}; + const std::array sizes{large, large, small}; + for (std::size_t index = 0; index < heaps.size(); ++index) std::memset(heaps[index], 0x21 + static_cast(index), sizes[index]); + const auto change = [&](std::initializer_list added, std::initializer_list removed) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + for (const auto index : removed) GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, heaps[index]); + for (const auto index : added) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, heaps[index], sizes[index], true, false); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + const auto mirrored = [&](std::size_t index) { + GuestBufferMemory leased(context); + leased.AcquireRegistered(); + leased.Upload(true); + const auto ranges = leased.AddressRanges(); + const auto begin = reinterpret_cast(heaps[index]); + const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& entry) { return entry.begin == begin && entry.end == begin + sizes[index]; }); + Require(found != ranges.end(), "a mirrored heap range is missing from the BDA table"); + const auto last = access.addressBytes(found->deviceAddress)[sizes[index] - 1]; + leased.WriteBack(); + return last; + }; + sweep(); + const auto before = MirrorCounters(); + change({0}, {}); + Require(mirrored(0) == std::byte{0x21}, "the first heap range was not mirrored"); + const auto one = MirrorCounters(); + access.limitMemory(large - 1); + change({1}, {0}); + Require(mirrored(1) == std::byte{0x22}, "a heap mirror past APS5_HEAP_MIRROR_MIB was not made in the memory of the expired mirror"); + const auto swept = MirrorCounters(); + Require(swept.heapMirrors == one.heapMirrors && swept.heapBytes == one.heapBytes, "the expired heap mirror was not swept before the allocation"); + access.limitMemory(small - 1); + change({2}, {}); + char expected[64]; + std::snprintf(expected, sizeof(expected), "heap mirror of 0x%llx+0x%llx: ", static_cast(reinterpret_cast(heaps[2])), static_cast(small)); + const auto attempts = access.allocationAttempts(); + bool refused = false; + try { + mirrored(2); + } catch (const std::runtime_error& error) { + const std::string what = error.what(); + refused = what.find(expected) != std::string::npos && what.find("Vulkan result -2") != std::string::npos; + } + Require(refused, "a heap mirror the memory cannot hold did not fail its build with the Vulkan result"); + Require(access.allocationAttempts() == attempts + 1, "a refused heap mirror was allocated again"); + Require(MirrorCounters().heapMirrors == swept.heapMirrors && MirrorCounters().heapBytes == swept.heapBytes, "a refused heap mirror was registered"); + access.limitMemory(std::nullopt); + Require(mirrored(2) == std::byte{0x23} && MirrorCounters().heapMirrors == swept.heapMirrors + 1, "a heap mirror was not made once the memory was free again"); + change({}, {1, 2}); + sweep(); + Require(MirrorCounters().heapMirrors == before.heapMirrors && MirrorCounters().heapBytes == before.heapBytes, "the heap mirrors outlived their ranges"); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(raw, total); +#endif + GuestArena::GuestArenaRelease_nid_postfix(raw, total); + } #ifdef _WIN32 GuestArena::GuestArenaReset_nid_postfix(block, bytes); #endif @@ -240,7 +434,7 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { Require(first.buffer == alias.buffer && alias.offset + adjustment == 16 && alias.range == 16 + adjustment, "aliased guest buffers have different owners"); const auto ranges = memory.AddressRanges(); Require(ranges.size() == 1 && ranges[0].begin == address && ranges[0].end == address + sizeof(guest), "incorrect BDA range bounds"); - Require(ranges[0].deviceAddress != 0 && ranges[0].permissions == ShaderRecompiler::BdaAbi::Read, "incorrect BDA address or permissions"); + Require(ranges[0].deviceAddress != 0 && ranges[0].permissions == (ShaderRecompiler::BdaAbi::Read | ShaderRecompiler::BdaAbi::Write), "incorrect BDA address or permissions"); std::uint32_t changed = 321; std::memcpy(access.bytes(alias.buffer).data() + alias.offset, &changed, sizeof(changed)); memory.WriteBack(); @@ -325,6 +519,20 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { const auto view = unaligned.Descriptor(address + 4, 4, adjustment); Require(adjustment == 4 && view.offset == 0 && view.range == 8, "a view off the offset alignment binds from below it"); reject([&] { unaligned.Descriptor(address + sizeof(guest), 4, adjustment); }, "exceeds its GPU owner"); + const auto odd = unaligned.Descriptor(address + 2, 6, adjustment); + Require(adjustment == 2 && odd.offset == 0 && odd.range == 8, "a view off a DWORD boundary does not bind the DWORDs around it"); + const auto late = unaligned.Descriptor(address + 0x13, 8, adjustment); + Require(adjustment == 3 && late.offset == 16 && late.range == 12, "a view off a DWORD boundary does not bind from the offset alignment below it"); + GuestBufferMemory lone(aligned); + lone.AddReadable(address + 6, 5); + lone.Upload(true); + const auto copied = lone.Descriptor(address + 6, 5, adjustment); + Require(adjustment == 2 && copied.offset == 0 && copied.range == 8, "a lone view off a DWORD boundary is not copied from the DWORD below it"); + Require(std::memcmp(access.bytes(copied.buffer).data(), reinterpret_cast(address + 4), 8) == 0, "a lone view off a DWORD boundary copied other bytes"); + GuestBufferMemory tail(aligned); + tail.AddWritable(address, 62); + tail.Upload(true); + reject([&] { tail.Descriptor(address + 58, 4, adjustment); }, "exceeds its GPU owner"); } { // The cached address space: a second build in an unchanged registry takes the first one's @@ -359,7 +567,9 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { const auto dropped = AddressSpaceCounters(); if (dropped.enabled) Require(dropped.waiterDrops == after.waiterDrops + 1 && LeaseCounters().cacheDrops == dropped.waiterDrops, "the free did not drop the cached address space"); } + importCrossingTests(context, access); heapMirrorTests(context, access); + importedHeapMirrorTests(context, access); Require(AddressCopyOverflow({{0x1000, 0x3000, 0x2000, "uncommitted pages"}}, 0x2000).empty(), "copies within the limit were refused"); const auto copies = AddressCopyOverflow({{0x1000, 0x2000, 0x1000, "not mirrored"}, {0x10000, 0x30000, 0x18000, "uncommitted pages"}}, 0x2000); Require(!copies.empty() && copies.find("0x10000+0x20000 (0.1 MiB committed, uncommitted pages)") < copies.find("0x1000+0x1000"), "the copy limit does not name the largest copy first"); diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp index fea8ed45e..0d90f1c58 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp @@ -3,16 +3,20 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include +#include #include struct BdaTestAccess { std::function(VkBuffer)> bytes; std::function descriptor; std::function(VkDeviceAddress)> addressBytes; + std::function)> limitMemory; + std::function allocationAttempts; }; void RunBdaResourceTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access); void RunGuestAllocationTests(); +void RunGuestLeaseWaitTests(); void RunUnmappedGapTests(); void RunColorTargetLayoutTests(); void RunLiveStackAccessTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp new file mode 100644 index 000000000..12edf8f22 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp @@ -0,0 +1,264 @@ +#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include +#include +#include +#include +#include +#include +#include + +using namespace AgcDriver::Graphics; + +namespace { + +int failures = 0; + +void Expect(bool condition, const std::string& what) { + if (condition) return; + std::fprintf(stderr, "FAIL: %s\n", what.c_str()); + ++failures; +} + +constexpr std::size_t MiB = std::size_t{1} << 20u; +constexpr VkMemoryPropertyFlags HostVisible = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + +struct MockDevice { + std::uint64_t next = 1; + std::map sizes; + std::map usages; + std::map> hostMemory; + std::uint64_t allocations = 0; + std::uint64_t frees = 0; + std::uint64_t destroyedBuffers = 0; + VkDeviceSize liveBytes = 0; + std::map memoryBytes; +}; + +MockDevice mock; + +VKAPI_ATTR VkResult VKAPI_CALL mockCreateBuffer(VkDevice, const VkBufferCreateInfo* info, const VkAllocationCallbacks*, VkBuffer* buffer) { + *buffer = reinterpret_cast(static_cast(mock.next++)); + mock.sizes[*buffer] = info->size; + mock.usages[*buffer] = info->usage; + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL mockGetBufferMemoryRequirements(VkDevice, VkBuffer buffer, VkMemoryRequirements* requirements) { + *requirements = {mock.sizes.at(buffer), 256, 3}; +} + +VKAPI_ATTR VkResult VKAPI_CALL mockAllocateMemory(VkDevice, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks*, VkDeviceMemory* memory) { + *memory = reinterpret_cast(static_cast(mock.next++)); + if (info->memoryTypeIndex == 1) mock.hostMemory[*memory].resize(info->allocationSize); + mock.memoryBytes[*memory] = info->allocationSize; + mock.liveBytes += info->allocationSize; + ++mock.allocations; + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL mockBindBufferMemory(VkDevice, VkBuffer, VkDeviceMemory, VkDeviceSize) { + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL mockMapMemory(VkDevice, VkDeviceMemory memory, VkDeviceSize, VkDeviceSize, VkMemoryMapFlags, void** data) { + *data = mock.hostMemory.at(memory).data(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL mockUnmapMemory(VkDevice, VkDeviceMemory) {} + +VKAPI_ATTR void VKAPI_CALL mockDestroyBuffer(VkDevice, VkBuffer, const VkAllocationCallbacks*) { + ++mock.destroyedBuffers; +} + +VKAPI_ATTR void VKAPI_CALL mockFreeMemory(VkDevice, VkDeviceMemory memory, const VkAllocationCallbacks*) { + mock.liveBytes -= mock.memoryBytes.at(memory); + mock.memoryBytes.erase(memory); + mock.hostMemory.erase(memory); + ++mock.frees; +} + +VKAPI_ATTR VkDeviceAddress VKAPI_CALL mockGetBufferDeviceAddress(VkDevice, const VkBufferDeviceAddressInfo* info) { + return 0x100000000000ULL + reinterpret_cast(info->buffer) * 0x10000; +} + +PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) { + static const std::map table{ + {"vkCreateBuffer", reinterpret_cast(mockCreateBuffer)}, + {"vkGetBufferMemoryRequirements", reinterpret_cast(mockGetBufferMemoryRequirements)}, + {"vkAllocateMemory", reinterpret_cast(mockAllocateMemory)}, + {"vkBindBufferMemory", reinterpret_cast(mockBindBufferMemory)}, + {"vkMapMemory", reinterpret_cast(mockMapMemory)}, + {"vkUnmapMemory", reinterpret_cast(mockUnmapMemory)}, + {"vkDestroyBuffer", reinterpret_cast(mockDestroyBuffer)}, + {"vkFreeMemory", reinterpret_cast(mockFreeMemory)}, + {"vkGetBufferDeviceAddressKHR", reinterpret_cast(mockGetBufferDeviceAddress)}, + }; + const auto it = table.find(name); + return it == table.end() ? nullptr : it->second; +} + +Context mockContext() { + Context context{}; + context.deviceProc = mockProc; + context.memory.memoryTypeCount = 2; + context.memory.memoryTypes[0].propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + context.memory.memoryTypes[1].propertyFlags = HostVisible | VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + context.limits.maxMemoryAllocationCount = 1u << 20u; + context.bufferDeviceAddress = true; + return context; +} + +void SizeClassesAndDirections() { + mock = MockDevice{}; + auto context = mockContext(); + VkBuffer first = VK_NULL_HANDLE; + { + DeviceBuffer upload(context, 10 * MiB + 4096, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + first = upload.Handle(); + Expect(mock.sizes.at(first) == 11 * MiB, "a 10 MiB + 4 KiB device buffer was created with " + std::to_string(mock.sizes.at(first)) + " bytes, not its 11 MiB class"); + constexpr VkBufferUsageFlags all = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + Expect((mock.usages.at(first) & all) == all, "a device buffer was created without storage and both transfer directions"); + Expect(upload.Size() == 10 * MiB + 4096, "a device buffer reports its class instead of the requested size"); + } + const auto made = mock.allocations; + DeviceBuffer writeBack(context, 10 * MiB + 512 * 1024, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + Expect(writeBack.Handle() == first && mock.allocations == made, "a retained 11 MiB device buffer did not serve a write-back of another size and direction in its class"); + Expect(writeBack.Size() == 10 * MiB + 512 * 1024, "a reused device buffer reports the retained size"); +} + +void LargerClass() { + mock = MockDevice{}; + auto context = mockContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + VkBuffer twelve = VK_NULL_HANDLE; + { + DeviceBuffer large(context, 12 * MiB, usage); + twelve = large.Handle(); + } + { + DeviceBuffer smaller(context, 7 * MiB, usage); + Expect(smaller.Handle() == twelve, "a 7 MiB request did not take the retained 12 MiB buffer"); + } + const auto made = mock.allocations; + { + DeviceBuffer tooSmall(context, 5 * MiB, usage); + Expect(tooSmall.Handle() != twelve && mock.allocations == made + 1, "a 5 MiB request took a retained buffer of more than twice its size"); + Expect(mock.sizes.at(tooSmall.Handle()) == 5 * MiB, "a 5 MiB device buffer was not created at its own class"); + } + DeviceBuffer again(context, 12 * MiB, usage); + Expect(again.Handle() == twelve, "a buffer served to a smaller request did not return to its own 12 MiB class"); + DeviceBuffer best(context, 4 * MiB + 1, usage); + Expect(best.Handle() != twelve && mock.sizes.at(best.Handle()) == 5 * MiB, "a 4 MiB + 1 request did not take the smallest retained class that fits (5 MiB)"); +} + +void KeptUntilFence() { + mock = MockDevice{}; + auto context = mockContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + std::vector> batch; + for (int i = 0; i < 3; ++i) batch.push_back(std::make_shared(context, 6 * MiB + static_cast(i + 1) * 4096, usage)); + for (std::size_t i = 0; i < batch.size(); ++i) { + for (std::size_t j = i + 1; j < batch.size(); ++j) Expect(batch[i]->Handle() != batch[j]->Handle(), "two buffers of one unfinished batch share a VkBuffer"); + } + const auto kept = batch.front()->Handle(); + const auto made = mock.allocations; + { + DeviceBuffer during(context, 6 * MiB, usage); + Expect(during.Handle() != kept && mock.allocations == made + 1, "a buffer kept by an unfinished batch was handed out again"); + } + batch.clear(); + std::vector> after; + for (int i = 0; i < 3; ++i) after.push_back(std::make_unique(context, 6 * MiB + 100, usage)); + after.push_back(std::make_unique(context, 6 * MiB, usage)); + Expect(mock.allocations == made + 1, "the buffers released after the batch fence were not reused (" + std::to_string(mock.allocations - made - 1) + " new allocations)"); +} + +void Budget() { + mock = MockDevice{}; + auto context = mockContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + std::vector> live; + for (int i = 0; i < 12; ++i) live.push_back(std::make_unique(context, (48 + static_cast(i) * 3) * MiB, usage)); + const auto first = live.front()->Handle(); + VkDeviceSize madeBytes = 0; + for (const auto& buffer : live) madeBytes += mock.sizes.at(buffer->Handle()); + Expect(madeBytes > 512 * MiB, "the budget case does not exceed the budget"); + for (auto& buffer : live) buffer.reset(); + Expect(mock.frees != 0 && mock.liveBytes <= 512 * MiB, "the device tier retains " + std::to_string(mock.liveBytes / MiB) + " MiB after " + std::to_string(mock.frees) + " evictions, over its 512 MiB budget"); + DeviceBuffer probe(context, 48 * MiB, usage); + Expect(probe.Handle() != first, "the least recently used device buffer survived the eviction"); +} + +void AddressAndHostUnchanged() { + mock = MockDevice{}; + auto context = mockContext(); + VkBuffer scratch = VK_NULL_HANDLE; + { + DeviceBuffer buffer(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + scratch = buffer.Handle(); + } + Buffer addressed(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Expect(addressed.Handle() != scratch && addressed.DeviceAddress() != 0, "an addressable device-local buffer took a scratch buffer without a device address"); + Buffer shadow(context, 2 * MiB - 256, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Expect(shadow.Handle() == scratch, "a device-local staging buffer did not share the scratch buffers' class"); + VkBuffer host = VK_NULL_HANDLE; + { + Buffer copy(context, 10 * MiB + 4096, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, HostVisible); + host = copy.Handle(); + Expect(mock.sizes.at(host) == 10 * MiB + 4096, "a large host buffer was rounded to a class"); + Expect(mock.usages.at(host) == VK_BUFFER_USAGE_TRANSFER_SRC_BIT, "a host buffer was created with more usages than asked"); + } + Buffer other(context, 10 * MiB + 8192, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, HostVisible); + Expect(other.Handle() != host, "a large host buffer of another exact size was reused"); + Buffer same(context, 10 * MiB + 4096, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, HostVisible); + Expect(same.Handle() == host, "a large host buffer of the same exact size was not reused"); +} + +void SmallDeviceClasses() { + mock = MockDevice{}; + auto context = mockContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + VkBuffer mebibyte = VK_NULL_HANDLE; + VkBuffer half = VK_NULL_HANDLE; + { + DeviceBuffer one(context, MiB, usage); + mebibyte = one.Handle(); + } + { + DeviceBuffer smaller(context, 300 * 1024, usage); + Expect(smaller.Handle() != mebibyte && mock.sizes.at(smaller.Handle()) == 512 * 1024, "a 300 KiB request took a 1 MiB buffer or skipped its power-of-two class"); + half = smaller.Handle(); + } + DeviceBuffer again(context, 260 * 1024, usage); + Expect(again.Handle() == half, "a 260 KiB request did not take the retained 512 KiB buffer"); + constexpr auto indirect = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + VkBuffer tiny = VK_NULL_HANDLE; + { + DeviceBuffer args(context, 20, indirect); + tiny = args.Handle(); + } + const auto made = mock.allocations; + DeviceBuffer nextArgs(context, 20, indirect); + Expect(nextArgs.Handle() == tiny && mock.allocations == made, "a 20-byte device request did not reuse the retained 256-byte buffer of its class"); +} + +} + +int main() { + try { + SizeClassesAndDirections(); + LargerClass(); + KeptUntilFence(); + Budget(); + AddressAndHostUnchanged(); + SmallDeviceClasses(); + } catch (const std::exception& error) { + std::fprintf(stderr, "FAIL: %s\n", error.what()); + return 1; + } + if (failures != 0) return 1; + std::puts("buffer pool tests passed"); + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp b/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp index 972270466..40b69b7a4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp @@ -50,6 +50,26 @@ void RunColorTargetLayoutTests() { standardSeen[address / 4] = true; } } + for (const auto mode : {ColorTileMode::Standard4KB, ColorTileMode::Standard64KB}) { + for (const std::uint32_t bpe : {1u, 2u, 4u, 8u, 16u}) { + const auto block = mode == ColorTileMode::Standard64KB ? 65536u : 4096u; + const ColorTargetLayout sized(512, 512, mode, bpe); + Require(sized.Alignment() == block && sized.Bytes() == ComputeSurfaceSize(ComputeElementMipLayout(ColorTextureTileMode(mode), bpe, 512, 512, 1), 1), "standard color layout differs from the texture layout"); + std::vector seen(block / bpe); + std::uint32_t count = 0; + for (std::uint32_t y = 0; y < 512 && count < seen.size(); ++y) { + for (std::uint32_t x = 0; x < 512; ++x) { + const auto address = sized.Offset(x, y); + if (address >= block) continue; + Require(address % bpe == 0 && !seen[address / bpe], "standard block aliases its texels"); + seen[address / bpe] = true; + ++count; + } + } + Require(count == seen.size(), "standard block leaves texels unaddressed"); + } + } + Require(DecodeColorTileMode(0x4dc24000) == ColorTileMode::Standard64KB, "64 KiB standard color descriptor was rejected"); reject([&] { layout.Offset(257, 0); }); std::vector tiled(layout.Bytes(), std::byte{0x5a}); std::vector linear(layout.LinearBytes()); diff --git a/core/libs/prx/libSceAgcDriver/tests/ControlFlow.cpp b/core/libs/prx/libSceAgcDriver/tests/ControlFlow.cpp index df874dd91..14e020f13 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ControlFlow.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ControlFlow.cpp @@ -285,7 +285,7 @@ void verifyNullSwappc() { verifyLongBranch("backward long branch", LongBranchBack, 0x24u, 0x08u); const std::array call{0xbe8c1f00u, 0xbe8e210cu, 0xbf810000u}; const std::vector, std::string_view>> refused{ - {"s_swappc_b64 with a return address", {call.begin(), call.end()}, "unsupported SOP1 opcode 33"}, + {"s_swappc_b64 with a return address", {call.begin(), call.end()}, "unclosable scalar call/return pairing"}, {"index rewritten after the bound", patched(DwordTableGetpcFirst, 5, {0xbe9203ffu, 0x00000040u}), "unsupported dynamic s_setpc_b64"}, {"index scaled for two-dword entries", patched(DwordTableGetpcFirst, 7, {0x8f128312u}), "unsupported dynamic s_setpc_b64"}, {"entry rewritten after the load", patched(DwordTableGetpcFirst, 11, {0xbe8e0380u}), "unsupported dynamic s_setpc_b64"}, diff --git a/core/libs/prx/libSceAgcDriver/tests/DisplayBufferDcc.cpp b/core/libs/prx/libSceAgcDriver/tests/DisplayBufferDcc.cpp index b2173b7ec..f5f1f159f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/DisplayBufferDcc.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/DisplayBufferDcc.cpp @@ -1,13 +1,30 @@ #include "prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" +#include #include +#include #include #include +#include #include #include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif using AgcDriver::DisplayBuffer; using AgcDriver::Graphics::DccKeys; +using AgcDriver::Graphics::GuestTextureResource; +using AgcDriver::Graphics::TextureDimension; +using AgcDriver::Graphics::TextureTileMode; namespace { @@ -38,6 +55,193 @@ void ExpectPixels(const DisplayBuffer& buffer, DccKeys keys, std::array; +constexpr DccPattern PipeAligned{X(3), Y(4), X(6), Y(6), X(7), Y(7), X(8), Y(8), X(3) ^ Y(3), X(4) ^ Y(4), X(6) ^ Y(5), X(5) ^ Y(6)}; +constexpr DccPattern Displayable{X(3), Y(3), X(4), Y(4), X(5), Y(5), X(6), Y(6), X(7), Y(7), X(8), Y(8)}; + +std::size_t PatternKey(const DccPattern& pattern, std::uint32_t width, std::uint32_t x, std::uint32_t y) { + std::size_t offset = 0; + for (std::size_t bit = 0; bit < pattern.size(); ++bit) offset |= static_cast((std::popcount(x & pattern[bit].x) + std::popcount(y & pattern[bit].y)) & 1) << bit; + return (static_cast(y / 512u) * ((width + 511u) / 512u) + x / 512u) * 4096u + offset; +} + +void Retile(std::span render, std::span display, std::uint32_t width, std::uint32_t height) { + for (std::uint32_t y = 0; y < (height + 511u) / 512u * 512u; y += 8) { + for (std::uint32_t x = 0; x < (width + 511u) / 512u * 512u; x += 8) display[PatternKey(Displayable, width, x, y)] = render[PatternKey(PipeAligned, width, x, y)]; + } +} + +struct KeyHistogram { + std::size_t uncompressed = 0; + std::size_t clear0000 = 0; + std::size_t firstDiffering = 0; + std::size_t zeroRuns = 0; + bool operator==(const KeyHistogram&) const = default; +}; + +KeyHistogram Histogram(std::span keys) { + KeyHistogram histogram; + for (std::size_t key = 0; key < keys.size(); ++key) { + histogram.uncompressed += keys[key] == 0xff; + histogram.clear0000 += keys[key] == 0x00; + histogram.zeroRuns += keys[key] == 0x00 && (key == 0 || keys[key - 1] != 0x00); + if (histogram.firstDiffering == 0 && keys[key] != keys[0]) histogram.firstDiffering = key; + } + return histogram; +} + +std::string Describe(const KeyHistogram& histogram) { + return std::to_string(histogram.uncompressed) + " x 0xff, " + std::to_string(histogram.clear0000) + " x 0x00, first differing key " + std::to_string(histogram.firstDiffering) + ", " + std::to_string(histogram.zeroRuns) + " runs of 0x00"; +} + +std::uint64_t TiledBytes(std::uint32_t width, std::uint32_t height) { + return static_cast((width + 127u) / 128u) * ((height + 127u) / 128u) * 65536u; +} + +GuestTextureResource Surface(std::uint32_t width, std::uint32_t height, std::uint64_t dccAddress, bool pipeAligned) { + GuestTextureResource surface{}; + surface.baseAddress = 0x10000; + surface.width = width; + surface.height = height; + surface.mipCount = 1; + surface.tileMode = TextureTileMode::kR64KBX; + surface.dimension = TextureDimension::k2D; + surface.format = AgcDriver::Graphics::FindGuestColorTargetFormat(VK_FORMAT_B8G8R8A8_UNORM, 4).value(); + surface.dccAddress = dccAddress; + surface.dccPipeAligned = pipeAligned; + return surface; +} + +std::uint8_t* ReserveKeys(std::size_t writable, std::size_t reserved) { +#ifdef _WIN32 + auto* block = static_cast(VirtualAlloc(nullptr, reserved, MEM_RESERVE, PAGE_NOACCESS)); + if (block == nullptr || VirtualAlloc(block, writable, MEM_COMMIT, PAGE_READWRITE) == nullptr) return nullptr; +#else + void* mapped = mmap(nullptr, reserved, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped == MAP_FAILED || mprotect(mapped, writable, PROT_READ | PROT_WRITE) != 0) return nullptr; + auto* block = static_cast(mapped); +#endif + return block; +} + +void ReleaseKeys(std::uint8_t* block, std::size_t reserved) { +#ifdef _WIN32 + static_cast(reserved); + VirtualFree(block, 0, MEM_RELEASE); +#else + munmap(block, reserved); +#endif +} + +void KeyCountTests() { + struct Extent { + std::uint32_t width; + std::uint32_t height; + std::size_t keys; + }; + for (const auto& extent : {Extent{1920, 1080, 49152}, Extent{3840, 2160, 163840}, Extent{1280, 720, 24576}, Extent{259, 137, 4096}, Extent{128, 128, 4096}, Extent{2560, 1440, 61440}}) { + const auto surfaceBytes = TiledBytes(extent.width, extent.height); + const auto name = std::to_string(extent.width) + "x" + std::to_string(extent.height); + const auto keys = AgcDriver::Graphics::DccKeyCount(TextureTileMode::kR64KBX, 4, extent.width, extent.height, surfaceBytes); + Expect(keys == extent.keys, name + " SW_64KB_R_X 4-byte DCC covers " + std::to_string(keys) + " key bytes, expected " + std::to_string(extent.keys)); + if (extent.keys == surfaceBytes / 256) continue; + const auto aligned = AgcDriver::Graphics::DccKeyCount(Surface(extent.width, extent.height, 0, true), surfaceBytes); + const auto unaligned = AgcDriver::Graphics::DccKeyCount(Surface(extent.width, extent.height, 0, false), surfaceBytes); + Expect(aligned == extent.keys && unaligned == surfaceBytes / 256, name + " surface descriptors' DCC covers " + std::to_string(aligned) + " key bytes pipe-aligned and " + std::to_string(unaligned) + " unaligned, expected " + std::to_string(extent.keys) + " and " + std::to_string(surfaceBytes / 256)); + } + Expect(TiledBytes(2560, 1440) / 256 == 61440, "2560x1440 does not fill its meta blocks"); + const auto bytes = TiledBytes(1920, 1080); + Expect(AgcDriver::Graphics::DccKeyCount(TextureTileMode::kD64KBX, 4, 1920, 1080, bytes) == bytes / 256, "an SW_64KB_D_X surface's DCC left one key per 256 bytes"); + Expect(AgcDriver::Graphics::DccKeyCount(TextureTileMode::kR64KBX, 8, 1920, 1080, 2 * bytes) == 2 * bytes / 256, "an 8-byte surface's DCC left one key per 256 bytes"); + const auto surface = Surface(1920, 1080, 0, true); + auto shape = surface; + shape.mipCount = 2; + Expect(AgcDriver::Graphics::DccKeyCount(shape, bytes + 65536) == (bytes + 65536) / 256, "a mip chain's DCC left one key per 256 bytes"); + shape = surface; + shape.dimension = TextureDimension::k2DArray; + shape.depthOrLastArray = 1; + Expect(AgcDriver::Graphics::DccKeyCount(shape, 2 * bytes) == 2 * bytes / 256, "a two-layer surface's DCC left one key per 256 bytes"); + shape = surface; + shape.format = AgcDriver::Graphics::FindGuestTextureFormat(VK_FORMAT_R16G16B16A16_SFLOAT, 8).value(); + Expect(AgcDriver::Graphics::DccKeyCount(shape, 2 * bytes) == 2 * bytes / 256, "an 8-byte surface descriptor's DCC left one key per 256 bytes"); + Expect(AgcDriver::DisplayBufferKeyBytes({65536, 0x8000000000000000ull, 1920, 1080}) == 49152 && AgcDriver::DisplayBufferKeyBytes({65536, 0x8000000000000000ull, 259, 137}) == 4096, "display buffers do not read the console's DCC extent"); +} + +void RetileTests() { + constexpr std::uint32_t width = 1920; + constexpr std::uint32_t height = 1080; + const auto surfaceBytes = TiledBytes(width, height); + const auto driverKeys = static_cast(surfaceBytes / 256); + std::vector render(49152); + std::vector display(render.size()); + std::fill_n(render.begin(), driverKeys, 0xff); + Retile(render, display, width, height); + const auto reported = Histogram(std::span(display).first(driverKeys)); + Expect(reported == KeyHistogram{33552, 1008, 32784, 105}, "a retile of 0xff over the first 34560 render keys does not give the PPSA12544 display keys: " + Describe(reported)); + std::fill(render.begin(), render.end(), 0x00); + std::fill(display.begin(), display.end(), 0x00); + const auto surface = Surface(width, height, reinterpret_cast(render.data()), true); + AgcDriver::Graphics::MarkDccUncompressed(surface.dccAddress, surfaceBytes, AgcDriver::Graphics::DccKeyCount(surface, surfaceBytes)); + Retile(render, display, width, height); + DisplayBuffer buffer{65536, 0x8000000000000000ull, width, height}; + buffer.dccAddress = reinterpret_cast(display.data()); + auto keys = DccKeys::Unreadable; + const auto message = Rejection([&] { keys = AgcDriver::DisplayBufferKeys(buffer); }); + Expect(message.empty() && keys == DccKeys::Uncompressed, "the driver's store over a 1920x1080 render DCC, retiled, does not present: " + message + " (display keys over the first " + std::to_string(driverKeys) + ": " + Describe(Histogram(std::span(display).first(driverKeys))) + ")"); + + constexpr std::size_t reserved = 65536; + constexpr std::size_t writable = 0x9000; + auto* small = ReserveKeys(writable, reserved); + Expect(small != nullptr, "cannot reserve metadata with an inaccessible tail"); + if (small == nullptr) return; + const auto stored = Rejection([&] { AgcDriver::Graphics::MarkDccUncompressed(reinterpret_cast(small), surfaceBytes, AgcDriver::Graphics::DccKeyCount(surface, surfaceBytes)); }); + Expect(stored.find("not writable over the 0xc000 key bytes") != std::string::npos && small[0] == 0x00, "a key store over metadata smaller than the console's extent did not throw: " + stored); + const auto unaligned = Rejection([&] { AgcDriver::Graphics::MarkDccUncompressed(reinterpret_cast(small), surfaceBytes, AgcDriver::Graphics::DccKeyCount(Surface(width, height, reinterpret_cast(small), false), surfaceBytes)); }); + Expect(unaligned.empty() && std::all_of(small, small + driverKeys, [](std::uint8_t key) { return key == 0xff; }) && small[driverKeys] == 0x00, "an unaligned DCC key store over metadata smaller than the console's extent did not cover one key per 256 bytes: " + unaligned); + ReleaseKeys(small, reserved); +} + +DccKeys ReadKeys(DisplayBuffer buffer, std::vector& keys) { + buffer.dccAddress = reinterpret_cast(keys.data()); + return AgcDriver::DisplayBufferKeys(buffer); +} + +void ExtentTests() { + const DisplayBuffer full{65536, 0x8000000000000000ull, 1920, 1080}; + std::vector keys(49152); + for (const auto& [code, expected] : {std::pair{std::uint8_t{0xff}, DccKeys::Uncompressed}, std::pair{std::uint8_t{0x00}, DccKeys::Clear0000}, std::pair{std::uint8_t{0xc0}, DccKeys::Clear1111}, std::pair{std::uint8_t{0x20}, DccKeys::ClearRegister}}) { + std::fill(keys.begin(), keys.end(), code); + auto read = DccKeys::Unreadable; + const auto message = Rejection([&] { read = ReadKeys(full, keys); }); + Expect(message.empty() && read == expected, std::string("uniform ") + AgcDriver::Graphics::DccKeysName(expected) + " keys over the extent read " + AgcDriver::Graphics::DccKeysName(read) + ": " + message); + } + std::fill(keys.begin(), keys.end(), 0xff); + const auto corner = PatternKey(Displayable, 1920, 1919, 1079); + Expect(corner == 46205, "pixel (1919, 1079) is not displayable key 46205"); + keys[corner] = 0x00; + auto message = Rejection([&] { ReadKeys(full, keys); }); + Expect(message.find("mixed") != std::string::npos && message.find("key 46205 (pixels 1912..1919 x 1072..1079 in the displayable order) is 0x00, key 0 is 0xff") != std::string::npos, "a 0000 key past one key per 256 bytes was not read or not named: " + message); + std::fill(keys.begin(), keys.end(), 0x40); + keys[0] = 0x80; + message = Rejection([&] { ReadKeys(full, keys); }); + Expect(message.find("key 1 (pixels 8..15 x 0..7 in the displayable order) is 0x40, key 0 is 0x80") != std::string::npos, "a 1110 key 0 among 0001 keys was not named: " + message); + const DisplayBuffer odd{65536, 0x8000000000000000ull, 259, 137}; + keys.assign(4096, 0x40); + keys.back() = 0xff; + message = Rejection([&] { ReadKeys(odd, keys); }); + Expect(message.find("key 4095 (pixels 504..511 x 504..511 in the displayable order) is 0xff") != std::string::npos, "259x137: an uncompressed padding key among 0001 keys was ignored: " + message); + Expect(!Rejection([&] { AgcDriver::DisplayBufferKeys(full); }).empty(), "a display buffer without DCC metadata had keys read"); +} + } int main() { @@ -77,6 +281,9 @@ int main() { Expect(!Rejection([&] { AgcDriver::DisplayBufferSize(buffer); }).empty(), "a linear display buffer with DCC metadata was accepted"); buffer.tilingMode = 0; Expect(AgcDriver::DisplayBufferSize(buffer) == 2 * 65536, "DCC metadata changed the display footprint"); + KeyCountTests(); + RetileTests(); + ExtentTests(); if (failures != 0) return 1; std::puts("display buffer DCC tests passed"); return 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/Flip.cpp b/core/libs/prx/libSceAgcDriver/tests/Flip.cpp index 596974c90..da4fb199c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Flip.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Flip.cpp @@ -193,7 +193,7 @@ void testReset(bool compute) { if (compute) sceAgcDriverSubmitAcb(0x20, &packet); else sceAgcDriverSubmitDcb(&packet); const auto message = expectFailure([] { AgcDriverWaitIdle_nid_postfix(); }); - check(message.find(compute ? "registered" : "required shader register") != std::string::npos, "suspend reset wrong queue state"); + check(message.find(compute ? "not readable" : "required shader register") != std::string::npos, "suspend reset wrong queue state"); } } @@ -203,7 +203,7 @@ int main(int argc, char** argv) { if (argc == 2) testReset(std::string(argv[1]) == "compute"); else { testFlipAndBoundary(); testFailure(); } const auto shutdown = expectFailure([] { LibcRunShutdown_nid_postfix(); }); - check(shutdown.find(argc == 2 ? (std::string(argv[1]) == "compute" ? "registered" : "required shader register") : "intentional flip failure") != std::string::npos, "shutdown lost worker failure"); + check(shutdown.find(argc == 2 ? (std::string(argv[1]) == "compute" ? "not readable" : "required shader register") : "intentional flip failure") != std::string::npos, "shutdown lost worker failure"); std::puts("AGC flip and suspend tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index e6e2faefe..f4b0550a8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -6,8 +6,11 @@ #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "SceShaders.hpp" #include "ControlFlow/RequestSerializer.hpp" #include "CacheKey.hpp" +#include "BdaAbi.hpp" #include #include #include @@ -22,6 +25,7 @@ #include #include #include +#include #include #include @@ -83,6 +87,14 @@ void stateTests() { initial.context[0xdead] = 1; initial.ClearContext(); Require(initial.context.at(0x200) == 0 && !initial.context.contains(0xdead), "context reset did not restore defaults"); + for (std::uint32_t slot = 0; slot < 8; ++slot) { + const auto info = 0x31cu + 0xfu * slot; + Require(initial.context.at(info) == 0, "reset color target was not disabled"); + initial.context[info] = 10u << 2u; + } + initial.ClearContext(); + initial.context[0x8e] = initial.context[0x8f] = 0xffffffffu; + Require(AgcDriver::Graphics::ColorWriteMask(initial.context) == 0, "context clear kept old color targets enabled"); Require(initial.userConfig.at(0x24b) == 0, "primitive restart must be disabled in initial queue state"); initial.userConfig[0x24b] = 1; initial.ClearContext(); @@ -206,11 +218,11 @@ void stateTests() { queue.context.erase(0x1c5); queue.shader[0x008] = 0x100; queue.shader[0x009] = 0; - Require(!AgcDriver::Graphics::PixelProgramUnset(queue), "a pixel program address was read as unset"); + Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), "a pixel program address was read as unset"); Require(AgcDriver::Graphics::DrawRejection(queue, true).find("missing register at DWORD 0x1b3") != std::string::npos, "a real pixel program without SPI_PS_INPUT_ENA was accepted"); expectFailure([&] { AgcDriver::Graphics::DecodePixelStageInfo(queue.context, std::array{0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u}); }, "missing register"); queue.shader[0x008] = 0; - Require(AgcDriver::Graphics::PixelProgramUnset(queue), "a zero pixel program address was not read as unset"); + Require(AgcDriver::Graphics::PixelProgramSkipped(queue), "a zero pixel program address was not read as unset"); Require(AgcDriver::Graphics::DrawRejection(queue, true).find("writes color") != std::string::npos, "a draw without a pixel program that writes color was accepted"); queue.context[0x8e] = 0; Require(AgcDriver::Graphics::DrawRejection(queue, true).empty(), "a depth-only draw without a pixel program was rejected"); @@ -219,6 +231,78 @@ void stateTests() { const auto unset = AgcDriver::Graphics::DecodePixelStageInfo(queue.context, AgcDriver::Graphics::ExportMappings(state), true); Require(unset.inputAddr == ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PerspectiveCenter), "unset pixel inputs of the null program did not read as PERSP_CENTER_ENA"); for (const auto mode : unset.targetOutputMode) Require(mode == 0, "an unset SPI_SHADER_COL_FORMAT exported a color"); + queue.shader[0x008] = 0x100; + queue.context[0x1c4] = 0; + queue.context[0x203] = 0; + Require(AgcDriver::Graphics::PixelProgramSkipped(queue), "a pixel program that writes nothing and cannot run was not skipped"); + Require(AgcDriver::Graphics::DrawRejection(queue, true).empty(), "a depth-only draw whose pixel program cannot run was rejected"); + for (const auto [control, what] : {std::pair{0x400u, "EXEC_ON_NOOP"}, std::pair{0x40u, "KILL_ENABLE"}}) { + queue.context[0x203] = control; + Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), std::string("a pixel program with ") + what + " was skipped"); + Require(AgcDriver::Graphics::DrawRejection(queue, true).find("missing register at DWORD 0x1b3") != std::string::npos, std::string("a pixel program with ") + what + " ran without SPI_PS_INPUT_ENA"); + } + queue.context[0x203] = 1; + Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), "a pixel program with Z_EXPORT_ENABLE was skipped"); + queue.context[0x203] = 0; + queue.context[0x1c4] = 1; + Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), "a pixel program exporting depth was skipped"); + queue.context[0x1c4] = 0; + queue.context[0x8e] = 0xf; + Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), "a pixel program writing color was skipped"); + queue.context[0x1b3] = queue.context[0x1b4] = queue.context[0x1b6] = 0xffffffffu; + queue.context[0x1c5] = queue.context[0x203] = 0xffffffffu; + const auto disabled = AgcDriver::Graphics::DecodePixelStageInfo(queue.context, AgcDriver::Graphics::ExportMappings(state), true); + Require(disabled.interpolatorCount == 0 && disabled.inputAddr == ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PerspectiveCenter), "the null pixel program inherited stale input state"); + Require(!disabled.pixelKillEnable && !disabled.depthExportEnable && !disabled.sampleMaskExportEnable, "the null pixel program inherited stale exports"); + for (const auto mode : disabled.targetOutputMode) Require(mode == 0, "the null pixel program inherited stale color exports"); + expectFailure([&] { AgcDriver::Graphics::DecodePixelStageInfo(queue.context, AgcDriver::Graphics::ExportMappings(state), false); }, "input count exceeds 32"); +} + +VkFormatFeatureFlags srgb8Features = 0; + +void srgb8FormatProperties(VkPhysicalDevice, VkFormat format, VkFormatProperties* properties) { + *properties = {}; + if (format == VK_FORMAT_R8_SRGB) properties->optimalTilingFeatures = srgb8Features; +} + +void setProxyVariable(const char* value) { +#ifdef _WIN32 + _putenv_s("APS5_SRGB_ATTACHMENT_PROXY", value); +#else + if (*value == 0) unsetenv("APS5_SRGB_ATTACHMENT_PROXY"); + else setenv("APS5_SRGB_ATTACHMENT_PROXY", value, 1); +#endif +} + +void srgb8TargetTests() { + setProxyVariable(""); + auto queue = makeState(); + queue.context[0x31c] = 0x8604; + const auto color = AgcDriver::Graphics::DecodeState(queue).color; + Require(color.format == VK_FORMAT_R8_SRGB && color.elementBytes == 1 && color.componentMapping == 0xe4u, "the 8_SRGB color target did not decode as R8_SRGB"); + for (std::uint32_t swap = 1; swap < 4; ++swap) { + queue.context[0x31c] = 0x8604 | (swap << 11u); + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "unsupported color format 1 number type 6 component swap " + std::to_string(swap)); + } + queue.context[0x31c] = 0x860c; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "unsupported color format 3 number type 6 component swap 0"); + AgcDriver::Graphics::Context context{}; + context.formatProperties = srgb8FormatProperties; + const auto proxyOn = [&](std::uintptr_t device, VkFormatFeatureFlags features, VkFormat format = VK_FORMAT_R8_SRGB) { + srgb8Features = features; + context.physical = reinterpret_cast(device); + return AgcDriver::Graphics::AttachmentProxyFormat(context, format); + }; + constexpr VkFormatFeatureFlags attachment = VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT; + constexpr VkFormatFeatureFlags blend = VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT; + Require(proxyOn(0x1000, VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT) == VK_FORMAT_R8G8B8A8_SRGB, "a device without R8_SRGB attachments must render 8_SRGB through RGBA8_SRGB"); + Require(proxyOn(0x2000, attachment) == VK_FORMAT_R8G8B8A8_SRGB, "a device without R8_SRGB blending must render 8_SRGB through RGBA8_SRGB"); + Require(proxyOn(0x3000, attachment | blend) == VK_FORMAT_UNDEFINED, "a device with R8_SRGB attachments must render 8_SRGB directly"); + Require(proxyOn(0x3000, 0) == VK_FORMAT_UNDEFINED, "the attachment proxy decision is not kept per device"); + Require(proxyOn(0x4000, 0, VK_FORMAT_R8_UNORM) == VK_FORMAT_UNDEFINED && proxyOn(0x4000, 0, VK_FORMAT_R8G8_SRGB) == VK_FORMAT_UNDEFINED, "only 8_SRGB targets may render through a proxy"); + setProxyVariable("1"); + Require(proxyOn(0x5000, attachment | blend) == VK_FORMAT_R8G8B8A8_SRGB, "APS5_SRGB_ATTACHMENT_PROXY=1 did not force the proxy"); + setProxyVariable(""); } void hardwareScreenOffsetTests() { @@ -319,7 +403,7 @@ void ShaderStageTests() { queue.context[0x2d5] = value; expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "reserved"); } - for (const auto bit : {1u, 8u, 0x40u, 0x100u, 0x200u, 0x400u, 0x1000u, 0x4000u, 0x8000u, 0x80000u, 0x200000u, 0x800000u, 0x1000000u}) { + for (const auto bit : {1u, 8u, 0x40u, 0x100u, 0x200u, 0x400u, 0x1000u, 0x4000u, 0x80000u, 0x200000u, 0x800000u, 0x1000000u}) { queue.context[0x2d5] = 0x2000u | bit; expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "unsupported vertex"); } @@ -410,6 +494,34 @@ std::vector pixelNoPerspectiveLocations(bool barycentricEnabled) return result2; } +void ComputeScratchTests() { + auto queue = makeState(); + auto& shader = queue.shader; + shader[0x207] = 64u; + shader[0x208] = 1u; + shader[0x209] = 1u; + shader[0x213] = 0x1u; + std::vector header(sizeof(Shader)); + Shader agc{}; + agc.scratch_size_dw_per_thread = 24; + std::memcpy(header.data(), &agc, sizeof(Shader)); + const auto compute = AgcDriver::Graphics::DecodeComputeStageInfo(shader, header); + Require(compute.scratchDwords == 24u, "SCRATCH_EN did not take the AGC header's per-thread scratch size"); + agc.scratch_size_dw_per_thread = 0; + std::memcpy(header.data(), &agc, sizeof(Shader)); + expectFailure([&] { static_cast(AgcDriver::Graphics::DecodeComputeStageInfo(shader, header)); }, "zero scratch size"); + shader[0x213] = 0u; + Require(AgcDriver::Graphics::DecodeComputeStageInfo(shader, {}).scratchDwords == 0u, "a dispatch without SCRATCH_EN got scratch"); + ShaderRecompiler::RecompileRequest request{}; + const std::array code{0xbf810000u}; + request.shader = {ShaderRecompiler::ShaderStage::Compute, 0x30000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.compute = compute; + const ShaderRecompiler::RequestSerializer serializer; + const auto back = serializer.Deserialize(serializer.Serialize(request)); + Require(back.request.context.compute->scratchDwords == 24u, "the compute scratch size did not survive serialization"); +} + void PixelInputLayoutTests() { using ShaderRecompiler::PixelInput; using ShaderRecompiler::PixelInputVgpr; @@ -475,6 +587,80 @@ void DisabledColorTests() { Require(partial.hasColorTarget && partial.blend.colorWriteMask == 3, "partial color write mask changed"); queue.context.erase(0x31c); expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "missing register"); + queue = makeState(); + queue.context[0x31c] = 0; + for (const auto offset : {0x31bu, 0x31du, 0x3b0u, 0x3b8u, 0x390u, 0x318u, 0x1e0u}) queue.context.erase(offset); + const auto disabled = AgcDriver::Graphics::DecodeState(queue); + Require(!disabled.hasColorTarget && disabled.colors.empty() && disabled.blends.empty(), "COLOR_INVALID retained an attachment despite the disabled buffer"); + Require(disabled.renderExtent.width == 64 && disabled.renderExtent.height == 4, "COLOR_INVALID lost the attachment-free render extent"); + queue.shader[0x008] = 0; + queue.shader[0x009] = 0; + Require(AgcDriver::Graphics::NullPixelProgramRejection(queue).empty(), "COLOR_INVALID rejected a draw without a pixel shader"); + queue.context[0x200] = 0x36; + queue.context[0x000] = 0; + queue.context[0x002] = 0; + queue.context[0x010] = 0x80000181; + queue.context[0x011] = 0x20000180; + queue.context[0x012] = 0x100; + queue.context[0x014] = 0x100; + queue.context[0x007] = 0x003f003f; + queue.context[0x00a] = 0; + queue.context[0x00b] = std::bit_cast(1.0f); + const auto depthOnly = AgcDriver::Graphics::DecodeState(queue); + Require(!depthOnly.hasColorTarget && depthOnly.depth && depthOnly.depthTest && depthOnly.depthWrite && depthOnly.renderExtent.height == 64, "COLOR_INVALID discarded a depth-only draw"); + for (const auto colorControl : {0x0u, 0xcc0000u}) { + queue = makeState(); + queue.context[0x202] = colorControl; + const auto unwritten = AgcDriver::Graphics::DecodeState(queue); + Require(!unwritten.hasColorTarget && unwritten.colors.empty() && unwritten.color.address == 0, "CB_COLOR_CONTROL mode disable kept a color attachment"); + Require(unwritten.renderExtent.width == 64 && unwritten.renderExtent.height == 4, "CB_COLOR_CONTROL mode disable lost the screen scissor extent"); + } +} + +void TuningFieldTests() { + auto queue = makeState(); + queue.context[0x1b3] = 2; + queue.context[0x1b4] = 2; + const auto baseline = AgcDriver::Graphics::DecodeState(queue); + queue.context[0x292] = 0x22; + auto state = AgcDriver::Graphics::DecodeState(queue); + Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "ALTERNATE_RBS_PER_TILE was rejected"); + Require(state.scissor.offset.x == baseline.scissor.offset.x && state.scissor.extent.width == baseline.scissor.extent.width && state.scissor.extent.height == baseline.scissor.extent.height, "ALTERNATE_RBS_PER_TILE changed the scissor"); + queue.context[0x292] = 0x26; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "scan conversion mode"); + queue.context[0x292] = 2; + queue.context[0x202] = 0xcc0011; + state = AgcDriver::Graphics::DecodeState(queue); + Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "DISABLE_DUAL_QUAD was rejected"); + Require(state.hasColorTarget && state.blend.colorWriteMask == baseline.blend.colorWriteMask, "DISABLE_DUAL_QUAD changed color output"); + queue.context[0x202] = 0xcc0013; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "copy ROP"); + queue.context[0x202] = 0xcc0010; + for (const auto groups : {1u, 2u, 15u}) { + queue.context[0x2d5] = 0x2000u | (groups << 15u); + state = AgcDriver::Graphics::DecodeState(queue); + Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "MAX_PRIMGRP_IN_WAVE was rejected"); + Require(state.stages.path == AgcDriver::Graphics::ShaderPath::Vertex && state.stages.vertexWaveSize == 64u, "MAX_PRIMGRP_IN_WAVE changed vertex routing"); + } +} + +void ReversedComponentOrderTests() { + for (const auto& [swap, mapping] : {std::pair{2u, 0x1bu}, std::pair{3u, 0x93u}}) { + auto queue = makeState(); + queue.context[0x31c] = (queue.context[0x31c] & ~(3u << 11u)) | (swap << 11u); + const auto state = AgcDriver::Graphics::DecodeState(queue); + Require(state.colors.size() == 1 && state.colors[0].format == VK_FORMAT_R8G8B8A8_UNORM && state.colors[0].componentMapping == mapping, "an 8_8_8_8 target with a reversed component order did not map exports onto RGBA8"); + Require(AgcDriver::Graphics::ExportMappings(state)[0] == mapping && state.blends[0].colorWriteMask == 0xfu, "a reversed 8_8_8_8 target did not write all four channels through its export mapping"); + queue.context[0x1e0] = 0x40010001u; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "reversed component order"); + queue.context[0x1e0] = 0; + queue.context[0x31c] = (queue.context[0x31c] & ~((0x1fu << 2u) | (7u << 8u))) | (12u << 2u) | (7u << 8u); + const auto wide = AgcDriver::Graphics::DecodeState(queue); + Require(wide.colors.size() == 1 && wide.colors[0].format == VK_FORMAT_R16G16B16A16_SFLOAT && wide.colors[0].componentMapping == mapping && wide.blends[0].colorWriteMask == 0xfu, "a 16_16_16_16 float target with a reversed component order did not map exports onto RGBA16"); + } + auto queue = makeState(); + queue.context[0x31c] = (queue.context[0x31c] & ~((0x1fu << 2u) | (7u << 8u) | (3u << 11u))) | (12u << 2u) | (7u << 8u) | (1u << 11u); + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "component swap 1"); } void CompactedExportTests() { @@ -496,6 +682,10 @@ void CompactedExportTests() { Require(state.colors[1].slot == 4 && state.colors[1].exportIndex == 1 && state.colors[1].address == slotFour, "export 1 did not reach MRT slot 4, the second slot CB_SHADER_MASK enables"); Require(!state.blends[0].blendEnable && state.blends[1].blendEnable && state.blends[1].colorWriteMask == (VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT), "export 1 did not take MRT slot 4's blend control and target mask"); Require(AgcDriver::Graphics::ExportMappings(state)[1] == state.colors[1].componentMapping, "export 1 did not take MRT slot 4's component mapping"); + auto disabledFirst = queue; + disabledFirst.context[0x31c] = 0; + const auto withHole = AgcDriver::Graphics::DecodeState(disabledFirst); + Require(withHole.colors.size() == 1 && withHole.colors[0].slot == 4 && withHole.colors[0].exportIndex == 1 && withHole.blends.size() == 2 && withHole.blends[0].colorWriteMask == 0, "COLOR_INVALID shifted a later MRT export"); queue.context[0x1c5] = 0x90009u; expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "color export format 0"); queue.context[0x8e] = 0xf000fu; @@ -582,6 +772,31 @@ void DepthStencilTests() { expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "DCC 3D color targets"); } +void depthMaintenanceTests() { + for (const auto mode : {0x4u, 0x8u, 0x10u, 0x80u, 0x100u, 0x1000u, 0x4000u}) { + for (const auto clear : {0u, 1u, 2u, 3u}) { + auto queue = makeState(); + queue.context[0x000] = mode | clear; + queue.context[0x200] = 0; + queue.context[0x8e] = 0; + queue.context[0x8f] = 0; + queue.shader.erase(0x8); + const auto reason = AgcDriver::Graphics::DepthMaintenanceRejection(queue); + Require(reason.find("DB_RENDER_CONTROL") != std::string::npos, "depth maintenance passed without depth tests or color writes"); + Require(AgcDriver::Graphics::DrawRejection(queue, false) == reason, "draw precheck did not reject depth maintenance first"); + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, reason); + } + } + for (const auto control : {0u, 1u, 2u, 3u, 0x20u, 0x40u, 0x2000u, 0x2063u}) { + auto queue = makeState(); + queue.context[0x000] = control; + Require(AgcDriver::Graphics::DepthMaintenanceRejection(queue).empty(), "ordinary depth controls were mistaken for maintenance"); + } + auto absent = makeState(); + absent.context.erase(0x000); + Require(AgcDriver::Graphics::DepthMaintenanceRejection(absent).empty(), "an absent depth control produced a maintenance verdict"); +} + // SPI_SHADER_Z_FORMAT (0x1c4) and the export enables of DB_SHADER_CONTROL (0x203): Z export needs a // format with a depth channel (1, 2, 3 or 32_ABGR 9), the sample mask needs 32_ABGR, and the // formats the export path does not lay out are refused. A missing 0x1c4 gives no verdict here. @@ -648,6 +863,14 @@ void DepthBoundsBiasTests() { Require(state.depthBias && state.depthBiasConstant == 4.0f, "culled back faces must not constrain the front depth bias"); queue.context[0x2de] = 0x1f0u; expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "units other than the depth format"); + queue.context[0x2de] = 0x1e9u; + auto cleared = queue; + cleared.ClearContext(); + for (const auto& [offset, value] : queue.context) { + if (offset != 0x2dfu) cleared.context[offset] = value; + } + state = AgcDriver::Graphics::DecodeState(cleared); + Require(state.depthBias && state.depthBiasConstant == 4.0f && state.depthBiasClamp == 0.0f, "a depth bias clamp the title never writes must decode as its reset value 0"); } alignas(256) std::array dccKeys{}; @@ -761,6 +984,110 @@ void metadataPassTests() { Require(AgcDriver::Graphics::FillDccClear(tenBitPass->targets[0].format, clear.keys, tenBitPass->targets[0].dccAlphaOnMsb, std::as_writable_bytes(std::span(copied))) && std::ranges::all_of(copied, [&](std::uint32_t word) { return word == clear.texel; }), "a copied " + name + " target under " + clear.code + " keys was not filled with its 10/10/10/2 texel"); } } + + constexpr std::uint32_t tiledSide = 128; + constexpr std::size_t tiledBytes = tiledSide * tiledSide * 4; + constexpr std::size_t blockAlignment = 65536; + const auto keyCount = AgcDriver::Graphics::DccKeyCount(AgcDriver::Graphics::TextureTileMode::kR64KBX, 4, tiledSide, tiledSide, tiledBytes); + Require(keyCount == 4096 && AgcDriver::Graphics::DccKeyBytes(tiledBytes) == 256, "the 128x128 SW_64KB_R_X DCC extent changed"); + std::vector tiledBlock(blockAlignment + tiledBytes + keyCount); + const auto tiledAddress = (reinterpret_cast(tiledBlock.data()) + blockAlignment - 1) / blockAlignment * blockAlignment; + auto* tiledTexels = reinterpret_cast(tiledAddress); + auto* tiledKeys = tiledTexels + tiledBytes; + for (const bool pipeAligned : {true, false}) { + auto tiled = queue; + tiled.context[0x3b8] |= (static_cast(AgcDriver::Graphics::ColorTileMode::RenderTarget) << 14u) | (pipeAligned ? 1u << 30u : 0u); + tiled.context[0x3b0] = ((tiledSide - 1u) << 14u) | (tiledSide - 1u); + tiled.context[0x318] = static_cast(tiledAddress >> 8u); + tiled.context[0x390] = static_cast(tiledAddress >> 40u); + tiled.context[0x325] = static_cast((tiledAddress + tiledBytes) >> 8u); + tiled.context[0x3a8] = static_cast((tiledAddress + tiledBytes) >> 40u); + for (const auto offset : {0xdu, 0x82u, 0x91u, 0x95u}) tiled.context[offset] = (tiledSide << 16u) | tiledSide; + tiled.context[0x10f] = std::bit_cast(64.0f); + tiled.context[0x110] = std::bit_cast(64.0f); + tiled.context[0x111] = std::bit_cast(-64.0f); + tiled.context[0x112] = std::bit_cast(64.0f); + const auto tiledPass = DecodeColorMetadataPass(tiled); + const std::string alignment = pipeAligned ? "pipe-aligned" : "unaligned"; + Require(tiledPass.has_value() && tiledPass->targets.size() == 1 && tiledPass->targets[0].tileMode == AgcDriver::Graphics::ColorTileMode::RenderTarget && tiledPass->targets[0].bytes == tiledBytes && tiledPass->targets[0].dccAddress == tiledAddress + tiledBytes && tiledPass->targets[0].dccPipeAligned == pipeAligned, "the 128x128 SW_64KB_R_X metadata pass target with " + alignment + " DCC changed"); + const auto expected = pipeAligned ? keyCount : AgcDriver::Graphics::DccKeyBytes(tiledBytes); + for (const auto& [key, texel, code] : {std::tuple{std::uint8_t{0x20}, 0x11223344u, "register"}, std::tuple{std::uint8_t{0xc0}, 0xffffffffu, "1111"}}) { + std::memset(tiledTexels, 0x5a, tiledBytes); + std::memset(tiledKeys, key, keyCount); + AgcDriver::Graphics::RunColorMetadataPass(context, *tiledPass); + const auto stored = static_cast(std::count(tiledKeys, tiledKeys + keyCount, std::uint8_t{0xff})); + bool filled = true; + for (std::size_t offset = 0; offset < tiledBytes; offset += 4) filled = filled && std::memcmp(tiledTexels + offset, &texel, 4) == 0; + Require(filled && stored == expected && std::all_of(tiledKeys, tiledKeys + expected, [](std::uint8_t value) { return value == 0xff; }), std::string("a ") + code + " fast clear eliminate of a 128x128 SW_64KB_R_X target with " + alignment + " DCC stored uncompressed keys over " + std::to_string(stored) + " of its " + std::to_string(keyCount) + " DCC key bytes, expected " + std::to_string(expected)); + } + } +} + +alignas(256) std::array cmaskMemory{}; + +void cmaskTests() { + using AgcDriver::Graphics::DecodeColorBuffer; + auto queue = makeState(); + queue.context[0x31c] |= 0x2000; + const auto cmaskAddress = reinterpret_cast(cmaskMemory.data()); + queue.context[0x31f] = static_cast(cmaskAddress >> 8u); + queue.context[0x398] = static_cast(cmaskAddress >> 40u); + queue.context[0x323] = 0x11223344; + queue.context[0x324] = 0; + const auto target = DecodeColorBuffer(queue.context, 0); + Require(target.cmaskAddress == cmaskAddress && target.cmaskBytes == cmaskMemory.size(), "the fast-clear target lost its CMASK"); + Require(DecodeColorBuffer(makeState().context, 0).cmaskAddress == 0, "a target without FAST_CLEAR got a CMASK"); + using AgcDriver::Graphics::CmaskBytes; + Require(CmaskBytes(1920, 1080) == 0x6000 && CmaskBytes(960, 544) == 0x2000 && CmaskBytes(800, 450) == 0x1000 && CmaskBytes(1024, 512) == 0x1000 && CmaskBytes(3840, 2160) == 0x14000, "a CMASK does not span whole 1024x512 metablocks"); + auto mipmapped = queue; + mipmapped.context[0x3b0] |= 1u << 28u; + expectFailure([&] { DecodeColorBuffer(mipmapped.context, 0); }, "mipmapped, 3D or array"); + auto unaddressed = queue; + unaddressed.context[0x31f] = 0; + unaddressed.context[0x398] = 0; + expectFailure([&] { DecodeColorBuffer(unaddressed.context, 0); }, "without a CMASK address"); + + queue.context[0x0] = 0; + queue.context[0x202] = 0xcc0020; + auto pass = AgcDriver::Graphics::DecodeColorMetadataPass(queue); + Require(pass.has_value() && pass->targets.size() == 1 && pass->targets[0].cmaskAddress == cmaskAddress, "the metadata pass lost the target's CMASK"); + const AgcDriver::Graphics::Context context{}; + const auto texels = [&](std::uint32_t value) { + for (std::size_t offset = 0; offset < colorMemory.size(); offset += 4) { + std::uint32_t texel = 0; + std::memcpy(&texel, colorMemory.data() + offset, 4); + if (texel != value) return false; + } + return true; + }; + const auto cmaskIs = [&](std::uint8_t value) { return std::all_of(cmaskMemory.begin(), cmaskMemory.end(), [&](std::uint8_t byte) { return byte == value; }); }; + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); + cmaskMemory.fill(0); + AgcDriver::Graphics::RunColorMetadataPass(context, *pass); + Require(texels(0x11223344) && cmaskIs(0xff), "a CMASK fast clear was not eliminated into the texels"); + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); + AgcDriver::Graphics::RunColorMetadataPass(context, *pass); + Require(texels(0x5a5a5a5au) && cmaskIs(0xff), "a pass over an expanded CMASK changed the texels"); + cmaskMemory.fill(0); + cmaskMemory[cmaskMemory.size() - 1] = 0xff; + expectFailure([&] { AgcDriver::Graphics::RunColorMetadataPass(context, *pass); }, "not all fast-cleared or all expanded"); + Require(texels(0x5a5a5a5au), "a refused pass changed the texels"); + + queue.context[0x31c] |= 0x10000000; + const auto keysAddress = reinterpret_cast(dccKeys.data()); + queue.context[0x325] = static_cast(keysAddress >> 8u); + queue.context[0x3a8] = static_cast(keysAddress >> 40u); + pass = AgcDriver::Graphics::DecodeColorMetadataPass(queue); + Require(pass.has_value() && pass->targets.size() == 1 && pass->targets[0].cmaskAddress == cmaskAddress && pass->targets[0].dccAddress == keysAddress, "a DCC fast-clear target lost its CMASK or DCC keys"); + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); + cmaskMemory.fill(0xff); + dccKeys.fill(0x20); + AgcDriver::Graphics::RunColorMetadataPass(context, *pass); + Require(texels(0x11223344) && cmaskIs(0xff) && std::ranges::all_of(dccKeys, [](std::uint8_t key) { return key == 0xff; }), "a DCC fast clear under an expanded CMASK was not eliminated through its keys"); + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); + cmaskMemory.fill(0); + expectFailure([&] { AgcDriver::Graphics::RunColorMetadataPass(context, *pass); }, "DCC color target that is not all expanded"); + Require(texels(0x5a5a5a5au), "a refused pass changed the texels of a DCC target"); } void DepthClipTests() { @@ -851,9 +1178,15 @@ struct MockVulkan { std::map allocationFlags; std::map bufferMemory; std::map> memories; + std::map allocationSizes; + VkDeviceSize allocatedBytes = 0; + std::optional memoryLimit; + std::uint64_t allocationAttempts = 0; std::vector layoutBindings; std::vector poolSizes; std::uint32_t poolMaxSets = 0; + VkDescriptorPoolCreateFlags poolFlags = 0; + std::uint32_t freedSets = 0; std::vector writes; std::uint32_t boundSets = 0; std::uint32_t boundFirst = 0; @@ -888,8 +1221,12 @@ VKAPI_ATTR void VKAPI_CALL mockGetBufferMemoryRequirements(VkDevice, VkBuffer bu } VKAPI_ATTR VkResult VKAPI_CALL mockAllocateMemory(VkDevice, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks*, VkDeviceMemory* memory) { + ++mock.allocationAttempts; + if (mock.memoryLimit.has_value() && mock.allocatedBytes + info->allocationSize > *mock.memoryLimit) return VK_ERROR_OUT_OF_DEVICE_MEMORY; *memory = makeHandle(); mock.memories[*memory] = std::vector(info->allocationSize); + mock.allocationSizes[*memory] = info->allocationSize; + mock.allocatedBytes += info->allocationSize; if (info->pNext != nullptr) { const auto* flags = static_cast(info->pNext); mock.allocationFlags[*memory] = flags->flags; @@ -917,7 +1254,11 @@ VKAPI_ATTR void VKAPI_CALL mockDestroyBuffer(VkDevice, VkBuffer, const VkAllocat --mock.live; } -VKAPI_ATTR void VKAPI_CALL mockFreeMemory(VkDevice, VkDeviceMemory, const VkAllocationCallbacks*) { +VKAPI_ATTR void VKAPI_CALL mockFreeMemory(VkDevice, VkDeviceMemory memory, const VkAllocationCallbacks*) { + if (const auto found = mock.allocationSizes.find(memory); found != mock.allocationSizes.end()) { + mock.allocatedBytes -= found->second; + mock.allocationSizes.erase(found); + } --mock.live; } @@ -936,6 +1277,7 @@ VKAPI_ATTR VkResult VKAPI_CALL mockCreateDescriptorPool(VkDevice, const VkDescri *pool = makeHandle(); mock.poolSizes.assign(info->pPoolSizes, info->pPoolSizes + info->poolSizeCount); mock.poolMaxSets = info->maxSets; + mock.poolFlags = info->flags; ++mock.live; return VK_SUCCESS; } @@ -950,6 +1292,11 @@ VKAPI_ATTR VkResult VKAPI_CALL mockAllocateDescriptorSets(VkDevice, const VkDesc return VK_SUCCESS; } +VKAPI_ATTR VkResult VKAPI_CALL mockFreeDescriptorSets(VkDevice, VkDescriptorPool, std::uint32_t count, const VkDescriptorSet*) { + mock.freedSets += count; + return VK_SUCCESS; +} + VKAPI_ATTR void VKAPI_CALL mockUpdateDescriptorSets(VkDevice, std::uint32_t count, const VkWriteDescriptorSet* writes, std::uint32_t copyCount, const VkCopyDescriptorSet*) { Require(copyCount == 0, "descriptor copies are not expected"); for (std::uint32_t i = 0; i < count; ++i) { @@ -965,6 +1312,12 @@ VKAPI_ATTR void VKAPI_CALL mockCmdBindDescriptorSets(VkCommandBuffer, VkPipeline mock.boundSets = count; } +VKAPI_ATTR VkResult VKAPI_CALL mockGetMemoryHostPointerProperties(VkDevice, VkExternalMemoryHandleTypeFlagBits type, const void* pointer, VkMemoryHostPointerPropertiesEXT* properties) { + Require(type == VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT && pointer != nullptr, "invalid host pointer import query"); + properties->memoryTypeBits = 1; + return VK_SUCCESS; +} + VKAPI_ATTR VkDeviceAddress VKAPI_CALL mockGetBufferDeviceAddress(VkDevice, const VkBufferDeviceAddressInfo* info) { Require(mock.bufferMemory.contains(info->buffer), "BDA buffer was not bound"); Require((mock.bufferUsage.at(info->buffer) & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0, "BDA buffer usage is missing"); @@ -1034,6 +1387,7 @@ VKAPI_ATTR void VKAPI_CALL mockCmdUpdateBuffer(VkCommandBuffer, VkBuffer buffer, PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) { static const std::map table{ {"vkGetBufferDeviceAddressKHR", reinterpret_cast(mockGetBufferDeviceAddress)}, + {"vkGetMemoryHostPointerPropertiesEXT", reinterpret_cast(mockGetMemoryHostPointerProperties)}, {"vkCreateBuffer", reinterpret_cast(mockCreateBuffer)}, {"vkGetBufferMemoryRequirements", reinterpret_cast(mockGetBufferMemoryRequirements)}, {"vkAllocateMemory", reinterpret_cast(mockAllocateMemory)}, @@ -1047,6 +1401,7 @@ PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) { {"vkCreateDescriptorPool", reinterpret_cast(mockCreateDescriptorPool)}, {"vkDestroyDescriptorPool", reinterpret_cast(mockDestroyDescriptorPool)}, {"vkAllocateDescriptorSets", reinterpret_cast(mockAllocateDescriptorSets)}, + {"vkFreeDescriptorSets", reinterpret_cast(mockFreeDescriptorSets)}, {"vkUpdateDescriptorSets", reinterpret_cast(mockUpdateDescriptorSets)}, {"vkCmdBindDescriptorSets", reinterpret_cast(mockCmdBindDescriptorSets)}, {"vkCreatePipelineLayout", reinterpret_cast(mockCreatePipelineLayout)}, @@ -1071,7 +1426,8 @@ AgcDriver::Graphics::Context mockContext() { context.memory.memoryTypes[0].propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; context.limits.minStorageBufferOffsetAlignment = 1; context.limits.maxBoundDescriptorSets = 1; - context.limits.maxStorageBufferRange = 4096; + context.limits.maxStorageBufferRange = 16384; + context.nullDescriptors = true; context.limits.maxPerStageDescriptorStorageBuffers = 16; context.limits.maxPerStageResources = 128; context.limits.maxDescriptorSetStorageBuffers = 32; @@ -1106,6 +1462,13 @@ ShaderRecompiler::DescriptorBinding makeBinding(Role role, std::uint32_t binding return result; } +std::vector ShaderDataWords(std::initializer_list userData) { + std::vector words(ShaderRecompiler::RuntimeAbi::ShaderDataDwords); + words[0] = ShaderRecompiler::RuntimeAbi::Version; + std::copy(userData.begin(), userData.end(), words.begin() + ShaderRecompiler::RuntimeAbi::UserDataDword); + return words; +} + bool sameBytes(const std::vector& memory, const void* expected, std::size_t bytes) { return memory.size() >= bytes && std::memcmp(memory.data(), expected, bytes) == 0; } @@ -1189,7 +1552,8 @@ void resourceTests() { ShaderRecompiler::RecompileResult vertex; ShaderRecompiler::RecompileResult fragment; vertex.bindings.push_back(makeBinding(Role::GuestBuffers, 0, 2, join(vsharp(guestFirst.data(), 16), vsharp(guestSecond.data(), 32)))); - vertex.bindings.push_back(makeBinding(Role::ShaderData, 5, 1, {7, 8, 9})); + vertex.bindings.push_back(makeBinding(Role::ShaderData, 5, 1, ShaderDataWords({7, 8, 9}))); + vertex.shaderDataDwords = ShaderRecompiler::RuntimeAbi::ShaderDataDwords; fragment.bindings.push_back(makeBinding(Role::FlattenedSrt, 43, 1, {1, 2})); fragment.bindings.push_back(makeBinding(Role::GuestBuffers, 44, 1, vsharp(guestThird.data(), 8))); AgcDriver::Graphics::ShaderResources resources(context, vertex, fragment, state.color, 0, 0); @@ -1204,8 +1568,8 @@ void resourceTests() { Require(array.count == 2 && array.buffers.size() == 2 && array.type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, "guest buffer array write is incorrect"); Require(array.buffers[0].offset == 0 && array.buffers[0].range == 16 && array.buffers[1].offset == 0 && array.buffers[1].range == 32, "guest buffers must be bound at zero offset with their descriptor size"); Require(sameBytes(bufferBytes(array.buffers[0].buffer), guestFirst.data(), 16) && sameBytes(bufferBytes(array.buffers[1].buffer), guestSecond.data(), 32), "guest buffer contents were not uploaded"); - const std::array data{7, 8, 9}; - Require(findWrite(5).buffers.size() == 1 && findWrite(5).buffers[0].range == 12 && sameBytes(bufferBytes(findWrite(5).buffers[0].buffer), data.data(), 12), "shader data buffer is incorrect"); + const auto data = ShaderDataWords({7, 8, 9}); + Require(findWrite(5).buffers.size() == 1 && findWrite(5).buffers[0].range == data.size() * 4u && sameBytes(bufferBytes(findWrite(5).buffers[0].buffer), data.data(), data.size() * 4u), "shader data buffer is incorrect"); const std::array srt{1, 2}; Require(findWrite(43).buffers.size() == 1 && findWrite(43).buffers[0].range == 8 && sameBytes(bufferBytes(findWrite(43).buffers[0].buffer), srt.data(), 8), "flattened SRT buffer is incorrect"); Require(findWrite(44).buffers.size() == 1 && findWrite(44).buffers[0].range == 8 && sameBytes(bufferBytes(findWrite(44).buffers[0].buffer), guestThird.data(), 8), "fragment guest buffer is incorrect"); @@ -1278,9 +1642,10 @@ void resourceTests() { mutate(binding); return binding; }; - expectSingleFailure(changed([](auto& binding) { binding.role = Role::GuestImages; binding.kind = Kind::SampledImage; }), "guest texture descriptor must contain 8 dwords"); - expectSingleFailure(changed([](auto& binding) { binding.role = Role::GuestImages; binding.kind = Kind::StorageImage; }), "guest storage image descriptors must contain 8 dwords"); - expectSingleFailure(changed([](auto& binding) { binding.role = Role::GuestSamplers; binding.kind = Kind::Sampler; }), "shader sampler descriptors exceed per-stage limits"); + expectSingleFailure(changed([](auto& binding) { binding.role = Role::GuestImages; binding.kind = Kind::SampledImage; binding.binding = 1u; }), "guest texture descriptor must contain 8 dwords"); + expectSingleFailure(changed([](auto& binding) { binding.role = Role::GuestImages; binding.kind = Kind::StorageImage; binding.binding = 29u; }), "guest storage image descriptors must contain 8 dwords"); + expectSingleFailure(changed([](auto& binding) { binding.role = Role::GuestSamplers; binding.kind = Kind::Sampler; binding.binding = static_cast(ShaderRecompiler::RuntimeAbi::Binding::Samplers); }), "shader sampler descriptors exceed per-stage limits"); + expectSingleFailure(changed([](auto& binding) { binding.role = Role::GuestImages; binding.kind = Kind::SampledImage; binding.binding = 29u; }), "resource class disagrees"); expectSingleFailure(changed([](auto& binding) { binding.role = Role::Gds; }), "invalid GDS descriptor contract"); expectSingleFailure(changed([](auto& binding) { binding.role = Role::BdaPagetable; binding.guestDescriptor.clear(); }), "BDA table and fault descriptors"); expectSingleFailure(changed([](auto& binding) { binding.role = Role::FaultBuffer; binding.guestDescriptor.clear(); }), "BDA table and fault descriptors"); @@ -1294,11 +1659,13 @@ void resourceTests() { expectSingleFailure(changed([](auto& binding) { binding.count = 2; }), "four DWORDs per array element"); expectSingleFailure(changed([](auto& binding) { binding.role = Role::ShaderData; binding.count = 2; binding.guestDescriptor = {1, 2}; }), "must not be arrays"); expectSingleFailure(changed([](auto& binding) { binding.role = Role::ShaderData; binding.guestDescriptor.clear(); }), "empty shader data descriptor"); + expectSingleFailure(changed([](auto& binding) { binding.role = Role::ShaderData; }), "invalid compact shader data size"); + expectSingleFailure(changed([](auto& binding) { binding.role = Role::ShaderData; binding.guestDescriptor = ShaderDataWords({}); binding.guestDescriptor.pop_back(); }), "invalid compact shader data size"); expectSingleFailure(changed([](auto& binding) { binding.role = Role::FlattenedSrt; binding.guestDescriptor.clear(); }), "empty shader data descriptor"); expectSingleFailure(changed([](auto& binding) { binding.guestDescriptor[1] |= 0x40000000u; }), "reserved bits"); expectSingleFailure(changed([](auto& binding) { binding.guestDescriptor[3] |= 0x40000000u; }), "unsupported type"); expectSingleFailure(changed([](auto& binding) { binding.guestDescriptor[1] |= 0x3fffu << 16u; binding.guestDescriptor[2] = 0xffffffffu; }), "descriptor range limit"); - expectSingleFailure(changed([](auto& binding) { binding.guestDescriptor[2] = 8192; }), "descriptor range limit"); + expectSingleFailure(changed([](auto& binding) { binding.guestDescriptor[2] = 32768; }), "descriptor range limit"); expectSingleAccepted(changed([](auto& binding) { binding.guestDescriptor = vsharp(reinterpret_cast(0x1000), 8); }), "an unmapped V#"); expectSingleFailure(changed([&](auto& binding) { binding.guestDescriptor = vsharp(reinterpret_cast(state.color.address), 64); }), "aliases the render target"); expectSingleAccepted(changed([](auto& binding) { binding.count = 3; binding.guestDescriptor = join(join(vsharp(guestFirst.data(), 16), vsharp(guestSecond.data(), 32)), vsharp(reinterpret_cast(0x1000), 8)); }), "an unmapped V# element"); @@ -1312,6 +1679,7 @@ void resourceTests() { const auto expectStageResources = [&](Role role, Kind kind, std::uint32_t words, std::uint32_t limit, std::string_view reason) { vertex.bindings.back().role = role; vertex.bindings.back().kind = kind; + vertex.bindings.back().binding = kind == Kind::StorageImage ? ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding : kind == Kind::Sampler ? static_cast(ShaderRecompiler::RuntimeAbi::Binding::Samplers) : ShaderRecompiler::RuntimeAbi::FirstImageBinding; vertex.bindings.back().guestDescriptor.assign(words, 0); mock = MockVulkan{}; auto limited = mockContext(); @@ -1336,6 +1704,102 @@ void resourceTests() { } } +void descriptorCacheTests() { + mock = MockVulkan{}; + auto context = mockContext(); + context.limits.maxDescriptorSetStorageBuffers = 8192; + context.limits.maxPerStageDescriptorStorageBuffers = 8192; + context.limits.maxPerStageResources = 8192; + context.limits.maxDescriptorSetSampledImages = 2048; + { + AgcDriver::Graphics::DescriptorCache cache(context); + const auto layout = [&](std::uint32_t count) { + const VkDescriptorSetLayoutBinding binding{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, count, VK_SHADER_STAGE_FRAGMENT_BIT, nullptr}; + const std::array key{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, count, VK_SHADER_STAGE_FRAGMENT_BIT}; + return cache.Layout(key, std::span(&binding, 1)); + }; + const auto large = layout(4097); + const auto live = mock.live; + const std::array oversized{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4097}, {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 3}}}; + const auto dedicated = cache.Allocate(large, oversized); + Require(dedicated.set != VK_NULL_HANDLE && dedicated.pool != VK_NULL_HANDLE, "a set above the chain pool's capacity got no set"); + Require(mock.live == live + 1 && mock.poolMaxSets == 1 && mock.poolFlags == 0, "an oversized set does not get a pool of its own"); + Require(mock.poolSizes.size() == 2 && mock.poolSizes[0].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER && mock.poolSizes[0].descriptorCount == 4097 && mock.poolSizes[1].type == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE && mock.poolSizes[1].descriptorCount == 3, "the dedicated pool is not sized to its set"); + Require(cache.Counters().pools == 0 && cache.Counters().sets == 1, "an oversized set was counted in the chain pools"); + cache.Free(dedicated); + Require(mock.live == live && mock.freedSets == 0, "freeing an oversized set did not destroy its pool"); + const std::array fitting{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4096}}}; + const auto chained = cache.Allocate(layout(4096), fitting); + Require(chained.set != VK_NULL_HANDLE && mock.poolMaxSets == 1024 && mock.poolFlags == VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT && cache.Counters().pools == 1, "a set the chain pool holds left the chain"); + cache.Free(chained); + Require(mock.freedSets == 1 && mock.live == live + 2, "a chain set was not freed back to its pool"); + const std::array beyondDevice{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 8193}}}; + expectFailure([&] { cache.Allocate(large, beyondDevice); }, "descriptor set exceeds the device's per-set descriptor limit"); + const std::array storageImages{{{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1}}}; + expectFailure([&] { cache.Allocate(large, storageImages); }, "descriptor set exceeds the device's per-set descriptor limit"); + const std::array uniformBuffers{{{VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1}}}; + expectFailure([&] { cache.Allocate(large, uniformBuffers); }, "descriptor set uses an unsupported descriptor type 6"); + } + Require(mock.live == 0, "the descriptor cache leaked a pool or layout"); +} + +void textureCacheBudgetTests() { + using AgcDriver::Graphics::TextureCacheBudget; + constexpr std::uint64_t GiB = 1ull << 30u; + VkPhysicalDeviceMemoryProperties memory{}; + Require(TextureCacheBudget(memory) == 2 * GiB, "a device without memory heaps does not keep 2 GiB of cached textures"); + memory.memoryHeapCount = 3; + memory.memoryHeaps[0] = {256ull << 20u, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT}; + memory.memoryHeaps[1] = {32 * GiB, 0}; + memory.memoryHeaps[2] = {16 * GiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT | VK_MEMORY_HEAP_MULTI_INSTANCE_BIT}; + Require(TextureCacheBudget(memory) == 4 * GiB, "a 16 GiB device-local heap does not give 4 GiB of cached textures"); + memory.memoryHeaps[2].size = 6 * GiB; + Require(TextureCacheBudget(memory) == 2 * GiB, "a 6 GiB device-local heap does not keep the 2 GiB floor"); + memory.memoryHeaps[2].size = 24 * GiB; + memory.memoryHeapCount = 2; + Require(TextureCacheBudget(memory) == 2 * GiB, "a heap past memoryHeapCount or a host heap counted toward the texture caches"); + memory.memoryHeapCount = 3; + Require(TextureCacheBudget(memory) == 6 * GiB, "a 24 GiB device-local heap does not give 6 GiB of cached textures"); +} + +void sampledTextureBudgetTests() { + using AgcDriver::Graphics::SampledTextureBudget; + constexpr std::uint64_t GiB = 1ull << 30u; + constexpr std::uint64_t MiB = 1ull << 20u; + VkPhysicalDeviceMemoryProperties memory{}; + memory.memoryHeapCount = 3; + memory.memoryHeaps[0] = {256 * MiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT}; + memory.memoryHeaps[1] = {32 * GiB, 0}; + memory.memoryHeaps[2] = {16 * GiB, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT}; + VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + Require(SampledTextureBudget(memory, nullptr, 0) == 4 * GiB, "without VK_EXT_memory_budget the sampled texture cache does not keep a quarter of the 16 GiB heap"); + reported.heapBudget[0] = 100 * GiB; + reported.heapBudget[1] = 100 * GiB; + Require(SampledTextureBudget(memory, &reported, 0) == 4 * GiB, "a zero budget for the largest device-local heap did not fall back to a quarter of the heap"); + reported.heapBudget[2] = 15 * GiB; + reported.heapUsage[2] = 9 * GiB; + Require(SampledTextureBudget(memory, &reported, 5 * GiB) == 5 * GiB + 128 * MiB, "a 15 GiB budget with 4 GiB used outside the texture caches does not leave 5 GiB 128 MiB after the 4 GiB storage cache and 1 GiB 896 MiB of headroom"); + reported.heapUsage[2] = 3 * GiB; + Require(SampledTextureBudget(memory, &reported, 5 * GiB) == 9 * GiB + 128 * MiB, "usage below the cached bytes is not read as no other device memory in use"); + reported.heapUsage[2] = 12 * GiB; + Require(SampledTextureBudget(memory, &reported, 1 * GiB) == 2 * GiB, "a heap whose other users leave no room does not keep the 2 GiB floor"); + memory.memoryHeaps[0].flags = 0; + memory.memoryHeaps[2].flags = 0; + Require(SampledTextureBudget(memory, &reported, 0) == 2 * GiB, "a device without a device-local heap does not keep 2 GiB of sampled textures"); +} + +void sampledBudgetReportTests() { + using AgcDriver::Graphics::SampledBudgetReportDue; + using std::chrono::seconds; + constexpr std::uint64_t MiB = 1ull << 20u; + Require(SampledBudgetReportDue(0, 2048 * MiB, seconds(10)), "a budget never reported is not due"); + Require(!SampledBudgetReportDue(4000 * MiB, 4400 * MiB, seconds(60)), "a budget 10% above the reported one is due, not only one more than 10% away"); + Require(!SampledBudgetReportDue(4000 * MiB, 3600 * MiB, seconds(60)), "a budget 10% below the reported one is due, not only one more than 10% away"); + Require(SampledBudgetReportDue(4000 * MiB, 4401 * MiB, seconds(10)), "a budget more than 10% above the reported one is not due after 10 s"); + Require(SampledBudgetReportDue(4000 * MiB, 3599 * MiB, seconds(10)), "a budget more than 10% below the reported one is not due after 10 s"); + Require(!SampledBudgetReportDue(4000 * MiB, 2048 * MiB, seconds(9)), "a budget change is reported again within 10 s of the last report"); +} + void misalignedShaderDataTests() { mock = MockVulkan{}; auto context = mockContext(); @@ -1345,21 +1809,24 @@ void misalignedShaderDataTests() { { ShaderRecompiler::RecompileResult compute; compute.bindings.push_back(makeBinding(Role::GuestBuffers, 0, 2, join(vsharp(guest.data(), 32), vsharp(guest.data() + 1, 8)))); - compute.bindings.push_back(makeBinding(Role::ShaderData, 1, 1, {0x11, 0x22, 0})); - compute.memoryOffsetDword = 2; + compute.bindings.push_back(makeBinding(Role::ShaderData, 1, 1, ShaderDataWords({0x11, 0x22}))); + compute.shaderDataDwords = ShaderRecompiler::RuntimeAbi::ShaderDataDwords; + compute.memoryOffsetDword = ShaderRecompiler::RuntimeAbi::BufferOffsetsDword; auto live = compute; - live.bindings[1].guestDescriptor = {0x33, 0x44, 0}; + live.bindings[1].guestDescriptor = ShaderDataWords({0x33, 0x44}); const AgcDriver::Graphics::CompiledShader shader{ShaderRecompiler::ShaderStage::Compute, &compute, 0}; const AgcDriver::Graphics::CompiledShader liveShader{ShaderRecompiler::ShaderStage::Compute, &live, 0}; AgcDriver::Graphics::ShaderResources resources(context, shader); const auto& views = findWrite(0); Require(views.buffers.size() == 2 && views.buffers[0].range == 32 && views.buffers[1].range == 12 && views.buffers[1].offset == views.buffers[0].offset, "the misaligned view is not bound from the aligned offset below it"); const auto data = findWrite(1).buffers.at(0).buffer; - const std::array patched{0x11, 0x22, 0x400}; - Require(sameBytes(bufferBytes(data), patched.data(), sizeof(patched)), "the shader data buffer does not hold the misaligned view's offset"); + auto patched = ShaderDataWords({0x11, 0x22}); + patched[ShaderRecompiler::RuntimeAbi::BufferOffsetsDword] = 0x400u; + Require(sameBytes(bufferBytes(data), patched.data(), patched.size() * 4u), "the shader data buffer does not hold the misaligned view's offset"); Require(resources.RefreshData(commands, liveShader), "a refresh with different words recorded nothing"); - const std::array refreshed{0x33, 0x44, 0x400}; - Require(sameBytes(bufferBytes(data), refreshed.data(), sizeof(refreshed)), "a data refresh dropped the misaligned view's offset"); + auto refreshed = ShaderDataWords({0x33, 0x44}); + refreshed[ShaderRecompiler::RuntimeAbi::BufferOffsetsDword] = 0x400u; + Require(sameBytes(bufferBytes(data), refreshed.data(), refreshed.size() * 4u), "a data refresh dropped the misaligned view's offset"); Require(!resources.DataWordsDiffer(liveShader) && resources.DataWordsHash() == AgcDriver::Graphics::ShaderResources::DataWordsHash(liveShader), "the refreshed template's words are not the dispatch's"); } Require(mock.live == 0, "misaligned shader data resources leaked Vulkan objects"); @@ -1765,6 +2232,202 @@ void conservativeZExportTests() { expectFailure([&] { AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false); }, "unsupported fragment execution mode"); } +constexpr std::array OrderedPixelCode{0x7e0e02f2u, 0xbf900007u, 0xf800180fu, 0x07070707u, 0xbf810000u}; +constexpr std::array OrderedBdaPixelCode{0x7e040f00u, 0x7e060280u, 0xdc208001u, 0x047d0002u, 0xdc308008u, 0x057d0002u, 0xbf8c3f70u, 0x4a080b04u, 0x7e0e0d04u, 0xf800180fu, 0x07070707u, 0xbf810000u}; + +AgcDriver::QueueState orderedPixelQueue(std::uint32_t shaderControl) { + auto queue = makeState(); + queue.context[0x1b3] = 0x2u; + queue.context[0x1b4] = 0x2u; + queue.context[0x203] = shaderControl; + return queue; +} + +ShaderRecompiler::RecompileRequest orderedPixelRequest(std::uint32_t shaderControl, std::span capabilities, std::span code = OrderedPixelCode) { + const auto queue = orderedPixelQueue(shaderControl); + ShaderRecompiler::RecompileRequest request{}; + request.shader = {ShaderRecompiler::ShaderStage::Fragment, 0x30000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.pixel = AgcDriver::Graphics::DecodePixelStageInfo(queue.context, IdentityExports); + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 64; + request.target.supportedCapabilities = capabilities; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + return request; +} + +std::string interlockShape(std::span words) { + std::string shape; + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + const auto op = static_cast(words[at] & 0xffffu); + if (op == spv::OpCapability && words[at + 1] == spv::CapabilityFragmentShaderPixelInterlockEXT) shape += "capability "; + if (op == spv::OpExtension && std::string_view(reinterpret_cast(&words[at + 1])) == "SPV_EXT_fragment_shader_interlock") shape += "extension "; + if (op == spv::OpExecutionMode && words[at + 2] == spv::ExecutionModePixelInterlockOrderedEXT) shape += "ordered "; + if (op == spv::OpBeginInvocationInterlockEXT) shape += "begin "; + if (op == spv::OpEndInvocationInterlockEXT) shape += "end "; + } + return shape; +} + +std::string interlockExits(std::span words, std::size_t& kills) { + struct Block { + std::vector ops; + std::vector next; + }; + std::map blocks; + std::uint32_t entry = 0, function = 0, label = 0, first = 0; + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + const auto op = static_cast(words[at] & 0xffffu); + const auto end = at + (words[at] >> 16u); + if (op == spv::OpEntryPoint) entry = words[at + 2]; + if (op == spv::OpFunction) { + function = words[at + 2]; + label = 0; + } + if (op == spv::OpLabel) label = words[at + 1]; + if (op == spv::OpLabel && function == entry && first == 0) first = label; + if (function != entry) { + if (op == spv::OpBeginInvocationInterlockEXT || op == spv::OpEndInvocationInterlockEXT || op == spv::OpKill) return "an interlock instruction or OpKill outside the entry point"; + continue; + } + if (label == 0) continue; + auto& block = blocks[label]; + block.ops.push_back(op); + if (op == spv::OpBranch) block.next = {words[at + 1]}; + if (op == spv::OpBranchConditional) block.next = {words[at + 2], words[at + 3]}; + if (op == spv::OpSwitch) { + block.next = {words[at + 2]}; + for (auto target = at + 4; target < end; target += 2) block.next.push_back(words[target]); + } + } + std::map state{{first, 0}}; + std::vector work{first}; + while (!work.empty()) { + const auto current = work.back(); + work.pop_back(); + auto stage = state[current]; + for (const auto op : blocks[current].ops) { + if (op == spv::OpBeginInvocationInterlockEXT && stage++ != 0) return "block " + std::to_string(current) + " begins the interlock twice"; + if (op == spv::OpEndInvocationInterlockEXT && stage++ != 1) return "block " + std::to_string(current) + " ends the interlock outside it"; + if ((op == spv::OpKill || op == spv::OpReturn || op == spv::OpReturnValue || op == spv::OpUnreachable || op == spv::OpTerminateInvocation) && stage != 2) return "an exit in block " + std::to_string(current) + " skips OpEndInvocationInterlockEXT"; + if (op == spv::OpKill) ++kills; + } + for (const auto next : blocks[current].next) { + const auto [known, added] = state.emplace(next, stage); + if (!added && known->second != stage) return "block " + std::to_string(next) + " is reached both inside and outside the interlock"; + if (added) work.push_back(next); + } + } + return {}; +} + +void orderedPixelShaderTests() { + const std::array interlock{spv::CapabilityShader, spv::CapabilityFragmentShaderPixelInterlockEXT}; + const std::array plain{spv::CapabilityShader}; + for (const auto shaderControl : {0x30640u, 0x30600u}) { + const auto queue = orderedPixelQueue(shaderControl); + static_cast(AgcDriver::Graphics::DecodeState(queue)); + Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "the precheck rejected DB_SHADER_CONTROL " + std::to_string(shaderControl)); + Require(AgcDriver::Graphics::DecodePixelStageInfo(queue.context, IdentityExports).orderedPixelShader, "PRIMITIVE_ORDERED_PIXEL_SHADER did not reach the pixel stage"); + } + for (const auto shaderControl : {0x130600u, 0x430600u}) { + const auto queue = orderedPixelQueue(shaderControl); + expectFailure([&] { static_cast(AgcDriver::Graphics::DecodeState(queue)); }, "ordered fragment execution"); + Require(!AgcDriver::Graphics::DrawRejection(queue, false).empty(), "the precheck accepted POPS_OVERLAP_NUM_SAMPLES in DB_SHADER_CONTROL " + std::to_string(shaderControl)); + } + const ShaderRecompiler::RequestSerializer serializer; + std::set> keys; + for (const auto& [shaderControl, expected] : std::array, 2>{{{0x30600u, "capability extension ordered begin end "}, {0x20600u, ""}}}) { + const auto request = orderedPixelRequest(shaderControl, interlock); + const auto replayed = serializer.Deserialize(serializer.Serialize(request)); + Require(replayed.request.context.pixel->orderedPixelShader == (shaderControl == 0x30600u), "the serialized request lost PRIMITIVE_ORDERED_PIXEL_SHADER"); + const auto result = ShaderRecompiler::Recompile(request); + Require(interlockShape(result.spirv.Words()) == expected && interlockShape(ShaderRecompiler::Recompile(replayed.request).spirv.Words()) == expected, "DB_SHADER_CONTROL " + std::to_string(shaderControl) + " declared the wrong pixel interlock: " + interlockShape(result.spirv.Words())); + std::vector key; + ShaderRecompiler::RecompileCacheKey::Build(request, key); + keys.insert(key); + AgcDriver::Graphics::State state{}; + state.stages.path = AgcDriver::Graphics::ShaderPath::Vertex; + ShaderRecompiler::RecompileResult vertex; + vertex.spirv = makeModule({}); + const std::array shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &result, 0}}}; + AgcDriver::Graphics::ValidateShaders(shaders, state, VkPhysicalDeviceSubgroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}, false); + } + Require(keys.size() == 2, "the recompile cache key ignores PRIMITIVE_ORDERED_PIXEL_SHADER"); + const std::array bdaInterlock{spv::CapabilityShader, spv::CapabilityFragmentShaderPixelInterlockEXT, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; + const std::array bdaExtensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; + auto faulting = orderedPixelRequest(0x30600u, bdaInterlock, OrderedBdaPixelCode); + faulting.target.supportedExtensions = bdaExtensions; + faulting.target.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version; + std::size_t kills = 0; + const auto exits = interlockExits(ShaderRecompiler::Recompile(faulting).spirv.Words(), kills); + Require(exits.empty() && kills == 6u, "a primitive-ordered pixel shader with BDA reads: " + (exits.empty() ? std::to_string(kills) + " OpKill, not the valid-mask kill and 5 BDA fault kills" : exits)); + expectFailure([&] { static_cast(ShaderRecompiler::Recompile(orderedPixelRequest(0x30600u, plain))); }, "fragmentShaderPixelInterlock"); +} + +void ConservativeRasterizationTests() { + auto queue = makeState(); + queue.context[0x1b3] = 2; + queue.context[0x1b4] = 2; + Require(AgcDriver::Graphics::DecodeState(queue).conservativeRasterization == VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT, "PA_SC_CONSERVATIVE_RASTERIZATION_CNTL 0x6000 enabled conservative rasterization"); + queue.context[0x313] = 0x6001; + for (const auto primitive : {4u, 5u, 6u}) { + queue.userConfig[0x242] = primitive; + Require(AgcDriver::Graphics::DecodeState(queue).conservativeRasterization == VK_CONSERVATIVE_RASTERIZATION_MODE_OVERESTIMATE_EXT, "OVER_RAST_ENABLE did not decode to overestimation for primitive type " + std::to_string(primitive)); + Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "the precheck rejected OVER_RAST_ENABLE"); + } + for (const auto& [primitive, rejected] : std::array, 3>{{{1u, "conservative rasterization of points is unsupported (VGT_PRIMITIVE_TYPE=0x1)"}, {2u, "conservative rasterization of lines is unsupported (VGT_PRIMITIVE_TYPE=0x2)"}, {17u, "conservative rasterization of rectangles is unsupported (VGT_PRIMITIVE_TYPE=0x11)"}}}) { + queue.userConfig[0x242] = primitive; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, rejected); + } + queue.userConfig[0x242] = 4; + auto geometry = queue; + geometry.context[0x2d5] = 0x2020; + geometry.userConfig[0x25b] = (64u << 9u) | 21u; + geometry.context[0x1ff] = 64; + geometry.context[0x2ce] = 3; + geometry.context[0x29b] = 2; + geometry.context[0x2ab] = 4; + geometry.shader[0x8a] = 3u << 29u; + geometry.shader[0x8b] = 3u << 16u; + for (const auto input : {1u, 2u, 4u}) { + geometry.userConfig[0x242] = input; + const auto state = AgcDriver::Graphics::DecodeState(geometry); + Require(state.stages.path == AgcDriver::Graphics::ShaderPath::Geometry && state.conservativeRasterization == VK_CONSERVATIVE_RASTERIZATION_MODE_OVERESTIMATE_EXT, "OVER_RAST_ENABLE did not overestimate the triangle strips of a geometry shader with input primitive type " + std::to_string(input)); + } + for (const auto& [output, rejected] : std::array, 3>{{{0u, "conservative rasterization of points is unsupported (VGT_GS_OUT_PRIM_TYPE=0x0)"}, {1u, "conservative rasterization of lines is unsupported (VGT_GS_OUT_PRIM_TYPE=0x1)"}, {0x80000002u, "conservative rasterization of per-stream primitive types is unsupported (VGT_GS_OUT_PRIM_TYPE=0x80000002)"}}}) { + geometry.context[0x29b] = output; + expectFailure([&] { AgcDriver::Graphics::DecodeState(geometry); }, rejected); + } + auto tessellation = queue; + tessellation.context[0x2d5] = 0x200d; + tessellation.userConfig[0x242] = 9; + tessellation.context[0x2d6] = (3u << 8u) | (3u << 14u); + tessellation.context[0x2db] = 1u | (2u << 2u) | (2u << 5u); + const auto tessellated = AgcDriver::Graphics::DecodeState(tessellation); + Require(tessellated.stages.path == AgcDriver::Graphics::ShaderPath::Tessellation && tessellated.conservativeRasterization == VK_CONSERVATIVE_RASTERIZATION_MODE_OVERESTIMATE_EXT, "OVER_RAST_ENABLE did not overestimate clockwise tessellated triangles"); + for (const auto& [parameters, rejected] : std::array, 3>{{{1u | (2u << 2u), "conservative rasterization of points is unsupported (VGT_TF_PARAM=0x9)"}, {1u | (2u << 2u) | (1u << 5u), "conservative rasterization of lines is unsupported (VGT_TF_PARAM=0x29)"}, {(2u << 2u) | (1u << 5u), "conservative rasterization of lines is unsupported (VGT_TF_PARAM=0x28)"}}}) { + tessellation.context[0x2db] = parameters; + expectFailure([&] { AgcDriver::Graphics::DecodeState(tessellation); }, rejected); + } + tessellation.context[0x2d5] = 0x202d; + tessellation.context[0x2db] = 1u | (2u << 2u) | (2u << 5u); + tessellation.context[0x29b] = 1; + expectFailure([&] { AgcDriver::Graphics::DecodeState(tessellation); }, "conservative rasterization of lines is unsupported (VGT_GS_OUT_PRIM_TYPE=0x1)"); + for (const auto inputs : {0x6u, 0x42u}) { + queue.context[0x1b3] = inputs; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "conservative rasterization with centroid interpolation"); + } + queue.context[0x1b3] = 2; + for (const auto control : {0x6003u, 0x6020u, 0x6401u, 0xe06001u, 0x1u}) { + queue.context[0x313] = control; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "PA_SC_CONSERVATIVE_RASTERIZATION_CNTL=0x"); + Require(AgcDriver::Graphics::DrawRejection(queue, false).find("PA_SC_CONSERVATIVE_RASTERIZATION_CNTL=0x") != std::string::npos, "the precheck accepted PA_SC_CONSERVATIVE_RASTERIZATION_CNTL " + std::to_string(control)); + } +} + void pixelParameterSlotTests() { using AgcDriver::Graphics::CompiledShader; const VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}; @@ -1928,6 +2591,21 @@ void validationTests() { subgroup.supportedOperations = capability == spv::CapabilityGroupNonUniform ? 0u : VK_SUBGROUP_FEATURE_BASIC_BIT; expectFailure([&] { AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false); }, "device lacks operations"); } + { + ShaderRecompiler::RecompileResult vertex; + vertex.spirv = makeModule({}); + vertex.spirv.insert(vertex.spirv.begin() + 5, {(2u << 16u) | spv::OpCapability, static_cast(spv::CapabilityGroupNonUniformArithmetic)}); + const std::array shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}}}; + VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}; + subgroup.supportedStages = VK_SHADER_STAGE_FRAGMENT_BIT; + subgroup.supportedOperations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_ARITHMETIC_BIT; + expectFailure([&] { AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false); }, "unsupported for shader stage"); + subgroup.supportedStages = VK_SHADER_STAGE_VERTEX_BIT; + subgroup.supportedOperations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_BALLOT_BIT | VK_SUBGROUP_FEATURE_SHUFFLE_BIT; + expectFailure([&] { AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false); }, "device lacks operations"); + subgroup.supportedOperations |= VK_SUBGROUP_FEATURE_ARITHMETIC_BIT; + AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false); + } const std::vector words(8, 0); const auto validate = [&](const ShaderRecompiler::RecompileResult& vertex, std::uint32_t fragmentOffset) { const std::array shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, fragmentOffset}}}; @@ -2027,6 +2705,16 @@ bool recompilesDebugBranch(std::uint32_t opcode) { return !ShaderRecompiler::Recompile(request).spirv.Words().empty(); } +void meshIndexBufferTests() { + const AgcDriver::Pm4::DrawParameters automatic{0, 96, 0, 1, 0, false}; + const auto unindexed = AgcDriver::Graphics::MeshIndexBufferDescriptor(automatic); + Require(unindexed[0] == 0 && unindexed[1] == 0 && unindexed[2] == 0 && unindexed[3] == 0x31016facu, "a non-indexed mesh draw did not get a null index buffer V#"); + const AgcDriver::Pm4::DrawParameters indexed{0x123456789a00ull, 5, 2, 1, 0, true}; + const auto words = AgcDriver::Graphics::MeshIndexBufferDescriptor(indexed); + Require(words[0] == 0x56789a00u && words[1] == 0x1234u && words[2] == 12u && words[3] == 0x31016facu, "an indexed mesh draw's index buffer V# changed"); + expectFailure([] { static_cast(AgcDriver::Graphics::MeshIndexBufferDescriptor(AgcDriver::Pm4::DrawParameters{0, 3, 2, 1, 0, true})); }, "invalid mesh index buffer range"); +} + void meshArgumentTests() { using AgcDriver::Graphics::MeshArguments; using AgcDriver::Graphics::ResolveMeshArguments; @@ -2103,8 +2791,10 @@ void vertexCopyTests() { int main() { #ifdef _WIN32 _putenv_s("APS5_PIN_WAIT_MS", "200"); + _putenv_s("APS5_HEAP_MIRROR_MIB", "4"); #else setenv("APS5_PIN_WAIT_MS", "200", 1); + setenv("APS5_HEAP_MIRROR_MIB", "4", 1); #endif try { { @@ -2137,24 +2827,38 @@ int main() { unrestricted.depthRangeUnrestricted = true; AgcDriver::Graphics::ValidateDepthBounds(unrestricted, bounded); } + RunGuestLeaseWaitTests(); stateTests(); + depthMaintenanceTests(); hardwareScreenOffsetTests(); + srgb8TargetTests(); DepthClipTests(); DepthStencilTests(); ZExportTests(); DepthBoundsBiasTests(); conservativeZExportTests(); + orderedPixelShaderTests(); + ConservativeRasterizationTests(); DisabledColorTests(); CompactedExportTests(); + ReversedComponentOrderTests(); metadataPassTests(); + cmaskTests(); ShaderStageTests(); + TuningFieldTests(); PixelInputLayoutTests(); + ComputeScratchTests(); InitialContextTests(); pushConstantTests(); resourceTests(); + descriptorCacheTests(); misalignedShaderDataTests(); debugBranchTests(); + textureCacheBudgetTests(); + sampledTextureBudgetTests(); + sampledBudgetReportTests(); meshArgumentTests(); + meshIndexBufferTests(); validationTests(); vertexCopyTests(); pixelParameterSlotTests(); @@ -2175,7 +2879,11 @@ int main() { const auto offset = address - 0x100000000000ULL; const auto buffer = reinterpret_cast(offset / 0x10000); return std::span(mock.memories.at(mock.bufferMemory.at(buffer))).subspan(offset % 0x10000); - } + }, + [](std::optional headroom) { + mock.memoryLimit = headroom.has_value() ? std::optional(mock.allocatedBytes + *headroom) : std::nullopt; + }, + [] { return mock.allocationAttempts; } }); Require(mock.live == 0, "BDA resources leaked Vulkan objects"); RunGuestAllocationTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp index bb338d02e..b016d793e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp @@ -4,6 +4,9 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include +#include +#include +#include #include #include #include @@ -26,6 +29,49 @@ void reject(TAction action) { } +void RunGuestLeaseWaitTests() { + std::array memory{}; + const auto address = reinterpret_cast(memory.data()); + { + GuestAllocations::Mutation mutation; + mutation.Add(memory.data(), 64, true, true); + mutation.Add(memory.data() + 64, 64, true, true); + } + auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + std::erase_if(lease, [&](const auto& range) { return range->address != address; }); + std::atomic mutationEntered = false; + std::atomic lookupFinished = false; + std::exception_ptr failure; + bool unmapped = false; + std::jthread mutator([&] { + try { + GuestAllocations::Mutation mutation; + mutationEntered.store(true); + mutationEntered.notify_one(); + mutation.Unmap(memory.data(), 64, [&](const void*, std::size_t, const void*, bool) { + Require(lookupFinished.load(), "the allocation was unmapped before its lease holder finished"); + unmapped = true; + }); + } catch (...) { failure = std::current_exception(); } + }); + mutationEntered.wait(false); + auto other = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + std::erase_if(other, [&](const auto& range) { return range->address != address + 64; }); + const bool found = other.size() == 1 && other.front()->address == address + 64; + lookupFinished.store(true); + lease.clear(); + mutator.join(); + other.clear(); + { + GuestAllocations::Mutation mutation; + if (!unmapped) mutation.Remove(memory.data()); + mutation.Remove(memory.data() + 64); + } + Require(found, "a lease holder could not query another allocation during an unmap wait"); + if (failure) std::rethrow_exception(failure); + Require(unmapped, "waiting unmap did not resume after the lease was released"); +} + void RunGuestAllocationTests() { void* pointer = GuestHeap::GuestHeapAllocate_nid_postfix(32); Require(reinterpret_cast(pointer) % alignof(std::max_align_t) == 0, "guest malloc is not suitably aligned"); diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp index f8023b972..44b30348e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp @@ -1,10 +1,13 @@ #include "GraphicsTests.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include #include #include #include #include +#include namespace { @@ -192,6 +195,15 @@ void RunGuestTextureResourceTests() { cube.baseArray = 0; result = DecodeTextureResource(pack(cube)); Require(result.dimension == TextureDimension::kCube, "cube descriptor did not decode to cube dimension"); + for (const auto first : {0u, 5u, 6u}) { + auto singleCube = cube; + singleCube.baseArray = first; + singleCube.depth = first; + result = DecodeTextureResource(pack(singleCube)); + Require(result.baseArray == first && result.depthOrLastArray == first + 5u, "a single cube descriptor did not expose six faces"); + const auto geometry = DescribeSurface(result); + Require(geometry.imageLayers == first + 6u, "a single cube allocation omitted faces"); + } Fields cubeNotSquare = cube; cubeNotSquare.height = 16; rejectFields(cubeNotSquare, "not square"); @@ -292,6 +304,28 @@ void RunGuestTextureResourceTests() { const auto compressed = DecodeTextureResource(pack(meta)); Require(compressed.dccAddress == 0x100 && compressed.dccAlphaOnMsb, "DCC metadata was decoded wrongly"); + Fields storage = base; + storage.base40 = 0x1000; + storage.width = 1920; + storage.height = 1080; + storage.tileModeRaw = 27; + storage.metaCompress = true; + constexpr std::uint64_t storageBytes = 15u * 9u * 65536u; + constexpr std::size_t extentKeys = 49152; + std::vector storageKeys(extentKeys + 256); + const auto keysAddress = (reinterpret_cast(storageKeys.data()) + 255u) / 256u * 256u; + const auto* keys = reinterpret_cast(keysAddress); + storage.metaAddr = keysAddress >> 8u; + for (const bool pipeAligned : {true, false}) { + storage.metaPipeAligned = pipeAligned; + const auto image = DecodeTextureResource(pack(storage)); + const auto expected = pipeAligned ? extentKeys : static_cast(storageBytes / 256u); + std::fill(storageKeys.begin(), storageKeys.end(), std::uint8_t{0x00}); + MarkDccUncompressed(image.dccAddress, storageBytes, DccKeyCount(image, storageBytes)); + const auto stored = static_cast(std::count(keys, keys + extentKeys, std::uint8_t{0xff})); + Require(image.dccPipeAligned == pipeAligned && stored == expected && std::all_of(keys, keys + expected, [](std::uint8_t key) { return key == 0xff; }), std::string("a 1920x1080 SW_64KB_R_X storage image with ") + (pipeAligned ? "pipe-aligned" : "unaligned") + " DCC stored " + std::to_string(stored) + " uncompressed keys, expected " + std::to_string(expected)); + } + Fields badSwizzle = base; badSwizzle.bcSwizzle = 1; rejectFields(badSwizzle, "BC swizzle"); diff --git a/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp b/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp index 025817a38..d4ce2560d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "Recompiler.hpp" +#include "execution/VulkanTestDevice.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include #include @@ -7,6 +8,7 @@ #include #include #include +#include #include #include @@ -122,7 +124,7 @@ struct MeshDraw { void DrawMesh(AgcDriver::VulkanDevice& device, const MeshDraw& setup) { const auto target = device.Target(); std::vector userData(12, 0u); - const auto index = AgcDriver::Graphics::MeshIndexBufferDescriptor(setup.draw, reinterpret_cast(GeometryCode.data())); + const auto index = AgcDriver::Graphics::MeshIndexBufferDescriptor(setup.draw); std::copy(index.begin(), index.end(), userData.begin() + ShaderRecompiler::MeshIndexBufferUserWord); std::copy(setup.vertexBuffer.begin(), setup.vertexBuffer.end(), userData.begin() + 8); const std::array geometryMemory{{{reinterpret_cast(GeometryCode.data()), std::as_bytes(std::span(GeometryCode))}}}; @@ -206,7 +208,11 @@ void CheckTriangles(const char* what) { int main() { try { - setenv("APS5_NO_SHADER_DISK_CACHE", "1", 1); +#ifdef _WIN32 + _putenv_s("ANYPS5_NO_SHADER_CACHE", "1"); +#else + setenv("ANYPS5_NO_SHADER_CACHE", "1", 1); +#endif for (std::uint32_t triangle = 0; triangle < Triangles; ++triangle) { const auto vertices = TriangleVertices(triangle); for (std::uint32_t k = 0; k < 3; ++k) { @@ -235,11 +241,17 @@ int main() { } FanRestart[FanRim] = 0xffffu; - AgcDriver::VulkanDevice device; + const auto testDevice = OpenVulkanTestDevice(); + if (!testDevice) return VulkanTestSkipped; + auto& device = *testDevice; const auto target = device.Target(); if (!target.mesh.has_value()) { - std::puts("Mesh tests skipped: the device has no VK_EXT_mesh_shader"); - return 0; + std::puts("skipped, the device has no VK_EXT_mesh_shader"); + return VulkanTestSkipped; + } + if (device.DeviceName().starts_with("llvmpipe")) { + std::puts("skipped, the device has no mesh shader compiler that handles these programs (llvmpipe)"); + return VulkanTestSkipped; } for (const auto& [name, subgroup] : {std::pair{"one-wave subgroups", SmallSubgroup}, std::pair{"two-wave subgroups", WideSubgroup}}) { @@ -253,8 +265,7 @@ int main() { } constexpr std::size_t recordBlockBytes = 65536; - auto* record = static_cast(std::aligned_alloc(65536, recordBlockBytes)); - Require(record != nullptr, "cannot allocate the indirect record block"); + auto* record = static_cast(::operator new(recordBlockBytes, std::align_val_t{65536})); { GuestAllocations::Mutation mutation; mutation.Add(record, recordBlockBytes, true, true); diff --git a/core/libs/prx/libSceAgcDriver/tests/NarrowConstantStoreFixture.hpp b/core/libs/prx/libSceAgcDriver/tests/NarrowConstantStoreFixture.hpp new file mode 100644 index 000000000..265dff327 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/NarrowConstantStoreFixture.hpp @@ -0,0 +1,28 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_NARROWCONSTANTSTOREFIXTURE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_NARROWCONSTANTSTOREFIXTURE_HPP + +#include +#include + +namespace NarrowConstantStoreFixture { + +constexpr std::uint32_t Threads = 8; +constexpr std::uint32_t LaneBytes = 16; +constexpr std::uint8_t Fill = 0xcd; + +alignas(256) constexpr std::array Code{ + 0x34020084, 0x7e040280, 0x7e0602ff, 0x00a10200, 0x7e0802ff, 0x0000fec3, + 0xdc608000, 0x00000201, + 0xdc688002, 0x00000401, + 0xdc708008, 0x00000301, + 0xbf810000, +}; + +constexpr std::array ExpectedLane{ + 0x00, Fill, 0xc3, 0xfe, Fill, Fill, Fill, Fill, + 0x00, 0x02, 0xa1, 0x00, Fill, Fill, Fill, Fill, +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp index 86640969c..9ef389177 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp @@ -317,6 +317,17 @@ void testCopies() { check(destination[0] == 11 && destination[1] == 12 && destination[2] == 0, "64-bit COPY_DATA failed"); execute(state, makePacket(0x40, {0x105, 0x12345678, 0, low(destination.data()), high(destination.data())})); check(destination[0] == 0x12345678, "immediate COPY_DATA failed"); + alignas(8) std::array clock{}; + const auto clockCopy = [&](std::uint64_t* target) { return makePacket(0x40, {0x06016209, 0, 0, low(target), high(target)}); }; + execute(state, clockCopy(&clock[0])); + execute(state, clockCopy(&clock[1])); + check(clock[0] != 0 && clock[1] >= clock[0], "GPU clock COPY_DATA failed"); + alignas(8) std::array clock32{0, 0xdeadbeef}; + execute(state, makePacket(0x40, {0x06006209, 0, 0, low(clock32.data()), high(clock32.data())})); + check(clock32[0] != 0 && clock32[1] == 0xdeadbeef, "32-bit GPU clock COPY_DATA did not write only the low half"); + const auto clockStore =AgcDriver::Pm4::ResolveStore(clockCopy(&clock[0]), state, 64); + check(clockStore.has_value() && clockStore->Bytes().size() == 8, "GPU clock COPY_DATA did not resolve as an 8-byte store"); + expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x40, {0x1020a, 0, 0, low(&clock[0]), high(&clock[0])}), 0); }, "reference-clock"); execute(state, makePacket(0x50, {0x60000000, low(source.data()), high(source.data()), low(destination.data()), high(destination.data()), 16})); check(source == destination, "DMA_DATA copy failed"); execute(state, makePacket(0x50, {0x40000000, 0x44332211, 0, low(destination.data()), high(destination.data()), 6})); @@ -677,6 +688,21 @@ void submitWords(std::vector& words, std::uint32_t queue = 0) { check((queue == 0 ? sceAgcDriverSubmitDcb(&packet) : sceAgcDriverSubmitAcb(queue, &packet)) == 0, "conditional submission failed"); } +void testRegisterListsReadAtSubmission() { + alignas(8) static std::uint32_t gate = 0; + static std::uint32_t done = 0; + static std::array registers{0x10, 74}; + auto words = joinPackets({ + makePacket(0x3c, {0x13, low(&gate), high(&gate), 1, 0xffffffffu, 0x19}), + makePacket(0x9f, {low(registers.data()), high(registers.data()), 0x80000000, 1}), + writeWord(done, 1)}); + submitWords(words); + registers[0] = 0x3a888889; + std::atomic_ref(gate).store(1); + AgcDriverWaitIdle_nid_postfix(); + check(std::atomic_ref(done).load() == 1, "register list rewritten after submission was read by the worker"); +} + void testConditionalSubmission() { alignas(8) static std::uint32_t zero = 0, one = 1, condition = 0; static std::array results{}; @@ -946,6 +972,7 @@ int main(int argc, char** argv) { testPredication(); testUnwrittenUserData(); testDriverSubmission(); + testRegisterListsReadAtSubmission(); testPredicatedSubmission(); testConditionalSubmission(); testBranchSubmission(); diff --git a/core/libs/prx/libSceAgcDriver/tests/ProfileOutput.cpp b/core/libs/prx/libSceAgcDriver/tests/ProfileOutput.cpp index 5ac7e3ee0..72fb292b4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ProfileOutput.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ProfileOutput.cpp @@ -104,6 +104,20 @@ void WriterFailure() { } } +void ExplicitStop() { + std::string written; + AgcDriver::ProfileOutput output([&](std::string_view text) { written += text; }); + output.Write("pending output"); + output.Stop(); + output.Stop(); + output.Flush(); + Check(written == "pending output", "explicit stop did not drain pending output"); + bool failed = false; + try { output.Write("after stop"); } + catch (const std::runtime_error& error) { failed = std::string(error.what()) == "profile output is stopping"; } + Check(failed, "write after stop was accepted"); +} + } int main() { @@ -112,6 +126,7 @@ int main() { SlowWriter(); ConcurrentWriters(); WriterFailure(); + ExplicitStop(); std::cout << "profile output tests passed\n"; return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index c452cd30d..a8c50a234 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -16,6 +16,7 @@ #include "prx/libc/include/GuestWriteWatch.hpp" #include "ResidentPresent.hpp" #include "SampleLod_spv.h" +#include "SampleArray_spv.h" #include #ifdef _WIN32 #ifndef NOMINMAX @@ -98,7 +99,26 @@ public: Require(count != 0, "no Vulkan device"); std::vector devices(count); Check(enumerate(instance, &count, devices.data()), "vkEnumeratePhysicalDevices"); - context.physical = devices.front(); + const auto rankDeviceType = [](VkPhysicalDeviceType type) { + switch (type) { + case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return 3; + case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return 2; + case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return 1; + default: return 0; + } + }; + const auto physicalProperties = function("vkGetPhysicalDeviceProperties"); + int selectedRank = -1; + for (const auto physical : devices) { + VkPhysicalDeviceProperties candidate{}; + physicalProperties(physical, &candidate); + if (candidate.apiVersion < VK_API_VERSION_1_1) continue; + const int rank = rankDeviceType(candidate.deviceType); + if (rank <= selectedRank) continue; + context.physical = physical; + selectedRank = rank; + } + Require(context.physical != VK_NULL_HANDLE, "no Vulkan 1.1 device"); const auto extensions = function("vkEnumerateDeviceExtensionProperties"); Check(extensions(context.physical, nullptr, &count, nullptr), "vkEnumerateDeviceExtensionProperties"); std::vector available(count); @@ -127,14 +147,15 @@ public: enabled.shaderInt64 = VK_TRUE; address.pNext = &bytes; std::vector extensionsEnabled{VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, VK_KHR_8BIT_STORAGE_EXTENSION_NAME}; + VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; if (hasExtension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME)) { extensionsEnabled.push_back(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME); - VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT}; + VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT, &driverProperties}; VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &hostProperties}; function("vkGetPhysicalDeviceProperties2")(context.physical, &properties); context.hostImportAlignment = hostProperties.minImportedHostPointerAlignment; } - if (hasExtension(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME) && hasExtension(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME)) { + if (driverProperties.driverID != VK_DRIVER_ID_NVIDIA_PROPRIETARY && hasExtension(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME) && hasExtension(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME)) { extensionsEnabled.push_back(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME); extensionsEnabled.push_back(VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME); context.dmaBufImport = true; @@ -168,6 +189,7 @@ public: context.formatProperties = function("vkGetPhysicalDeviceFormatProperties"); context.Function("vkGetDeviceQueue")(context.device, family, 0, &context.queue); VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; pool.queueFamilyIndex = family; Check(context.Function("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); } catch (...) { @@ -191,7 +213,10 @@ private: void release() noexcept { if (context.pool != VK_NULL_HANDLE) context.Function("vkDestroyCommandPool")(context.device, context.pool, nullptr); context.bufferPool.reset(); - if (context.device != VK_NULL_HANDLE) function("vkDestroyDevice")(context.device, nullptr); + if (context.device != VK_NULL_HANDLE) { + DestroyShadows(context.device); + function("vkDestroyDevice")(context.device, nullptr); + } if (instance != VK_NULL_HANDLE) function("vkDestroyInstance")(instance, nullptr); if (library != nullptr) SDL_UnloadObject(library); } @@ -1147,6 +1172,10 @@ void drawSnapshotReuseTests(const Device& device, Recorder& recorder) { std::cout << "host import of the watched block refused: draw snapshot reuse not tested\n"; return; } + if (!AgcDriver::GuestMemory::Watched(address, bytes)) { + std::cout << "host imports are compared, not watched: draw snapshot reuse not tested\n"; + return; + } const auto element = address + 4096; constexpr std::size_t elementBytes = 1024; ShaderRecompiler::RecompileResult program; @@ -1663,8 +1692,8 @@ void importWatchTests(const Device& device) { using namespace AgcDriver::GuestMemory; const auto& context = device.GetContext(); #ifdef _WIN32 - static_cast(context); - std::cout << "import watch decisions: Linux write watch only\n"; + Require(PrepareImportWatch(context) == ImportWatch::Unwatch, "Windows host imports stayed watched by default"); + std::cout << "import watch decisions: Windows host imports use comparisons\n"; #else if (context.hostImportAlignment == 0 || !WriteWatched()) { std::cout << "host imports or write watching unavailable: import watch decisions not tested\n"; @@ -2036,6 +2065,10 @@ void importWindowTests(const Device& device, Recorder& recorder) { Require(probe.failure == nullptr, "(w) the import probe failed"); const bool importWrites = probe.writtenAtImport != 0; const auto decided = PrepareImportWatch(base); + if (decided == ImportWatch::Unwatch) { + std::cout << "host imports are compared, not watched: the import window not tested\n"; + return; + } SetImportWatch(base, ImportWatch::Watch); struct Restore { const Context& context; @@ -2190,6 +2223,18 @@ void pendingKeyStoreTests(const Device& device, Recorder& recorder) { recorder.NotePendingWrite(address, 2 * keyCount); Require(CurrentDccKeys(address, surfaceBytes) == DccKeys::Uncompressed && !recorder.PendingWriteOverlaps(address, keyCount), "a later wider writer over a pending fill was not waited for"); recorder.Sync(); + constexpr std::uint64_t displayBytes = 15u * 9u * 65536u; + constexpr std::size_t consoleKeys = 49152; + const auto displayKeys = DccKeyCount(TextureTileMode::kR64KBX, 4, 1920, 1080, displayBytes); + std::memset(keys, 0x00, consoleKeys); + MarkDccUncompressed(context, address, displayBytes, displayKeys); + Require(recorder.PendingWriteOverlaps(address, consoleKeys) && keys[consoleKeys - 1] == 0x00, "the uncompressed key store over a 1920x1080 surface did not stay pending"); + Require(CurrentDccKeys(address + displayBytes / 256, (consoleKeys - displayBytes / 256) * 256) == DccKeys::Uncompressed && recorder.PendingWriteOverlaps(address, consoleKeys), "the keys past one per 256 bytes of a pending 1920x1080 store did not read as uncompressed without a wait"); + Require(CurrentDccKeys(address, displayBytes, displayKeys) == DccKeys::Uncompressed && recorder.PendingWriteOverlaps(address, consoleKeys), "the keys of a pending 1920x1080 store did not read as uncompressed over the console's extent without a wait"); + recorder.NotePendingWrite(address + 40000, 1); + Require(CurrentDccKeys(address, displayBytes, displayKeys) == DccKeys::Uncompressed && !recorder.PendingWriteOverlaps(address + 40000, 1), "a later writer of key 40000 over a pending 1920x1080 store was not waited for"); + Require(std::all_of(keys, keys + consoleKeys, [](std::uint8_t key) { return key == 0xff; }), "the uncompressed key store over a 1920x1080 surface did not land on all 49152 keys"); + recorder.Sync(); } void sampleDumpTests(const Device& device, Recorder& recorder) { @@ -2306,6 +2351,7 @@ void metadataPassTests(const Device& device, Recorder& recorder) { color.tileMode = ColorTileMode::RenderTarget; color.elementBytes = 4; color.dccAddress = address + surfaceBytes; + color.dccPipeAligned = true; color.clearWords = {0x80402010u, 0}; const ColorMetadataPass pass{ColorMetadataPass::Mode::EliminateFastClear, {color}}; const auto memoryHolds = [&](std::array texel) { @@ -2321,13 +2367,15 @@ void metadataPassTests(const Device& device, Recorder& recorder) { return true; }; const auto keysUncompressed = [&] { return ReadDccKeys(color.dccAddress, surfaceBytes) == DccKeys::Uncompressed; }; + const auto extent = DccKeyCount(TextureTileMode::kR64KBX, 4, side, side, surfaceBytes); std::memset(texels, 0x55, surfaceBytes); - std::memset(keys, 0x20, keyCount); + std::memset(keys, 0x20, extent); RunColorMetadataPass(context, pass); Require(StorageTexture::FindPending(address, surfaceBytes) != nullptr && texels[0] == 0x55, "the register fast clear eliminate did not stay in the resident image"); Require(keysUncompressed(), "the register fast clear eliminate left the keys compressed"); + Require(CurrentDccKeys(color.dccAddress, surfaceBytes, extent) == DccKeys::Uncompressed, "the register fast clear eliminate did not store uncompressed keys over the target's console DCC extent"); Require(memoryHolds({0x10, 0x20, 0x40, 0x80}), "the register fast clear eliminate did not store CB_COLOR_CLEAR_WORD"); - Require(std::all_of(keys, keys + keyCount, [](std::uint8_t key) { return key == 0xff; }), "the stored keys are not uncompressed"); + Require(std::all_of(keys, keys + extent, [](std::uint8_t key) { return key == 0xff; }), "the stored keys are not uncompressed"); std::memset(keys, 0xc0, keyCount); RunColorMetadataPass(context, {ColorMetadataPass::Mode::DccDecompress, {color}}); Require(StorageTexture::FindPending(address, surfaceBytes) != nullptr, "the DCC decompress did not stay in the resident image"); @@ -2424,6 +2472,17 @@ void dataRefreshTests(const Device& device, Recorder& recorder) { Require(resources.RefreshData(recorder.Commands(), original, &recorder), "the refresh back recorded nothing"); Require(resources.DataWordsHash() == ShaderResources::DataWordsHash(original), "the refreshed template's hash is not the original words'"); Require(!resources.RefreshData(recorder.Commands(), original, &recorder), "a refresh with equal words recorded"); + auto replaced = patched; + replaced.bindings[0].role = ShaderRecompiler::DescriptorRole::GuestSamplers; + replaced.bindings[0].kind = ShaderRecompiler::DescriptorKind::Sampler; + const CompiledShader changedResources{ShaderRecompiler::ShaderStage::Compute, &replaced, 0}; + bool rejected = false; + try { + static_cast(resources.RefreshData(recorder.Commands(), changedResources, &recorder)); + } catch (const std::exception& error) { + rejected = std::string_view(error.what()).find("cannot replace bound resources") != std::string_view::npos; + } + Require(rejected && resources.DataWordsHash() == ShaderResources::DataWordsHash(original), "a data refresh changed the bound resources"); recorder.Submit(); device.WaitQueue(); recorder.Sync(); @@ -2433,7 +2492,7 @@ void dataRefreshTests(const Device& device, Recorder& recorder) { class SampleProgram { public: - SampleProgram(const Context& context, Recorder& recorder) : context(context), recorder(recorder), result(context, sizeof(float) * 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) { + SampleProgram(const Context& context, Recorder& recorder, std::span code = SAMPLE_LOD_SPV) : context(context), recorder(recorder), result(context, sizeof(float) * 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) { VkDescriptorSetLayoutBinding bindings[2]{}; bindings[0] = {0, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}; bindings[1] = {1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}; @@ -2441,7 +2500,7 @@ public: layoutInfo.bindingCount = 2; layoutInfo.pBindings = bindings; Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &layoutInfo, nullptr, &setLayout), "vkCreateDescriptorSetLayout"); - const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(float)}; + const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(float) * 2}; VkPipelineLayoutCreateInfo pipelineLayoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; pipelineLayoutInfo.setLayoutCount = 1; pipelineLayoutInfo.pSetLayouts = &setLayout; @@ -2449,8 +2508,8 @@ public: pipelineLayoutInfo.pPushConstantRanges = &push; Check(context.Function("vkCreatePipelineLayout")(context.device, &pipelineLayoutInfo, nullptr, &pipelineLayout), "vkCreatePipelineLayout"); VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; - moduleInfo.codeSize = sizeof(SAMPLE_LOD_SPV); - moduleInfo.pCode = SAMPLE_LOD_SPV; + moduleInfo.codeSize = code.size_bytes(); + moduleInfo.pCode = code.data(); Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &module), "vkCreateShaderModule"); VkComputePipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; pipelineInfo.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_COMPUTE_BIT, module, "main", nullptr}; @@ -2482,7 +2541,7 @@ public: SampleProgram(const SampleProgram&) = delete; SampleProgram& operator=(const SampleProgram&) = delete; - float Red(VkImageView view, VkImageLayout layout, float lod) { + float Red(VkImageView view, VkImageLayout layout, float lod, float layer = 0) { VkDescriptorSetAllocateInfo allocateInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; allocateInfo.descriptorPool = pool; allocateInfo.descriptorSetCount = 1; @@ -2507,7 +2566,8 @@ public: RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &set, 0, nullptr); - context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(lod), &lod); + const float parameters[]{lod, layer}; + context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(parameters), parameters); context.Function("vkCmdDispatch")(commands, 1, 1, 1); RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); recorder.Submit(); @@ -2575,13 +2635,47 @@ void minLodTests(const Device& device, Recorder& recorder) { const auto unorm = [&](std::size_t level) { return levels[level] / 255.0f; }; expectRed(sample(0, 0.0f), unorm(0), "minimum LOD clamp: no clamp reads level 0"); expectRed(sample(0x100, 0.0f), unorm(1), "minimum LOD clamp: MIN_LOD 1 reads level 1 at LOD 0"); - expectRed(sample(0x180, 0.0f), (unorm(1) + unorm(2)) / 2.0f, "minimum LOD clamp: MIN_LOD 1.5 blends levels 1 and 2"); + const auto fractional = sample(0x180, 0.0f); + if (std::abs(fractional - unorm(1)) <= 1.5f / 255.0f) { + std::cout << "minimum LOD clamp: the device takes the integer part of a fractional view minimum LOD\n"; + } else { + expectRed(fractional, (unorm(1) + unorm(2)) / 2.0f, "minimum LOD clamp: MIN_LOD 1.5 blends levels 1 and 2"); + } expectRed(sample(0x100, 2.0f), unorm(2), "minimum LOD clamp: MIN_LOD 1 lowered LOD 2"); expectRed(sample(0xfff, 0.0f), unorm(3), "minimum LOD clamp: MIN_LOD past the last level reads the last level"); expectRed(sample(0x200, 0.0f, 1), unorm(2), "minimum LOD clamp: MIN_LOD 2 over a view from level 1 reads level 2"); expectRed(sample(0x100, 0.0f, 1), unorm(1), "minimum LOD clamp: MIN_LOD at the view's base level reads its base level"); } +void singleCubeTests(const Device& device, Recorder& recorder) { + auto context = device.GetContext(); + TextureDetiler detiler(context); + context.detiler = &detiler; + SampleProgram program(context, recorder, SAMPLE_ARRAY_SPV); + const VkComponentMapping identity{}; + for (const auto first : {0u, 5u, 6u}) { + std::array words{0x1000u, (56u << 20u) | (3u << 30u), 15u | (63u << 14u), 0xb0000facu, first | (first << 16u), 0, 0, 0}; + auto resource = DecodeTextureResource(words); + const auto geometry = DescribeSurface(resource); + std::vector memory(static_cast(geometry.guestBytes) + 256); + auto* surface = reinterpret_cast((reinterpret_cast(memory.data()) + 255) & ~std::uintptr_t{255}); + for (std::uint32_t layer = 0; layer < geometry.imageLayers; ++layer) { + std::memset(surface + geometry.GuestLayerOffset(layer), 16 * (layer + 1), static_cast(geometry.layerBytes)); + } + resource.baseAddress = reinterpret_cast(surface); + const std::span snapshot(surface, static_cast(geometry.guestBytes)); + Texture texture(context, detiler, resource, identity, snapshot); + auto storage = std::make_shared(context, detiler, resource, 0); + recorder.Keep(storage); + Texture storageView(context, storage, resource, identity); + for (std::uint32_t face = 0; face < 6; ++face) { + const auto expected = 16.0f * (first + face + 1) / 255.0f; + expectRed(program.Red(texture.View(), texture.Layout(), 0, face), expected, "single cube snapshot sampled the wrong face"); + expectRed(program.Red(storageView.View(), storageView.Layout(), 0, face), expected, "single cube storage view sampled the wrong face"); + } + } +} + void firstLayerViewTests(const Device& device, Recorder& recorder) { const auto& context = device.GetContext(); TextureDetiler detiler(context); @@ -2780,13 +2874,18 @@ void keysFillTests(const Device& device, Recorder& recorder) { recorder.Sync(); } -int main() { +int main(int argc, char** argv) { try { Device device; std::lock_guard gpu(GpuMutex()); std::cout << "host imports " << (PrepareImportWatch(device.GetContext()) == ImportWatch::Unwatch ? "are compared" : "stay watched") << '\n'; Recorder recorder(device.GetContext()); recorder.Activate(); + if (argc == 2 && std::string_view(argv[1]) == "--cube-only") { + singleCubeTests(device, recorder); + std::cout << "Single cube snapshot and storage sampling tests passed\n"; + return 0; + } readTrackingTests(device, recorder); writeSettledTests(device, recorder); completionCountTests(device, recorder); @@ -2821,6 +2920,7 @@ int main() { dataRefreshTests(device, recorder); minLodTests(device, recorder); firstLayerViewTests(device, recorder); + singleCubeTests(device, recorder); atomicViewTests(device, recorder); metadataPassTests(device, recorder); pendingKeyStoreTests(device, recorder); diff --git a/core/libs/prx/libSceAgcDriver/tests/RecorderKeptBytes.cpp b/core/libs/prx/libSceAgcDriver/tests/RecorderKeptBytes.cpp new file mode 100644 index 000000000..a8d342563 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/RecorderKeptBytes.cpp @@ -0,0 +1,215 @@ +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace AgcDriver::Graphics; + +namespace { + +int failures = 0; + +void Expect(bool condition, const std::string& what) { + if (condition) return; + std::fprintf(stderr, "FAIL: %s\n", what.c_str()); + ++failures; +} + +constexpr std::size_t Budget = Recorder::KeptBytesBudget; + +struct MockDevice { + std::uintptr_t next = 1; + std::map signaled; + std::uint64_t submits = 0; + std::uint64_t fenceWaits = 0; +}; + +MockDevice mock; + +VKAPI_ATTR VkResult VKAPI_CALL mockAllocateCommandBuffers(VkDevice, const VkCommandBufferAllocateInfo*, VkCommandBuffer* commands) { + *commands = reinterpret_cast(mock.next++); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL mockFreeCommandBuffers(VkDevice, VkCommandPool, std::uint32_t, const VkCommandBuffer*) {} + +VKAPI_ATTR VkResult VKAPI_CALL mockCreateFence(VkDevice, const VkFenceCreateInfo*, const VkAllocationCallbacks*, VkFence* fence) { + *fence = reinterpret_cast(mock.next++); + mock.signaled[*fence] = false; + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL mockDestroyFence(VkDevice, VkFence fence, const VkAllocationCallbacks*) { + mock.signaled.erase(fence); +} + +VKAPI_ATTR VkResult VKAPI_CALL mockGetFenceStatus(VkDevice, VkFence fence) { + return mock.signaled.at(fence) ? VK_SUCCESS : VK_NOT_READY; +} + +VKAPI_ATTR VkResult VKAPI_CALL mockWaitForFences(VkDevice, std::uint32_t count, const VkFence* fences, VkBool32, std::uint64_t) { + for (std::uint32_t i = 0; i < count; ++i) mock.signaled.at(fences[i]) = true; + ++mock.fenceWaits; + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL mockResetFences(VkDevice, std::uint32_t count, const VkFence* fences) { + for (std::uint32_t i = 0; i < count; ++i) mock.signaled.at(fences[i]) = false; + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL mockBeginCommandBuffer(VkCommandBuffer, const VkCommandBufferBeginInfo*) { + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL mockEndCommandBuffer(VkCommandBuffer) { + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL mockQueueSubmit(VkQueue, std::uint32_t, const VkSubmitInfo*, VkFence) { + ++mock.submits; + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL mockCmdUpdateBuffer(VkCommandBuffer, VkBuffer, VkDeviceSize, VkDeviceSize, const void*) {} +VKAPI_ATTR void VKAPI_CALL mockCmdPipelineBarrier(VkCommandBuffer, VkPipelineStageFlags, VkPipelineStageFlags, VkDependencyFlags, std::uint32_t, const VkMemoryBarrier*, std::uint32_t, const VkBufferMemoryBarrier*, std::uint32_t, const VkImageMemoryBarrier*) {} +VKAPI_ATTR void VKAPI_CALL mockCmdBeginQuery(VkCommandBuffer, VkQueryPool, std::uint32_t, VkQueryControlFlags) {} +VKAPI_ATTR void VKAPI_CALL mockCmdEndQuery(VkCommandBuffer, VkQueryPool, std::uint32_t) {} +VKAPI_ATTR void VKAPI_CALL mockCmdResetQueryPool(VkCommandBuffer, VkQueryPool, std::uint32_t, std::uint32_t) {} +VKAPI_ATTR void VKAPI_CALL mockCmdCopyQueryPoolResults(VkCommandBuffer, VkQueryPool, std::uint32_t, std::uint32_t, VkBuffer, VkDeviceSize, VkDeviceSize, VkQueryResultFlags) {} +VKAPI_ATTR void VKAPI_CALL mockCmdBindPipeline(VkCommandBuffer, VkPipelineBindPoint, VkPipeline) {} +VKAPI_ATTR void VKAPI_CALL mockCmdPushConstants(VkCommandBuffer, VkPipelineLayout, VkShaderStageFlags, std::uint32_t, std::uint32_t, const void*) {} +VKAPI_ATTR void VKAPI_CALL mockCmdDispatch(VkCommandBuffer, std::uint32_t, std::uint32_t, std::uint32_t) {} + +PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) { + static const std::map table{ + {"vkAllocateCommandBuffers", reinterpret_cast(mockAllocateCommandBuffers)}, + {"vkFreeCommandBuffers", reinterpret_cast(mockFreeCommandBuffers)}, + {"vkCreateFence", reinterpret_cast(mockCreateFence)}, + {"vkDestroyFence", reinterpret_cast(mockDestroyFence)}, + {"vkGetFenceStatus", reinterpret_cast(mockGetFenceStatus)}, + {"vkWaitForFences", reinterpret_cast(mockWaitForFences)}, + {"vkResetFences", reinterpret_cast(mockResetFences)}, + {"vkBeginCommandBuffer", reinterpret_cast(mockBeginCommandBuffer)}, + {"vkEndCommandBuffer", reinterpret_cast(mockEndCommandBuffer)}, + {"vkQueueSubmit", reinterpret_cast(mockQueueSubmit)}, + {"vkCmdUpdateBuffer", reinterpret_cast(mockCmdUpdateBuffer)}, + {"vkCmdPipelineBarrier", reinterpret_cast(mockCmdPipelineBarrier)}, + {"vkCmdBeginQuery", reinterpret_cast(mockCmdBeginQuery)}, + {"vkCmdEndQuery", reinterpret_cast(mockCmdEndQuery)}, + {"vkCmdResetQueryPool", reinterpret_cast(mockCmdResetQueryPool)}, + {"vkCmdCopyQueryPoolResults", reinterpret_cast(mockCmdCopyQueryPoolResults)}, + {"vkCmdBindPipeline", reinterpret_cast(mockCmdBindPipeline)}, + {"vkCmdPushConstants", reinterpret_cast(mockCmdPushConstants)}, + {"vkCmdDispatch", reinterpret_cast(mockCmdDispatch)}, + }; + const auto it = table.find(name); + return it == table.end() ? nullptr : it->second; +} + +Context mockContext() { + Context context{}; + context.device = reinterpret_cast(mock.next++); + context.queue = reinterpret_cast(mock.next++); + context.deviceProc = mockProc; + return context; +} + +void OpenBatchUnderBudget() { + mock = MockDevice{}; + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + Recorder recorder(mockContext()); + recorder.Keep(std::make_shared(0)); + recorder.Keep(std::make_shared(1), Budget - 1); + recorder.BoundKeptBytes(); + Expect(recorder.Recording() && recorder.Submissions() == 0 && mock.submits == 0, "a batch keeping one byte less than the budget was submitted"); + recorder.Keep(std::make_shared(2), 1); + recorder.BoundKeptBytes(); + Expect(!recorder.Recording() && recorder.Submissions() == 1 && mock.submits == 1, "the open batch was not submitted once its kept bytes reached the budget"); + Expect(recorder.InFlightKeptBytes() == Budget, "the submitted batch counts " + std::to_string(recorder.InFlightKeptBytes()) + " kept bytes in flight, not the budget"); + Expect(mock.fenceWaits == 0, "the recorder waited for a batch with one budget in flight"); + recorder.Sync(); + Expect(recorder.InFlightKeptBytes() == 0, "a synced recorder still counts " + std::to_string(recorder.InFlightKeptBytes()) + " kept bytes in flight"); +} + +void InFlightWithinTwiceTheBudget() { + mock = MockDevice{}; + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + Recorder recorder(mockContext()); + std::vector> batches; + for (int batch = 0; batch < 6; ++batch) { + auto object = std::make_shared(batch); + batches.push_back(object); + recorder.Keep(std::move(object), Budget / 2); + recorder.Keep(std::make_shared(batch), Budget / 2); + recorder.BoundKeptBytes(); + Expect(recorder.InFlightKeptBytes() <= 2 * Budget, "batch " + std::to_string(batch) + " left " + std::to_string(recorder.InFlightKeptBytes()) + " kept bytes in flight, more than twice the budget"); + } + Expect(recorder.Submissions() == 6, "six budget batches made " + std::to_string(recorder.Submissions()) + " submissions"); + Expect(recorder.InFlightBatches() == 2 && recorder.InFlightKeptBytes() == 2 * Budget, "the newest two batches are not the ones in flight"); + Expect(mock.fenceWaits == 4, "the recorder waited for " + std::to_string(mock.fenceWaits) + " batches, not the four oldest"); + for (std::size_t batch = 0; batch < batches.size(); ++batch) { + const bool released = batches[batch].expired(); + Expect(released == (batch < 4), "batch " + std::to_string(batch) + (released ? " released its kept objects while in flight" : " still holds its kept objects under the GPU mutex after the wait for it")); + } +} + +void SyncedWithinTwiceTheBudget() { + mock = MockDevice{}; + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + Recorder recorder(mockContext()); + std::vector> batches; + const auto alive = [&] { return std::count_if(batches.begin(), batches.end(), [](const auto& batch) { return !batch.expired(); }); }; + for (int batch = 0; batch < 6; ++batch) { + auto object = std::make_shared(batch); + batches.push_back(object); + recorder.Keep(std::move(object), Budget); + recorder.BoundKeptBytes(); + recorder.Sync(); + Expect(alive() <= 2, "after batch " + std::to_string(batch) + ", " + std::to_string(alive()) + " synced budget batches still hold their kept objects under the GPU mutex"); + } + Expect(recorder.Submissions() == 6 && recorder.InFlightKeptBytes() == 0, "six synced budget batches made " + std::to_string(recorder.Submissions()) + " submissions and left " + std::to_string(recorder.InFlightKeptBytes()) + " kept bytes in flight"); + Expect(!batches[4].expired() && !batches[5].expired(), "the newest two synced batches released their kept objects before the unlock"); +} + +void CountFollowsEveryPath() { + mock = MockDevice{}; + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + Recorder recorder(mockContext()); + recorder.Keep(std::make_shared(0), 3 * Budget); + recorder.BoundKeptBytes(); + Expect(recorder.Submissions() == 1 && recorder.InFlightBatches() == 0 && recorder.InFlightKeptBytes() == 0, "a batch keeping three budgets was not submitted, waited for and released"); + recorder.Keep(std::make_shared(1), Budget); + recorder.Submit(); + Expect(recorder.InFlightKeptBytes() == Budget, "a plain Submit did not count the batch's kept bytes in flight"); + for (auto& [fence, signaled] : mock.signaled) signaled = true; + Expect(recorder.Reap() && recorder.InFlightKeptBytes() == 0, "a reaped batch still counts " + std::to_string(recorder.InFlightKeptBytes()) + " kept bytes in flight"); +} + +} + +int main() { + try { + OpenBatchUnderBudget(); + InFlightWithinTwiceTheBudget(); + SyncedWithinTwiceTheBudget(); + CountFollowsEveryPath(); + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } + if (failures != 0) { + std::fprintf(stderr, "%d recorder kept bytes checks failed\n", failures); + return 1; + } + std::printf("recorder kept bytes tests passed\n"); + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp new file mode 100644 index 000000000..f8466db76 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp @@ -0,0 +1,183 @@ +#include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp" +#include "BdaAbi.hpp" +#include "Optimization/BindingAllocator.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; + +struct ProfileInput { + std::vector capabilities{spv::CapabilityShader, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; + std::array extensionStrings{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; + std::array extensions{extensionStrings[0], extensionStrings[1]}; + VkPhysicalDeviceRobustness2FeaturesEXT robustness{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT, nullptr, VK_FALSE, VK_FALSE, VK_TRUE}; + VkPhysicalDeviceDescriptorIndexingFeatures indexing{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES, &robustness}; + VkPhysicalDevice8BitStorageFeatures bytes{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_8BIT_STORAGE_FEATURES, &indexing, VK_TRUE}; + VkPhysicalDeviceBufferDeviceAddressFeatures bda{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES, &bytes, VK_TRUE}; + VkPhysicalDeviceFeatures core{}; + VkDeviceCreateInfo device{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + VkPhysicalDeviceLimits limits{}; + + ProfileInput() { + core.shaderInt64 = VK_TRUE; + core.vertexPipelineStoresAndAtomics = VK_TRUE; + core.fragmentStoresAndAtomics = VK_TRUE; + device.pEnabledFeatures = &core; + device.pNext = &bda; + limits.maxPushConstantsSize = 128u; + limits.maxBoundDescriptorSets = 1u; + limits.maxStorageBufferRange = sizeof(ShaderRecompiler::RuntimeAbi::ShaderData); + } + + ShaderRecompiler::SpirvTarget Target() const { + ShaderRecompiler::SpirvTarget target{}; + target.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version; + target.supportedCapabilities = capabilities; + target.supportedExtensions = extensions; + return target; + } +}; + +template +void Reject(TAction action, const char* expected) { + try { + action(); + } catch (const std::runtime_error& error) { + Require(std::string(error.what()).find(expected) != std::string::npos, "unexpected profile validation error"); + return; + } + throw std::runtime_error("invalid shader device profile was accepted"); +} + +void CheckProfile() { + ProfileInput input; + const AgcDriver::ShaderDeviceProfile profile(input.Target(), input.device, input.limits); + input.capabilities.clear(); + input.extensionStrings[0] = "changed"; + input.core.shaderInt64 = VK_FALSE; + input.robustness.nullDescriptor = VK_FALSE; + input.limits.maxPushConstantsSize = 0u; + const auto target = profile.Target(); + Require(target.supportedCapabilities.size() == 4u && std::ranges::is_sorted(target.supportedCapabilities), "profile capabilities were not frozen"); + Require(std::ranges::find(target.supportedExtensions, "SPV_KHR_physical_storage_buffer") != target.supportedExtensions.end(), "profile extension storage is borrowed"); + Require(profile.Limits().maxPushConstantsSize == 128u, "profile limits were not frozen"); + Require(profile.NullDescriptors(), "profile null descriptor support was not frozen"); + static_assert(!std::is_copy_constructible_v && !std::is_move_constructible_v); + const auto invalid = [](auto mutate, const char* expected) { + ProfileInput candidate; + mutate(candidate); + Reject([&] { AgcDriver::ShaderDeviceProfile rejected(candidate.Target(), candidate.device, candidate.limits); }, expected); + }; + invalid([](auto& value) { value.device.pEnabledFeatures = nullptr; }, "enabled core features"); + invalid([](auto& value) { value.bda.bufferDeviceAddress = VK_FALSE; }, "bufferDeviceAddress"); + invalid([](auto& value) { value.device.pNext = nullptr; }, "bufferDeviceAddress"); + invalid([](auto& value) { value.core.shaderInt64 = VK_FALSE; }, "shaderInt64"); + invalid([](auto& value) { value.bytes.storageBuffer8BitAccess = VK_FALSE; }, "storageBuffer8BitAccess"); + invalid([](auto& value) { value.robustness.nullDescriptor = VK_FALSE; }, "nullDescriptor"); + invalid([](auto& value) { value.indexing.pNext = nullptr; }, "nullDescriptor"); + invalid([](auto& value) { value.core.fragmentStoresAndAtomics = VK_FALSE; }, "graphics stores and atomics"); + invalid([](auto& value) { value.limits.maxPushConstantsSize = 127u; }, "exceeds device limits"); + invalid([](auto& value) { value.limits.maxBoundDescriptorSets = 0u; }, "exceeds device limits"); + invalid([](auto& value) { --value.limits.maxStorageBufferRange; }, "ShaderData exceeds device limits"); + invalid([](auto& value) { value.capabilities.clear(); }, "missing runtime capabilities"); + invalid([](auto& value) { value.extensions[0] = "missing"; }, "missing runtime extensions"); + invalid([](auto& value) { value.capabilities.push_back(spv::CapabilitySampledImageArrayDynamicIndexing); }, "shaderSampledImageArrayDynamicIndexing"); + invalid([](auto& value) { value.capabilities.push_back(spv::CapabilityStorageImageArrayNonUniformIndexing); }, "shaderStorageImageArrayNonUniformIndexing"); + invalid([](auto& value) { value.capabilities.push_back(spv::CapabilityStorageImageReadWithoutFormat); }, "shaderStorageImageReadWithoutFormat"); + invalid([](auto& value) { value.capabilities.push_back(spv::CapabilityStorageImageMultisample); }, "shaderStorageImageMultisample"); + invalid([](auto& value) { value.capabilities.push_back(spv::CapabilityFragmentBarycentricKHR); }, "fragmentShaderBarycentric"); + invalid([](auto& value) { value.capabilities.push_back(spv::CapabilityMeshShadingEXT); }, "meshShader"); + ProfileInput enabled; + enabled.capabilities.push_back(spv::CapabilitySampledImageArrayDynamicIndexing); + enabled.capabilities.push_back(spv::CapabilityStorageImageArrayNonUniformIndexing); + enabled.core.shaderSampledImageArrayDynamicIndexing = VK_TRUE; + enabled.indexing.shaderStorageImageArrayNonUniformIndexing = VK_TRUE; + const AgcDriver::ShaderDeviceProfile accepted(enabled.Target(), enabled.device, enabled.limits); + Require(accepted.Target().supportedCapabilities.size() == 6u, "enabled descriptor indexing was rejected"); +} + +void CheckAbi() { + using ShaderRecompiler::RuntimeAbi::Binding; + using ShaderRecompiler::RuntimeAbi::BindingNumber; + using ShaderRecompiler::RuntimeAbi::Stage; + std::set bindings; + for (std::uint32_t stage = 0u; stage < ShaderRecompiler::RuntimeAbi::StageCount; ++stage) { + for (std::uint32_t binding = 0u; binding < static_cast(Binding::Count); ++binding) { + Require(bindings.insert(BindingNumber(static_cast(stage), static_cast(binding))).second, "runtime ABI bindings overlap"); + } + } + Require(BindingNumber(Stage::Main, Binding::ShaderData) == 62u && BindingNumber(Stage::Fragment, Binding::ShaderData) == 125u, "runtime ABI binding numbers changed"); + Reject([] { BindingNumber(static_cast(4u), Binding::Buffers); }, "invalid stage or binding"); + Reject([] { BindingNumber(Stage::Main, Binding::Count); }, "invalid stage or binding"); + Reject([] { ShaderRecompiler::RuntimeAbi::RequireVersion(0u); }, "incompatible version"); + ShaderRecompiler::RuntimeAbi::RequireVersion(ShaderRecompiler::RuntimeAbi::Version); +} + +void CheckHeaps() { + using namespace ShaderRecompiler; + const auto allocate = [](const ImageResource& image, std::uint32_t count, std::uint32_t samplers = 0u) { + IrProgram program; + program.Metadata().shaderInfoComplete = true; + program.Resources().info.images.assign(count, image); + program.Resources().info.samplers.resize(samplers); + return BindingAllocator{}.Allocate(program, {0u, 0u, 0u, 128u}); + }; + ImageResource image; + image.resourceClass = ImageResourceClass::Sampled; + image.numericClass = IrTextureNumericClass::Float; + image.dimension = RdnaImageDimension::Dim2D; + const auto single = allocate(image, 1u); + const auto full = allocate(image, RuntimeAbi::SampledHeapCapacity, RuntimeAbi::SamplerHeapCapacity / 2u); + Require(single.layout.ShaderDataDwords() == full.layout.ShaderDataDwords() && single.layout.memoryOffsetDword == full.layout.memoryOffsetDword && !full.layout.UsesPushData(), "runtime layout depends on resource count"); + Require(full.layout.memoryOffsetDword == 0u && full.layout.DispatchThreadLimitDword() == 0u && full.layout.ShaderDataDwords() == 0u, "direct image resources allocated runtime metadata"); + Reject([&] { allocate(image, RuntimeAbi::SampledHeapCapacity + 1u); }, "heap capacity exceeded"); + Reject([&] { allocate(image, 1u, RuntimeAbi::SamplerHeapCapacity + 1u); }, "metadata capacity"); + image.resourceClass = ImageResourceClass::Storage; + image.mipMode = ImageMipMode::DynamicStorage; + image.mipCount = RuntimeAbi::StorageHeapCapacity; + const auto storage = allocate(image, 1u); + Require(BindingAllocator{}.FindBinding(storage.layout, DescriptorBindingForImage(image)).resources.size() == image.mipCount, "storage heap did not reserve each mip"); + Reject([&] { allocate(image, 2u); }, "heap capacity exceeded"); + Reject([&] { allocate(image, 0u, RuntimeAbi::SamplerHeapCapacity / 2u + 1u); }, "sampler pairs"); + const std::array dimensions{RdnaImageDimension::Dim1D, RdnaImageDimension::Dim1DArray, RdnaImageDimension::Dim2D, RdnaImageDimension::Dim2DArray, RdnaImageDimension::Dim3D, RdnaImageDimension::Dim2DMsaa, RdnaImageDimension::Dim2DMsaaArray}; + std::set classes; + for (std::uint32_t group = 0u; group < 8u; ++group) { + for (const auto dimension : dimensions) { + image = {}; + image.resourceClass = group < 4u ? ImageResourceClass::Sampled : ImageResourceClass::Storage; + image.numericClass = group == 1u || group >= 5u ? IrTextureNumericClass::Uint : group == 2u ? IrTextureNumericClass::Sint : IrTextureNumericClass::Float; + image.depthCompare = group == 3u; + image.atomic = group >= 6u; + image.atomic64 = group == 7u; + image.dimension = dimension; + const auto binding = DescriptorBindingForImage(image); + Require(classes.insert(static_cast(binding)).second, "typed image classes overlap"); + Require(RuntimeAbi::HeapCapacity(binding) == (group < 4u ? RuntimeAbi::SampledHeapCapacity : RuntimeAbi::StorageHeapCapacity), "typed image class has an invalid capacity"); + } + } + Require(classes.size() == RuntimeAbi::ImageBindingCount && *classes.begin() == 1u && *classes.rbegin() == RuntimeAbi::ImageBindingCount, "typed image class mapping is incomplete"); + Reject([] { RuntimeAbi::HeapCapacity(RuntimeAbi::Binding::ShaderData); }, "not a typed heap"); +} + +} + +int main() { + try { + CheckAbi(); + CheckHeaps(); + CheckProfile(); + std::cout << "shader runtime ABI and device profile tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 11af875da..e5db59f1b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -1,5 +1,12 @@ #include "ShaderDiskCache.hpp" +#include "CacheKey.hpp" +#include "VertexInputSpecialization.hpp" +#include "SpirvBackend/SpirvSpecialization.hpp" +#include "Optimization/ResourceMaterializer.hpp" +#include "Optimization/ResourceProgram.hpp" +#include "Optimization/DescriptorBindingBuilder.hpp" #include "ShaderCacheDirectory.hpp" +#include "ControlFlow/RequestSerializer.hpp" #include #include #include @@ -10,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -44,18 +52,16 @@ bool sameBindings(const std::vector& left, const std::vector< return true; } -void requireSameResult(const RecompileResult& left, const RecompileResult& right, const char* what) { +void requireSameArtifact(const CompiledShaderArtifact& left, const CompiledShaderArtifact& right, const char* what) { const std::string prefix = std::string(what) + ": "; require(left.spirv.Words() == right.spirv.Words(), prefix + "SPIR-V differs"); - require(sameBindings(left.bindings, right.bindings), prefix + "bindings differ"); - require(left.pushConstants == right.pushConstants, prefix + "push constants differ"); require(left.bdaAbiVersion == right.bdaAbiVersion, prefix + "BDA ABI version differs"); - require(left.vertexAttributes.size() == right.vertexAttributes.size(), prefix + "vertex attribute count differs"); - for (std::size_t i = 0; i < left.vertexAttributes.size(); ++i) { - const auto& a = left.vertexAttributes[i]; - const auto& b = right.vertexAttributes[i]; - require(a.location == b.location && a.components == b.components && a.resource.fields == b.resource.fields && a.fetchIndex == b.fetchIndex, prefix + "vertex attribute differs"); - } + require(left.runtimeAbiVersion == right.runtimeAbiVersion, prefix + "runtime ABI version differs"); + require(left.memoryOffsetDword == right.memoryOffsetDword, prefix + "memory offset differs"); + require(left.shaderDataDwords == right.shaderDataDwords && left.imageMetadataDword == right.imageMetadataDword && left.runtimeImageCount == right.runtimeImageCount && left.runtimeImageResources == right.runtimeImageResources, prefix + "compact data layout differs"); + require(left.hostSubgroupSize == right.hostSubgroupSize, prefix + "host subgroup size differs"); + require(left.vertexInputs == right.vertexInputs, prefix + "vertex inputs differ"); + require(left.vertexInputPatches == right.vertexInputPatches, prefix + "vertex type patches differ"); require(left.vertexOffsetSgpr == right.vertexOffsetSgpr && left.instanceOffsetSgpr == right.instanceOffsetSgpr, prefix + "offset SGPRs differ"); require(left.vertexOffsetShared == right.vertexOffsetShared && left.instanceOffsetShared == right.instanceOffsetShared && left.vertexOffsetConflict == right.vertexOffsetConflict && left.instanceOffsetConflict == right.instanceOffsetConflict, prefix + "offset flags differ"); require(left.parameterExports == right.parameterExports, prefix + "parameter exports differ"); @@ -63,22 +69,35 @@ void requireSameResult(const RecompileResult& left, const RecompileResult& right for (std::size_t i = 0; i < left.fragmentParameters.size(); ++i) { const auto& a = left.fragmentParameters[i]; const auto& b = right.fragmentParameters[i]; - require(a.location == b.location && a.sourceLocation == b.sourceLocation && a.flat == b.flat && a.perVertex == b.perVertex, prefix + "fragment parameter differs"); + require(a.location == b.location && a.sourceLocation == b.sourceLocation && a.flat == b.flat && a.perVertex == b.perVertex && a.custom == b.custom, prefix + "fragment parameter differs"); + } +} + +void requireSameResult(const RecompileResult& left, const RecompileResult& right, const char* what) { + requireSameArtifact(left, right, what); + const std::string prefix = std::string(what) + ": "; + require(sameBindings(left.bindings, right.bindings), prefix + "bindings differ"); + require(left.pushConstants == right.pushConstants, prefix + "push constants differ"); + require(left.specialization == right.specialization, prefix + "specialization constants differ"); + require(left.poisonedSrtReads == right.poisonedSrtReads, prefix + "poisoned SRT read counts differ"); + require(left.vertexAttributes.size() == right.vertexAttributes.size(), prefix + "vertex attribute count differs"); + for (std::size_t i = 0; i < left.vertexAttributes.size(); ++i) { + const auto& a = left.vertexAttributes[i]; + const auto& b = right.vertexAttributes[i]; + require(a.location == b.location && a.components == b.components && a.resource.fields == b.resource.fields && a.fetchIndex == b.fetchIndex && a.formatComponents == b.formatComponents, prefix + "vertex attribute differs"); } } void requireSameVariant(const CompiledVariant& left, const CompiledVariant& right, const char* what) { - requireSameResult(left.result, right.result, what); + requireSameArtifact(left.artifact, right.artifact, what); const std::string prefix = std::string(what) + ": "; const auto& a = left.info; const auto& b = right.info; require(a.stage == b.stage && a.shaderHash == b.shaderHash && a.waveSize == b.waveSize && a.userDataBase == b.userDataBase && a.userDataCount == b.userDataCount && a.scratchDwords == b.scratchDwords && a.paramExportMask == b.paramExportMask, prefix + "compiled info differs"); require(a.info == b.info, prefix + "shader info differs"); require(a.bindings == b.bindings, prefix + "info binding layout differs"); - require(sameBindings(left.bindings.bindings, right.bindings.bindings), prefix + "allocation bindings differ"); require(left.bindings.layout == right.bindings.layout, prefix + "allocation layout differs"); require(left.bindings.pushConstantOffsetBytes == right.bindings.pushConstantOffsetBytes && left.bindings.pushConstantSizeBytes == right.bindings.pushConstantSizeBytes, prefix + "push constant range differs"); - require(left.bindings.pushConstants == right.bindings.pushConstants, prefix + "allocation push constants differ"); } DescriptorBinding sampleBinding(std::uint32_t seed) { @@ -115,7 +134,14 @@ RecompileResult sampleResult() { result.bindings[1].imageShape.reset(); result.pushConstants = {std::byte{1}, std::byte{0xff}, std::byte{0}, std::byte{0x80}, std::byte{7}}; result.bdaAbiVersion = 3; + result.memoryOffsetDword = 7; + result.hostSubgroupSize = 32; + result.vertexInputs = {{1, 4, 2}, {5, 2, 0}}; + result.vertexInputPatches = {{1, 20, {7, 8, 9}}, {5, 40, {10, 11, 12}}}; + result.specialization = {{512, 4}, {516, 0x3f800000u}, {517, 0}}; result.vertexAttributes = {{1, 4, {{0x1000u, 0x20000u, 0x30u, 0x4u}}, 2}, {5, 2, {{9u, 8u, 7u, 6u}}, 0}}; + result.vertexInputs[0].outputMask = 5u; + result.vertexAttributes[0].formatComponents = 3u; result.vertexOffsetSgpr = 12; result.instanceOffsetSgpr = -1; result.vertexOffsetShared = true; @@ -123,16 +149,15 @@ RecompileResult sampleResult() { result.vertexOffsetConflict = false; result.instanceOffsetConflict = true; result.parameterExports = {0, 3, 7}; - result.fragmentParameters = {{0, 1, true, false}, {2, 3, false, true}}; + result.fragmentParameters = {{0, 1, true, false, true}, {2, 3, false, true}}; + result.poisonedSrtReads = 3; result.variantId = 99; return result; } CompiledVariant sampleVariant() { CompiledVariant variant; - variant.result = sampleResult(); - variant.result.bindings.clear(); - variant.result.pushConstants.clear(); + variant.artifact = sampleResult(); variant.layout = {0, 0, 0, 128}; auto& info = variant.info; info.stage = IrShaderStage::Compute; @@ -148,12 +173,13 @@ CompiledVariant sampleVariant() { buffer.source = 3; buffer.firstUsePc = 0x40; buffer.maxByteExtent = 256; - buffer.packedStride = 16; - buffer.descriptorSwizzle = 0x123u; buffer.imageAlias = 1; buffer.read = true; buffer.atomic = true; buffer.scalar = true; + buffer.descriptorFormatted = true; + buffer.formattedReadMask = 5u; + buffer.typedAlignment = 2u; info.info.buffers = {buffer, BufferResource{}}; ImageResource image{}; image.source = 5; @@ -167,12 +193,17 @@ CompiledVariant sampleVariant() { image.srgbDecode = true; image.cube = true; image.r128 = true; + image.fmaskCompatible = false; + image.depthBitsCompatible = false; + image.byElements = 4; + image.byComponents = 1; image.indirectRoot = 0; image.indirectMappingOffset = 12; image.indirectSearchIterations = 3; image.indirectResources = {1, 2, 3}; info.info.images = {image}; - info.info.samplers = {{7, 0x10, true, false, SamplerUseExplicitLod | SamplerUseGather}}; + ResourceMaterializer::PrepareImageModes(info.info); + info.info.samplers = {{7, 0x10, 3, true, false, SamplerUseExplicitLod | SamplerUseGather}}; info.info.sampledPairs = {{0, 0, 0x10}}; StageInput input{}; input.kind = StageInputKind::GlobalInvocationId; @@ -191,6 +222,7 @@ CompiledVariant sampleVariant() { info.info.vertexOffsetSgpr = 6; info.info.hasBitwiseXor = true; info.info.usesDma = true; + info.info.usesFaultBuffer = true; info.bindings.pushDataStartDword = 2; info.bindings.memoryOffsetDword = 1; info.bindings.memoryOffsetCount = 5; @@ -199,8 +231,6 @@ CompiledVariant sampleVariant() { variant.bindings.layout = info.bindings; variant.bindings.pushConstantOffsetBytes = 16; variant.bindings.pushConstantSizeBytes = 112; - variant.bindings.bindings = {sampleBinding(3)}; - variant.bindings.pushConstants = {std::byte{9}, std::byte{8}}; return variant; } @@ -211,7 +241,6 @@ struct SampleRequest { std::array extensions{"SPV_KHR_storage_buffer_storage_class"}; std::array header{}; RecompileRequest request{}; - ResourceSpecialization specialization; std::uint32_t hostSubgroupSize = 32; SampleRequest() { @@ -230,9 +259,6 @@ struct SampleRequest { request.target.maxWorkgroupInvocations = 1024; request.target.maxWorkgroupSharedMemoryBytes = 49152; request.layout = {0, 0, 0, 128}; - specialization.buffers = {{16u, IrBufferFormat::Invalid, 0xfacu}}; - specialization.images = {ResourceSpecialization::Image{}}; - specialization.boundDescriptors = {0, 0}; } std::vector Key() { @@ -240,7 +266,7 @@ struct SampleRequest { request.context.userData = userData; request.target.supportedCapabilities = capabilities; std::vector key; - ShaderDiskCache::BuildKey(request, hostSubgroupSize, specialization, key); + ShaderDiskCache::BuildKey(request, hostSubgroupSize, key); return key; } }; @@ -264,6 +290,11 @@ void verifyResultRoundTrip() { ShaderDiskCache::EncodeResult(RecompileResult{}, bytes); require(ShaderDiskCache::DecodeResult(bytes, decoded), "an empty result does not decode"); requireSameResult(RecompileResult{}, decoded, "empty result round trip"); + auto incompatible = result; + incompatible.runtimeAbiVersion = RuntimeAbi::Version + 1u; + bytes.clear(); + ShaderDiskCache::EncodeResult(incompatible, bytes); + require(!ShaderDiskCache::DecodeResult(bytes, decoded), "an incompatible runtime ABI result decodes"); } void verifyEntryRoundTrip() { @@ -274,6 +305,7 @@ void verifyEntryRoundTrip() { CompiledVariant decoded; require(ShaderDiskCache::DecodeEntry(file, key, decoded) == ShaderDiskCache::LoadStatus::Loaded, "an encoded entry does not decode"); requireSameVariant(variant, decoded, "entry round trip"); + require(decoded.artifact.variantId == 0, "the artifact stored a per-process id"); for (std::size_t size = 0; size < file.size(); size += size < 512 ? 1 : 131) { CompiledVariant partial; @@ -292,6 +324,30 @@ void verifyEntryRoundTrip() { auto otherKey = key; otherKey.back() ^= std::byte{1}; require(ShaderDiskCache::DecodeEntry(file, otherKey, decoded) == ShaderDiskCache::LoadStatus::KeyMismatch, "an entry for another key loads"); + auto previousVersion = file; + previousVersion[4] = static_cast(ShaderDiskCache::FormatVersion - 1u); + require(ShaderDiskCache::DecodeEntry(previousVersion, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, "an entry with the previous format version loads"); + auto incompatible = variant; + incompatible.artifact.runtimeAbiVersion = RuntimeAbi::Version + 1u; + const auto incompatibleFile = ShaderDiskCache::EncodeEntry(key, incompatible); + require(ShaderDiskCache::DecodeEntry(incompatibleFile, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, "an entry with an incompatible runtime ABI loads"); +} + +void verifyArtifactStorageIsolation() { + SampleRequest sample; + const auto key = sample.Key(); + auto result = sampleResult(); + auto variant = sampleVariant(); + variant.artifact = result; + const auto before = ShaderDiskCache::EncodeEntry(key, variant); + result.bindings[0].guestDescriptor[0] ^= 0x10000u; + result.pushConstants[0] ^= std::byte{0xff}; + result.vertexAttributes[0].resource.fields[0] ^= 0x10000u; + variant.artifact = result; + require(ShaderDiskCache::EncodeEntry(key, variant) == before, "invocation data changed the stored artifact"); + result.vertexInputs[0].components = 2; + variant.artifact = result; + require(ShaderDiskCache::EncodeEntry(key, variant) != before, "the stored artifact ignores vertex input metadata"); } void verifyKeySensitivity() { @@ -333,6 +389,7 @@ void verifyKeySensitivity() { changes("barycentrics", [](SampleRequest& sample) { sample.request.target.fragmentShaderBarycentricEnabled = true; }); changes("non-constant texel offsets", [](SampleRequest& sample) { sample.request.target.nonConstantImageOffsets = true; }); changes("the sRGB formats decoded in the shader", [](SampleRequest& sample) { sample.request.target.srgbDecodeFormats = 2u; }); + changes("the narrow subgroup clock", [](SampleRequest& sample) { sample.request.target.narrowSubgroupClock = true; }); changes("the workgroup size limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSize[2] = 128; }); changes("the invocation limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupInvocations = 512; }); changes("the shared memory limit", [](SampleRequest& sample) { sample.request.target.maxWorkgroupSharedMemoryBytes = 32768; }); @@ -341,23 +398,21 @@ void verifyKeySensitivity() { changes("the first binding", [](SampleRequest& sample) { sample.request.layout.firstBinding = 1; }); changes("the push constant offset", [](SampleRequest& sample) { sample.request.layout.pushConstantOffsetBytes = 16; }); changes("the push constant size", [](SampleRequest& sample) { sample.request.layout.pushConstantSizeBytes = 64; }); - changes("a buffer stride", [](SampleRequest& sample) { sample.specialization.buffers[0].packedStride = 32; }); - changes("a buffer format", [](SampleRequest& sample) { sample.specialization.buffers[0].descriptorFormat = static_cast(1); }); - changes("a buffer swizzle", [](SampleRequest& sample) { sample.specialization.buffers[0].descriptorSwizzle = 0xfadu; }); - changes("the buffer count", [](SampleRequest& sample) { sample.specialization.buffers.emplace_back(); }); - changes("an image class", [](SampleRequest& sample) { sample.specialization.images[0].numericClass = IrTextureNumericClass::Float; }); - changes("an image dimension", [](SampleRequest& sample) { sample.specialization.images[0].dimension = static_cast(1); }); - changes("an image mip count", [](SampleRequest& sample) { sample.specialization.images[0].mipCount = 2; }); - changes("an image conversion", [](SampleRequest& sample) { sample.specialization.images[0].conversionFormat = static_cast(1); }); - changes("an image swizzle", [](SampleRequest& sample) { sample.specialization.images[0].shaderSwizzle = 0; }); - changes("an image indirect root", [](SampleRequest& sample) { sample.specialization.images[0].indirectRoot = 0; }); - changes("an image mapping offset", [](SampleRequest& sample) { sample.specialization.images[0].indirectMappingOffset = 4; }); - changes("an image search depth", [](SampleRequest& sample) { sample.specialization.images[0].indirectSearchIterations = 2; }); - changes("an image cube flag", [](SampleRequest& sample) { sample.specialization.images[0].cube = true; }); - changes("an image FMASK flag", [](SampleRequest& sample) { sample.specialization.images[0].fmask = true; }); - changes("an image sRGB decode", [](SampleRequest& sample) { sample.specialization.images[0].srgbDecode = true; }); - changes("the image count", [](SampleRequest& sample) { sample.specialization.images.emplace_back(); }); - changes("the bound descriptors", [](SampleRequest& sample) { sample.specialization.boundDescriptors.push_back(1); }); + changes("a float mode", [](SampleRequest& sample) { sample.request.context.floatMode = ShaderFloatMode{}; }); + const ShaderFloatMode astroMode{0xc0u, true, false, false}; + const auto withMode = [&](const ShaderFloatMode& mode) { + SampleRequest sample; + sample.request.context.floatMode = mode; + return std::pair{sample.Key(), RecompileCacheKey::ContextHash(sample.request)}; + }; + const auto astro = withMode(astroMode); + require(withMode(astroMode) == astro, "the float mode key is not deterministic"); + for (const auto& [what, mode] : std::initializer_list>{ + {"FLOAT_MODE", {0x00u, true, false, false}}, {"DX10_CLAMP", {0xc0u, false, false, false}}, + {"IEEE_MODE", {0xc0u, true, true, false}}, {"FP16_OVFL", {0xc0u, true, false, true}}}) { + const auto changed = withMode(mode); + require(changed.first != astro.first && changed.second != astro.second, std::string("the key ignores ") + what); + } SampleRequest moved; moved.userData[0] ^= 0x10000u; @@ -365,6 +420,58 @@ void verifyKeySensitivity() { moved.request.shader.headerAddress += 0x1000000u; moved.header[3] = std::byte{1}; require(moved.Key() == key, "the key depends on the user data values or the addresses"); + std::array descriptors{}; + const std::array regions{MemoryRegion{0x100000u, descriptors}}; + moved.request.context.memory = regions; + std::fill(moved.userData.begin(), moved.userData.end(), 0xffffffffu); + require(moved.Key() == key, "runtime descriptors changed the artifact key"); + SampleRequest vertex; + vertex.request.shader.stage = ShaderStage::Vertex; + vertex.request.context.compute.reset(); + vertex.request.context.vertex.emplace(); + auto& input = *vertex.request.context.vertex; + input.resourcesNum = 1u; + input.resources[0].fields = {0x10000u, 16u << 16u, 64u, (22u << 12u) | 0xfacu}; + input.resourcesDst[0] = {0u, 4u, 0u, 0u}; + const auto vertexKey = vertex.Key(); + const auto contextKey = RecompileCacheKey::ContextHash(vertex.request); + input.resources[0].fields = {0x20000u, 32u << 16u, 128u, (77u << 12u) | 0xfa9u}; + require(vertex.Key() == vertexKey && RecompileCacheKey::ContextHash(vertex.request) == contextKey, "runtime vertex format, stride, swizzle or address changed the static interface key"); + input.resources[0].fields[3] = (20u << 12u) | 0xfacu; + require(vertex.Key() != vertexKey && RecompileCacheKey::ContextHash(vertex.request) != contextKey, "integer vertex input reused a float interface"); + input.resources[0].fields[3] = (77u << 12u) | 0xfa9u; + input.resourcesDst[0].registersNum = 2u; + require(vertex.Key() != vertexKey, "vertex component interface did not change the artifact key"); + input.fetchEmbedded = true; + const auto embeddedKey = vertex.Key(); + input.resources[0].fields = {}; + input.resourcesDst[0].fetchIndex = 1u; + require(vertex.Key() == embeddedKey, "runtime vertex fetch descriptors changed the artifact key"); + input.resources[0].fields[3] = (20u << 12u) | 0xfacu; + require(vertex.Key() == embeddedKey, "embedded numeric class changed the prepared template key"); + input.resourcesNum = 0u; + require(vertex.Key() != embeddedKey, "embedded semantic interface did not change the artifact key"); + SampleRequest fragment; + fragment.request.shader.stage = ShaderStage::Fragment; + fragment.request.context.compute.reset(); + fragment.request.context.pixel.emplace(); + fragment.request.context.pixel->targetOutputMode[0] = 9; + const auto fragmentKey = fragment.Key(); + const auto fragmentContextKey = RecompileCacheKey::ContextHash(fragment.request); + fragment.request.context.pixel->targetExportMapping.fill(0x1bu); + require(fragment.Key() == fragmentKey && RecompileCacheKey::ContextHash(fragment.request) == fragmentContextKey, "runtime export mapping changed the fragment artifact key"); + fragment.request.context.pixel->targetOutputMode[0] = 7; + require(fragment.Key() != fragmentKey, "integer fragment output reused a float interface"); +} + +void verifyFloatModeSerialization() { + const RequestSerializer serializer; + SampleRequest sample; + require(!serializer.Deserialize(serializer.Serialize(sample.request)).request.context.floatMode.has_value(), "an unknown float mode came back from serialization"); + const ShaderFloatMode mode{0x04u, false, true, true}; + sample.request.context.floatMode = mode; + const auto back = serializer.Deserialize(serializer.Serialize(sample.request)); + require(back.request.context.floatMode == mode, "the float mode did not survive serialization"); } void verifyStore() { @@ -401,7 +508,8 @@ void verifyStore() { struct ComputeRequest { std::vector code{0xe0700000u, 0x80000000u, 0xbf810000u}; std::array userData{0x10000000u, 0x00000000u, 0x40u, 0x00027facu}; - std::array capabilities{1u}; + std::array capabilities{1u, 11u, 5347u, 4448u}; + std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; RecompileRequest request{}; explicit ComputeRequest(bool useCache) { @@ -414,6 +522,8 @@ struct ComputeRequest { request.target.spirvVersion = 0x00010300u; request.target.subgroupSize = 32; request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.bdaAbiVersion = 1u; request.layout.pushConstantSizeBytes = 128; request.useCache = useCache; } @@ -447,7 +557,8 @@ void verifyAcrossProcesses(const char* self) { struct ClockRequest { std::vector code{0xf4900100u, 0x00000000u, 0xbf8cc07fu, 0x7e020204u, 0xe0700000u, 0x80000100u, 0xf800180fu, 0x01010101u, 0xbf810000u}; std::array userData{0x10000000u, 0x00000000u, 0x40u, 0x00027facu}; - std::array capabilities{1u}; + std::array capabilities{1u, 11u, 5347u, 4448u}; + std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; RecompileRequest request{}; explicit ClockRequest(std::uint32_t stride = 0) { @@ -466,13 +577,15 @@ struct ClockRequest { request.target.spirvVersion = 0x00010300u; request.target.subgroupSize = 32; request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.bdaAbiVersion = 1u; request.layout.pushConstantSizeBytes = 128; } }; std::string emissionFailure(const RecompileRequest& request) { try { - static_cast(Recompile(request)); + static_cast(PrepareShader(request)); } catch (const std::runtime_error& error) { const std::string message = error.what(); require(message.find("reads the shader clock, which needs the device's VK_KHR_shader_clock") != std::string::npos, "unexpected failure: " + message.substr(0, message.find("RecompileRequest:"))); @@ -499,12 +612,12 @@ void verifyEmissionFailureMemo() { require(diskMisses() == before.misses + 2, "another binding layout reused the failure"); ClockRequest stride(16); static_cast(emissionFailure(stride.request)); - require(diskMisses() == before.misses + 3, "another buffer specialization reused the failure"); + require(diskMisses() == before.misses + 2, "a runtime buffer stride change repeated static preparation"); ClockRequest relocated; relocated.request.shader.codeAddress += 0x1000u; static_cast(emissionFailure(relocated.request)); - require(diskMisses() == before.misses + 4, "a relocated copy reused the failure"); - require(emissionFailure(clock.request) == first && diskMisses() == before.misses + 4, "the other requests replaced the failure"); + require(diskMisses() == before.misses + 3, "a relocated copy reused the failure"); + require(emissionFailure(clock.request) == first && diskMisses() == before.misses + 3, "the other requests replaced the failure"); ClockRequest concurrent; concurrent.request.layout.pushConstantSizeBytes = 64; @@ -513,7 +626,7 @@ void verifyEmissionFailureMemo() { std::vector messages; for (auto& future : failures) messages.push_back(future.get()); require(std::all_of(messages.begin(), messages.end(), [&](const std::string& message) { return message == messages.front(); }), "concurrent requests failed with different messages"); - require(diskMisses() == before.misses + 5, "concurrent requests emitted the program " + std::to_string(diskMisses() - before.misses - 4) + " times"); + require(diskMisses() == before.misses + 4, "concurrent requests emitted the program " + std::to_string(diskMisses() - before.misses - 3) + " times"); ShaderDiskCache::Flush(); require(ShaderDiskCache::Totals().writes == before.writes, "an emission failure was stored in the disk cache"); @@ -534,6 +647,358 @@ void verifyFailureMemoSwitch(const char* self) { require(std::system(command.c_str()) == 0, "the process with APS5_NO_FAILURE_MEMO=1 failed"); } +void verifyInvocationIsolation() { + ComputeRequest request(true); + const auto first = Recompile(request.request); + request.userData[0] += 0x10000u; + const auto second = Recompile(request.request); + require(first.variantId != 0 && first.variantId == second.variantId, "a buffer address change did not reuse the artifact"); + require(second.cacheHit, "the second invocation missed the compiled cache"); + requireSameArtifact(first, second, "invocations sharing an artifact"); + require(first.spirv.data() == second.spirv.data(), "invocations duplicated SPIR-V storage"); + require(!sameBindings(first.bindings, second.bindings) || first.pushConstants != second.pushConstants, "the second invocation reused stale resource data"); + request.userData[0] -= 0x10000u; + const auto restored = Recompile(request.request); + requireSameResult(first, restored, "restored invocation"); + const auto original = request.userData; + const std::array, 10> descriptors{{ + {original[0] + 0x20000u, original[1], original[2], original[3]}, + {original[0] + 1u, original[1], original[2], original[3]}, + {original[0] + 2u, original[1], original[2], original[3]}, + {original[0] + 3u, original[1], original[2], original[3]}, + {original[0], original[1] | 1u, original[2], original[3]}, + {original[0], original[1], original[2] / 2u, original[3]}, + {original[0], original[1] | (16u << 16u), original[2], original[3]}, + {original[0], original[1], original[2], 0x31004688u}, + {original[0], original[1] | 0x80100000u, original[2], original[3] | 0x00600000u}, + {0u, 0u, 0u, 0u} + }}; + for (const auto& descriptor : descriptors) { + request.userData = descriptor; + const auto changed = Recompile(request.request); + require(changed.cacheHit && first.variantId == changed.variantId, "buffer metadata changed the compiled variant"); + if ((descriptor[0] & 3u) == (original[0] & 3u) && (descriptor[1] & 0xffff0000u) == (original[1] & 0xffff0000u) && descriptor[3] == original[3] && (descriptor[0] != 0u || (descriptor[1] & 0xffffu) != 0u)) { + require(first.spirv.data() == changed.spirv.data() && first.PipelineVariantId() == changed.PipelineVariantId(), "buffer address or size rebuilt the specialized module"); + } + ComputeRequest fresh(false); + fresh.userData = descriptor; + requireSameResult(Recompile(fresh.request), changed, "reused binding plan against fresh compilation"); + const auto repeated = Recompile(request.request); + require(changed.spirv.data() == repeated.spirv.data() && changed.PipelineVariantId() == repeated.PipelineVariantId(), "buffer specialization did not reuse its module"); + } + request.userData = original; + request.userData[3] |= 0x40000000u; + bool rejected = false; + try { static_cast(Recompile(request.request)); } + catch (const std::runtime_error&) { rejected = true; } + require(rejected, "an unsupported buffer descriptor type was accepted"); +} + +void verifyBufferAlignmentSpecialization() { + ComputeRequest request(true); + request.request.context.waveSize = 32u; + request.request.context.compute->numThreads = {32u, 1u, 1u}; + request.userData[3] = 0x31027facu; + static_cast(PrepareShader(request.request)); + const auto misses = diskMisses(); + const auto aligned = Recompile(request.request); + const auto countAtomics = [](const RecompileResult& result) { + std::size_t atomics = 0; + for (std::size_t cursor = 5; cursor < result.spirv.size();) { + const auto count = result.spirv[cursor] >> 16u; + require(count != 0u && count <= result.spirv.size() - cursor, "invalid specialized buffer instruction"); + if ((result.spirv[cursor] & 0xffffu) == spv::OpAtomicCompareExchange) ++atomics; + cursor += count; + } + return atomics; + }; + require(countAtomics(aligned) == 0u, "aligned DWORD store retained unaligned atomic updates"); + for (std::uint32_t offset = 1; offset < 4u; ++offset) { + request.userData[0] = 0x10000000u + offset; + const auto unaligned = Recompile(request.request); + require(unaligned.variantId == aligned.variantId && diskMisses() == misses, "base alignment repeated static shader preparation"); + require(countAtomics(unaligned) == 2u, "unaligned DWORD store did not split into two atomic updates"); + request.userData[0] += 0x10000u; + const auto relocated = Recompile(request.request); + require(unaligned.spirv.data() == relocated.spirv.data(), "relocation with unchanged alignment rebuilt the buffer specialization"); + } + request.userData[0] = 0x10000000u; + const auto restored = Recompile(request.request); + require(restored.spirv.data() == aligned.spirv.data(), "unaligned buffers replaced the aligned specialization"); +} + +void verifyBindingPlanSelection() { + ShaderInfo info{}; + info.buffers.resize(2); + info.buffers[0].atomic = true; + info.buffers[1].read = true; + CompiledBindingLayout compiled{}; + compiled.layout.descriptors = {{DescriptorBindingKind::Buffers, {0, 1}}, {DescriptorBindingKind::FlattenedSrt, {}}, {DescriptorBindingKind::ShaderData, {}}, {DescriptorBindingKind::Gds, {}}}; + compiled.layout.userDataRegisters = {0}; + compiled.layout.memoryOffsetDword = 1; + ResourceSnapshot snapshot{}; + snapshot.buffers.resize(2); + for (auto& buffer : snapshot.buffers) { + buffer.dwordCount = 4u; + buffer.dwords = {0x10000000u, 0u, 64u, 0x31016facu}; + } + snapshot.userData = {17u}; + snapshot.flattenedSrt = {23u}; + const DescriptorBindingBuilder builder; + const auto plan = builder.Prepare(compiled.layout, info, IrShaderStage::Compute, snapshot); + BindingAllocationResult full; + builder.Populate(full, compiled, plan, 0u, snapshot, {}); + require(full.bindings.size() == 4u, "binding plan lost a descriptor before module selection"); + require(full.bindings[0].bufferAtomic == std::vector{true, false} && full.bindings[0].bufferWritten == std::vector{true, false}, "binding plan lost buffer access metadata"); + const std::array liveBindings{full.bindings[0].binding, full.bindings[0].binding, full.bindings[3].binding}; + const auto selected = builder.Select(plan, liveBindings); + snapshot.userData.clear(); + snapshot.flattenedSrt.clear(); + snapshot.buffers[0].dwords[0] += 0x10000u; + snapshot.buffers[1].dwords[2] = 128u; + BindingAllocationResult populated; + builder.Populate(populated, compiled, selected, 0u, snapshot, {}); + require(populated.bindings.size() == 2u && populated.pushConstants.empty(), "module selection retained an unused descriptor"); + auto expected = full.bindings[0]; + expected.guestDescriptor[0] += 0x10000u; + expected.guestDescriptor[6] = 128u; + require(sameBinding(expected, populated.bindings[0]) && sameBinding(full.bindings[3], populated.bindings[1]), "selected plan changed binding order or reused stale buffer data"); + bool rejected = false; + try { static_cast(builder.Select(plan, std::array{UINT32_MAX})); } + catch (const std::runtime_error&) { rejected = true; } + require(rejected, "module selection accepted an unknown binding"); + snapshot.buffers[0].dwordCount = 3u; + rejected = false; + try { builder.Populate(populated, compiled, selected, 0u, snapshot, {}); } + catch (const std::runtime_error&) { rejected = true; } + require(rejected, "selected plan accepted an invalid live buffer descriptor"); + snapshot.buffers[0].dwordCount = 4u; + const auto rejectsAlignment = [&](const char* diagnostic) { + try { static_cast(builder.Prepare(compiled.layout, info, IrShaderStage::Compute, snapshot)); } + catch (const std::runtime_error& error) { + require(std::string_view(error.what()).find(diagnostic) != std::string_view::npos, "alignment check returned an unrelated failure"); + return; + } + throw std::runtime_error("binding plan accepted an unsupported buffer alignment"); + }; + snapshot.buffers[0].dwords[0] |= 2u; + rejectsAlignment("buffer atomic on a V# whose base is not DWORD aligned"); + snapshot.buffers[0].dwords[0] &= ~3u; + info.buffers[1].typedAlignment = 2u; + snapshot.buffers[1].dwords[0] |= 2u; + static_cast(builder.Prepare(compiled.layout, info, IrShaderStage::Compute, snapshot)); + snapshot.buffers[1].dwords[0] |= 1u; + rejectsAlignment("base is not aligned to its element"); + info.buffers[1].typedAlignment = 1u; + info.buffers[1].descriptorFormatted = true; + snapshot.buffers[1].dwords[3] = 0x3100bfacu; + rejectsAlignment("base is not aligned to its element"); + snapshot.buffers[1].dwords[0] &= ~1u; + static_cast(builder.Prepare(compiled.layout, info, IrShaderStage::Compute, snapshot)); + snapshot.buffers[1].dwords[3] = 0x31014facu; + rejectsAlignment("base is not aligned to its element"); +} + +void verifyVertexTypeSpecialization() { + CompiledShaderArtifact artifact; + artifact.vertexInputs = {{0, 4, 0, 1}, {1, 4, 0, 1}}; + artifact.spirv = std::vector{ + spv::MagicNumber, 0x00010000u, 0, 28, 0, + (2u << 16u) | spv::OpCapability, spv::CapabilityShader, + (3u << 16u) | spv::OpMemoryModel, spv::AddressingModelLogical, spv::MemoryModelGLSL450, + (7u << 16u) | spv::OpEntryPoint, spv::ExecutionModelVertex, 20, 0x6e69616du, 0, 10, 11, + (4u << 16u) | spv::OpDecorate, 10, spv::DecorationLocation, 0, + (4u << 16u) | spv::OpDecorate, 11, spv::DecorationLocation, 1, + (2u << 16u) | spv::OpTypeVoid, 1, + (3u << 16u) | spv::OpTypeFloat, 2, 32, + (4u << 16u) | spv::OpTypeInt, 3, 32, 0, + (4u << 16u) | spv::OpTypeVector, 4, 2, 4, + (4u << 16u) | spv::OpTypePointer, 5, spv::StorageClassInput, 4, + (4u << 16u) | spv::OpTypePointer, 6, spv::StorageClassInput, 2, + (3u << 16u) | spv::OpTypeFunction, 7, 1, + (4u << 16u) | spv::OpConstant, 3, 8, 0, + (4u << 16u) | spv::OpVariable, 5, 10, spv::StorageClassInput, + (4u << 16u) | spv::OpVariable, 5, 11, spv::StorageClassInput, + (5u << 16u) | spv::OpFunction, 1, 20, spv::FunctionControlMaskNone, 7, + (2u << 16u) | spv::OpLabel, 21, + (5u << 16u) | spv::OpAccessChain, 6, 22, 10, 8, + (4u << 16u) | spv::OpLoad, 2, 23, 22, + (4u << 16u) | spv::OpBitcast, 3, 24, 23, + (5u << 16u) | spv::OpAccessChain, 6, 25, 11, 8, + (4u << 16u) | spv::OpLoad, 2, 26, 25, + (4u << 16u) | spv::OpBitcast, 3, 27, 26, + (1u << 16u) | spv::OpReturn, + (1u << 16u) | spv::OpFunctionEnd + }; + PrepareVertexInputSpecialization(artifact); + const auto original = artifact.spirv.Words(); + for (std::uint32_t first = 0; first < 3u; ++first) { + for (std::uint32_t second = 0; second < 3u; ++second) { + const std::array classes{first, second}; + const auto specialized = SpecializeVertexInputTypes(artifact, classes); + const auto& words = specialized.Words(); + std::map scalarKinds; + std::uint32_t checked = 0; + for (std::size_t cursor = 5; cursor < words.size(); cursor += words[cursor] >> 16u) { + const auto op = static_cast(words[cursor] & 0xffffu); + if (op == spv::OpTypeFloat) scalarKinds.emplace(words[cursor + 1u], 0u); + if (op == spv::OpTypeInt) scalarKinds.emplace(words[cursor + 1u], words[cursor + 3u] != 0u ? 1u : 2u); + if (op == spv::OpLoad) { + const auto attribute = words[cursor + 2u] == 23u ? 0u : 1u; + require(scalarKinds.at(words[cursor + 1u]) == classes[attribute], "vertex input loaded the wrong numeric type"); + ++checked; + } + if (op == spv::OpBitcast || op == spv::OpCopyObject) { + const auto attribute = words[cursor + 2u] == 24u ? 0u : 1u; + require(op == (classes[attribute] == 2u ? spv::OpCopyObject : spv::OpBitcast), "vertex input did not preserve raw component bits"); + ++checked; + } + } + require(checked == 4u, "vertex specialization lost attribute loads"); + require(artifact.spirv.Words() == original, "vertex specialization mutated the shared template"); + } + } + auto invalid = artifact; + invalid.vertexInputPatches.front().word = static_cast(original.size()); + bool rejected = false; + try { static_cast(SpecializeVertexInputTypes(invalid, std::array{0u, 0u})); } + catch (const std::runtime_error&) { rejected = true; } + require(rejected, "out-of-range vertex type patch was accepted"); +} + +void verifyBuiltinSpecialization() { + std::vector words{spv::MagicNumber, 0x00010300u, 0u, 40u, 0u}; + const auto emit = [&](spv::Op op, std::initializer_list operands) { + words.push_back((static_cast(operands.size() + 1u) << 16u) | op); + words.insert(words.end(), operands); + }; + emit(spv::OpCapability, {spv::CapabilityShader}); + emit(spv::OpMemoryModel, {spv::AddressingModelLogical, spv::MemoryModelGLSL450}); + emit(spv::OpEntryPoint, {spv::ExecutionModelGLCompute, 10u, 0x6e69616du, 0u}); + emit(spv::OpExecutionMode, {10u, spv::ExecutionModeLocalSize, 1u, 1u, 1u}); + emit(spv::OpTypeVoid, {1u}); + emit(spv::OpTypeBool, {2u}); + emit(spv::OpTypeInt, {3u, 32u, 0u}); + emit(spv::OpTypeVector, {4u, 3u, 4u}); + emit(spv::OpTypeFunction, {5u, 1u}); + emit(spv::OpConstant, {3u, 6u, 0u}); + emit(spv::OpConstant, {3u, 7u, 1u}); + emit(spv::OpConstant, {3u, 8u, 2u}); + emit(spv::OpConstant, {3u, 9u, 3u}); + emit(spv::OpConstantComposite, {4u, 30u, 6u, 7u, 8u, 9u}); + emit(spv::OpTypePointer, {32u, spv::StorageClassPrivate, 4u}); + emit(spv::OpVariable, {32u, 31u, spv::StorageClassPrivate}); + emit(spv::OpTypePointer, {33u, spv::StorageClassPrivate, 3u}); + emit(spv::OpVariable, {33u, 34u, spv::StorageClassPrivate}); + emit(spv::OpFunction, {1u, 10u, spv::FunctionControlMaskNone, 5u}); + emit(spv::OpLabel, {11u}); + emit(spv::OpBitFieldUExtract, {3u, 12u, 9u, 7u, 7u}); + emit(spv::OpSelectionMerge, {16u, spv::SelectionControlMaskNone}); + emit(spv::OpSwitch, {12u, 15u, 1u, 14u}); + emit(spv::OpLabel, {14u}); + emit(spv::OpBranch, {16u}); + emit(spv::OpLabel, {15u}); + emit(spv::OpIAdd, {3u, 17u, 8u, 9u}); + emit(spv::OpBranch, {16u}); + emit(spv::OpLabel, {16u}); + emit(spv::OpPhi, {3u, 18u, 7u, 14u, 17u, 15u}); + emit(spv::OpStore, {34u, 18u}); + emit(spv::OpIEqual, {2u, 19u, 18u, 7u}); + emit(spv::OpSelectionMerge, {22u, spv::SelectionControlMaskNone}); + emit(spv::OpBranchConditional, {19u, 20u, 21u}); + emit(spv::OpLabel, {20u}); + emit(spv::OpVectorExtractDynamic, {3u, 23u, 30u, 6u}); + emit(spv::OpVectorExtractDynamic, {3u, 24u, 30u, 7u}); + emit(spv::OpVectorExtractDynamic, {3u, 25u, 30u, 8u}); + emit(spv::OpVectorExtractDynamic, {3u, 26u, 30u, 9u}); + emit(spv::OpCompositeConstruct, {4u, 27u, 23u, 24u, 25u, 26u}); + emit(spv::OpStore, {31u, 27u}); + emit(spv::OpBranch, {22u}); + emit(spv::OpLabel, {21u}); + emit(spv::OpUnreachable, {}); + emit(spv::OpLabel, {22u}); + emit(spv::OpReturn, {}); + emit(spv::OpFunctionEnd, {}); + const auto specialized = SpecializeSpirv(words); + bool identity = false; + std::uint32_t phiValue = 0u; + for (std::size_t cursor = 5; cursor < specialized.size();) { + const auto count = specialized[cursor] >> 16u; + require(count != 0u && count <= specialized.size() - cursor, "builtin specialization produced a truncated instruction"); + const auto op = static_cast(specialized[cursor] & 0xffffu); + require(op != spv::OpSwitch && op != spv::OpBranchConditional && op != spv::OpPhi && op != spv::OpVectorExtractDynamic, "builtin specialization retained constant control flow or dynamic exports"); + if (op == spv::OpStore && specialized[cursor + 1u] == 31u) identity = specialized[cursor + 2u] == 30u; + if (op == spv::OpStore && specialized[cursor + 1u] == 34u) phiValue = specialized[cursor + 2u]; + cursor += count; + } + bool correctPhi = false; + for (std::size_t cursor = 5; cursor < specialized.size(); cursor += specialized[cursor] >> 16u) { + if ((specialized[cursor] & 0xffffu) == spv::OpConstant && specialized[cursor + 2u] == phiValue) correctPhi = specialized[cursor + 3u] == 1u; + } + require(identity && correctPhi, "builtin specialization selected the wrong export or phi value"); + require(SpecializeSpirv(specialized) == specialized, "builtin specialization is not stable"); +} + +void verifySpecializationLiveness() { + std::vector words{spv::MagicNumber, 0x00010300u, 0u, 64u, 0u}; + const auto emit = [&](spv::Op op, std::initializer_list operands) { + words.push_back((static_cast(operands.size() + 1u) << 16u) | op); + words.insert(words.end(), operands); + }; + emit(spv::OpCapability, {spv::CapabilityShader}); + emit(spv::OpMemoryModel, {spv::AddressingModelLogical, spv::MemoryModelGLSL450}); + emit(spv::OpEntryPoint, {spv::ExecutionModelGLCompute, 10u, 0x6e69616du, 0u}); + emit(spv::OpExecutionMode, {10u, spv::ExecutionModeLocalSize, 1u, 1u, 1u}); + emit(spv::OpDecorate, {27u, spv::DecorationRelaxedPrecision}); + emit(spv::OpTypeVoid, {1u}); + emit(spv::OpTypeInt, {3u, 32u, 0u}); + emit(spv::OpTypePointer, {4u, spv::StorageClassPrivate, 3u}); + emit(spv::OpTypeFunction, {5u, 1u}); + emit(spv::OpConstant, {3u, 6u, 0u}); + emit(spv::OpConstant, {3u, 7u, 1u}); + emit(spv::OpTypeStruct, {24u, 3u, 3u, 3u, 3u, 3u, 3u, 3u, 3u, 3u, 3u, 3u, 3u, 3u, 3u, 3u, 3u}); + emit(spv::OpConstantComposite, {24u, 25u, 7u, 7u, 7u, 7u, 7u, 7u, 7u, 7u, 7u, 7u, 7u, 7u, 7u, 7u, 7u, 7u}); + emit(spv::OpVariable, {4u, 8u, spv::StorageClassPrivate, 6u}); + emit(spv::OpFunction, {1u, 10u, spv::FunctionControlMaskNone, 5u}); + emit(spv::OpLabel, {11u}); + emit(spv::OpLoad, {3u, 12u, 8u}); + emit(spv::OpIMul, {3u, 13u, 12u, 7u}); + emit(spv::OpIAdd, {3u, 14u, 13u, 7u}); + emit(spv::OpCopyObject, {3u, 15u, 12u}); + emit(spv::OpCopyObject, {3u, 16u, 15u}); + emit(spv::OpIAdd, {3u, 17u, 16u, 7u}); + emit(spv::OpStore, {8u, 17u}); + emit(spv::OpLoad, {3u, 18u, 8u, spv::MemoryAccessVolatileMask}); + emit(spv::OpFunctionCall, {1u, 20u, 30u}); + emit(spv::OpCompositeExtract, {3u, 26u, 25u, 15u}); + emit(spv::OpStore, {8u, 26u}); + emit(spv::OpCopyObject, {3u, 27u, 12u}); + emit(spv::OpStore, {8u, 27u}); + emit(spv::OpLoad, {3u, 28u, 8u}); + emit(spv::OpReturn, {}); + emit(spv::OpFunctionEnd, {}); + emit(spv::OpFunction, {1u, 30u, spv::FunctionControlMaskNone, 5u}); + emit(spv::OpLabel, {31u}); + emit(spv::OpReturn, {}); + emit(spv::OpFunctionEnd, {}); + const auto specialized = SpecializeSpirv(words); + std::map> results; + std::size_t stores = 0; + for (std::size_t cursor = 5; cursor < specialized.size();) { + const auto count = specialized[cursor] >> 16u; + const auto op = static_cast(specialized[cursor] & 0xffffu); + require(count != 0u && count <= specialized.size() - cursor, "dead computation removal truncated an instruction"); + if (op == spv::OpLoad || op == spv::OpCopyObject || op == spv::OpIAdd || op == spv::OpIMul || op == spv::OpCompositeExtract || op == spv::OpFunctionCall) results.emplace(specialized[cursor + 2u], std::vector(specialized.begin() + cursor, specialized.begin() + cursor + count)); + if (op == spv::OpStore) ++stores; + cursor += count; + } + for (const auto id : {13u, 14u, 15u, 16u, 28u}) require(!results.contains(id), "specialization retained a dead computation or copy chain"); + require(results.at(17u).at(3) == 12u, "copy propagation lost the live source"); + require(results.at(26u).at(4) == 15u, "copy propagation rewrote a literal as an ID"); + require(results.contains(18u) && results.contains(20u) && results.contains(27u) && stores == 3u, "specialization removed an effect or decorated copy"); + require(SpecializeSpirv(specialized) == specialized, "dead computation removal is not stable"); +} + void verifyDefaultDirectory(const char* self) { setEnvironment("ANYPS5_SHADER_CACHE_DIR", ""); setEnvironment("ANYPS5_NO_SHADER_CACHE", "1"); @@ -558,11 +1023,19 @@ int main(int argc, char** argv) { setEnvironment("ANYPS5_SHADER_CACHE_DIR", directory.string()); verifyResultRoundTrip(); verifyEntryRoundTrip(); + verifyArtifactStorageIsolation(); verifyKeySensitivity(); + verifyFloatModeSerialization(); verifyStore(); verifyAcrossProcesses(argv[0]); verifyEmissionFailureMemo(); verifyFailureMemoSwitch(argv[0]); + verifyInvocationIsolation(); + verifyBufferAlignmentSpecialization(); + verifyBindingPlanSelection(); + verifyVertexTypeSpecialization(); + verifyBuiltinSpecialization(); + verifySpecializationLiveness(); std::error_code error; std::filesystem::remove_all(directory, error); std::cout << "shader disk cache tests passed\n"; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 7a2e71bca..c6d129fa1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -29,9 +29,30 @@ #include #include #include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#endif namespace { +ShaderRecompiler::SpirvTarget BufferTarget() { + static constexpr std::array capabilities{spv::CapabilityShader, spv::CapabilityImageGatherExtended, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; + static constexpr std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; + ShaderRecompiler::SpirvTarget target{}; + target.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version; + target.supportedCapabilities = capabilities; + target.supportedExtensions = extensions; + return target; +} + void require(bool condition, const char* message) { if (!condition) throw std::runtime_error(message); } @@ -41,7 +62,7 @@ void expectFailure(TAction action, const char* expected, const char* message) { try { action(); } catch (const std::runtime_error& error) { - require(std::string(error.what()).find(expected) != std::string::npos, "unexpected failure reason"); + if (std::string(error.what()).find(expected) == std::string::npos) throw std::runtime_error(std::string(message) + ": expected failure containing '" + expected + "', got: " + error.what()); return; } throw std::runtime_error(message); @@ -242,7 +263,8 @@ void verifyBindlessTable() { const std::vector wholeCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; const auto srtAddress = static_cast(reinterpret_cast(srt.data())); const std::array userData{static_cast(srtAddress), static_cast(srtAddress >> 32u)}; - const std::array capabilities{29u}; + const std::array capabilities{29u, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; + const std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; // A wave64 workgroup on a 32-wide host is held by one subgroup (two lanes per invocation). const auto makeRequest = [&](const std::vector& code) { RecompileRequest request{}; @@ -255,6 +277,8 @@ void verifyBindlessTable() { request.target.spirvVersion = 0x00010300u; request.target.subgroupSize = 32; request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.bdaAbiVersion = BdaAbi::Version; request.target.fragmentShaderBarycentricEnabled = false; request.layout.pushConstantSizeBytes = 128; return request; @@ -274,14 +298,12 @@ void verifyBindlessTable() { return std::vector(snapshot.flattenedSrt.end() - static_cast(1u + 2u * slots), snapshot.flattenedSrt.end()); }; const auto tableRoot = [&](const ResourceCapture& capture, std::uint32_t direct) { - require(capture.specialization.images.size() == direct + slots - 1u, "bindless: the specialization does not hold the table slots"); require(capture.snapshot.images.size() == direct + slots - 1u, "bindless: the snapshot does not hold the table slots"); std::uint32_t root = ImageResource::NoIndirectImage; for (std::uint32_t i = 0; i < direct; i++) { - if (capture.specialization.images[i].indirectRoot == i) root = i; + if (capture.plan->descriptorSources.at(capture.plan->info.images.at(i).source).indirectImage.has_value()) root = i; } require(root != ImageResource::NoIndirectImage, "bindless: no table root"); - for (std::uint32_t i = direct; i < capture.specialization.images.size(); i++) require(capture.specialization.images[i].indirectRoot == root, "bindless: an extra image is not the root's slot"); return root; }; @@ -297,15 +319,25 @@ void verifyBindlessTable() { require(tables == 1, "bindless: the table source was not planned"); for (const auto& image : plan->info.images) require(image.indirectSearchIterations == 0u, "bindless: the plan carries a search depth"); const auto direct = static_cast(plan->info.images.size()); + auto staticRequest = request; + const std::array absentResources{}; + staticRequest.context.userData = absentResources; + staticRequest.context.memory = {}; + auto staticProgram = PrepareResourceProgram(staticRequest); + ResourceMaterializer{}.ApplyStaticInterface(staticProgram); + const auto& staticImages = staticProgram.Resources().info.images; + require(staticImages.size() == direct + slots - 1u, "bindless: the static interface needs runtime descriptors"); + const auto staticRoot = std::ranges::find_if(staticImages, [](const ImageResource& image) { return image.indirectSearchIterations != 0u; }); + require(staticRoot != staticImages.end() && staticRoot->indirectResources.size() == slots && staticRoot->indirectMappingOffset == plan->srtReads.size(), "bindless: the static table interface is incomplete"); AgcDriver::ShaderMemory memory({}); const auto capture = memory.Capture(request); const auto root = tableRoot(*capture, direct); require(capture->snapshot.images[root].dwords == heap[0] && capture->snapshot.images[direct].dwords == heap[1] && capture->snapshot.images[direct + 1u].dwords == heap[3], "bindless: the slots do not hold the keyed entries"); - for (std::uint32_t i = direct + 2u; i < capture->snapshot.images.size(); i++) require(capture->snapshot.images[i].dwords == heap[0], "bindless: a pad slot is not a copy of slot 0"); + for (std::uint32_t i = direct + 2u; i < capture->snapshot.images.size(); i++) require(capture->snapshot.images[i].dwords == heap[2], "bindless: a pad slot is not null"); const auto mapping = mappingOf(capture->snapshot); require(std::vector(mapping.begin(), mapping.begin() + 7) == std::vector{3u, 0u, 0u, 1u, 1u, 3u, 2u}, "bindless: the (key, slot) mapping is wrong"); - require(capture->specialization.images[root].indirectMappingOffset + mapping.size() == capture->snapshot.flattenedSrt.size(), "bindless: the mapping offset does not name the block"); + require(capture->plan->srtReads.size() + mapping.size() == capture->snapshot.flattenedSrt.size(), "bindless: the mapping offset does not name the block"); auto regions = memory.Regions(); for (const auto& material : materials) require(covered(regions, &material[1], sizeof(std::uint32_t)), "bindless: a material key was not captured"); for (const auto entry : {0u, 1u, 3u}) require(covered(regions, heap[entry].data(), 32u), "bindless: a table entry was not captured"); @@ -342,7 +374,7 @@ void verifyBindlessTable() { return result; }; const auto uniform = scan(compiled->spirv); - require(uniform.dynamicIndexing && !uniform.switched, "bindless: the SPIR-V does not index the image array dynamically"); + require(uniform.dynamicIndexing && uniform.switched, "bindless: the SPIR-V does not index the image array dynamically"); require(!uniform.shaderNonUniform && !uniform.nonUniform, "bindless: a single-subgroup workgroup was decorated NonUniform"); #if ANYPS5_ENABLE_SPIRV_TOOLS static_cast(ValidateAndOptimizeSpirv(compiled->spirv, request.target.vulkanVersion, request.target.spirvVersion)); @@ -356,8 +388,8 @@ void verifyBindlessTable() { AgcDriver::ShaderMemory splitMemory({}); const auto splitCapture = splitMemory.Capture(split); expectFailure([&] { static_cast(Recompile(split, *splitCapture)); }, "not uniform over the workgroup", "bindless: a split wave indexed the image array as uniform"); - const std::array indexingCapabilities{29u, 5301u, 5307u}; - const std::array indexingExtensions{"SPV_EXT_descriptor_indexing"}; + const std::array indexingCapabilities{29u, 5301u, 5307u, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; + const std::array indexingExtensions{"SPV_EXT_descriptor_indexing", "SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; split.target.supportedCapabilities = indexingCapabilities; split.target.supportedExtensions = indexingExtensions; AgcDriver::ShaderMemory indexingMemory({}); @@ -381,11 +413,9 @@ void verifyBindlessTable() { const auto nullCapture = nullMemory.Capture(request); const auto nullMapping = mappingOf(nullCapture->snapshot); require(std::vector(nullMapping.begin(), nullMapping.begin() + 5) == std::vector{2u, 0u, 0u, 1u, 1u}, "bindless: a null entry's key was mapped"); - require(nullCapture->snapshot.images[direct + 1u].dwords == heap[0], "bindless: a null entry's slot is not the pad"); + require(nullCapture->snapshot.images[direct + 1u].dwords == heap[2], "bindless: a null entry's slot is not the pad"); materials[2][1] = 3u; - // Mode T: every entry keeps its slot; the null entry's slot holds the pad and its key is - // left out of the mapping. auto whole = makeRequest(wholeCode); const auto wholePlan = GetResourcePlan(whole); for (const auto& source : wholePlan->descriptorSources) { @@ -397,7 +427,7 @@ void verifyBindlessTable() { const auto wholeRoot = tableRoot(*wholeCapture, wholeDirect); const auto wholeMapping = mappingOf(wholeCapture->snapshot); require(std::vector(wholeMapping.begin(), wholeMapping.begin() + 7) == std::vector{3u, 0u, 0u, 1u, 1u, 3u, 3u}, "bindless: mode T is not the identity mapping"); - require(wholeCapture->snapshot.images[wholeRoot].dwords == heap[0] && wholeCapture->snapshot.images[wholeDirect].dwords == heap[1] && wholeCapture->snapshot.images[wholeDirect + 1u].dwords == heap[0] && wholeCapture->snapshot.images[wholeDirect + 2u].dwords == heap[3], "bindless: mode T slots are wrong"); + require(wholeCapture->snapshot.images[wholeRoot].dwords == heap[0] && wholeCapture->snapshot.images[wholeDirect].dwords == heap[1] && wholeCapture->snapshot.images[wholeDirect + 1u].dwords == heap[2] && wholeCapture->snapshot.images[wholeDirect + 2u].dwords == heap[3], "bindless: mode T slots are wrong"); whole.context.memory = wholeMemory.Regions(); require(!Recompile(whole, *wholeCapture)->spirv.empty(), "bindless: mode T did not compile"); @@ -406,6 +436,23 @@ void verifyBindlessTable() { AgcDriver::ShaderMemory wideMemory({}); expectFailure([&] { static_cast(wideMemory.Capture(whole)); }, "bindless image table has 100 entries", "bindless: a wide table was bound"); fillSrt(4u); + const auto savedHeap = heap; + heap = {}; + AgcDriver::ShaderMemory emptyMemory({}); + const auto emptyCapture = emptyMemory.Capture(request); + require(mappingOf(emptyCapture->snapshot).front() == 0u, "bindless: an empty table has mapped keys"); + request.context.memory = emptyMemory.Regions(); + require(Recompile(request, *emptyCapture)->variantId == compiled->variantId, "bindless: an empty table changed the artifact"); + heap = savedHeap; + heap[0][3] &= 0x0fffffffu; + AgcDriver::ShaderMemory invalidMemory({}); + expectFailure([&] { static_cast(invalidMemory.Capture(request)); }, "invalid descriptor", "bindless: an invalid T# was accepted"); + heap = savedHeap; + srt[1] &= 0xffffu; + srt[2] = 127u; + AgcDriver::ShaderMemory partialMemory({}); + expectFailure([&] { static_cast(partialMemory.Capture(request)); }, "partial descriptor", "bindless: a partial heap descriptor was accepted"); + fillSrt(4u); } @@ -435,7 +482,8 @@ void verifyDescriptorPhis() { std::copy(second.begin(), second.end(), srt.begin() + 24); const auto srtAddress = static_cast(reinterpret_cast(srt.data())); const std::array userData{static_cast(srtAddress), static_cast(srtAddress >> 32u)}; - const std::array capabilities{29u}; + const std::array capabilities{29u, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; + const std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; const auto makeRequest = [&](const std::vector& code, std::uint32_t waveSize = 32u) { RecompileRequest request{}; request.shader = {ShaderStage::Compute, 0x21000u, code, 0, {}}; @@ -447,6 +495,8 @@ void verifyDescriptorPhis() { request.target.spirvVersion = 0x00010300u; request.target.subgroupSize = 32; request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.bdaAbiVersion = BdaAbi::Version; request.target.fragmentShaderBarycentricEnabled = false; request.layout.pushConstantSizeBytes = 128; return request; @@ -475,7 +525,7 @@ void verifyDescriptorPhis() { const auto capture = memory.Capture(request); request.context.memory = memory.Regions(); const auto compiled = Recompile(request, *capture); - require(countOps(compiled->spirv, OpImageSampleExplicitLod) == 2u, "descriptor Phi: the SPIR-V does not sample once per edge"); + require(countOps(compiled->spirv, OpImageSampleExplicitLod) == 2u, "descriptor Phi: the specialized SPIR-V does not sample once per edge"); #if ANYPS5_ENABLE_SPIRV_TOOLS static_cast(ValidateAndOptimizeSpirv(compiled->spirv, request.target.vulkanVersion, request.target.spirvVersion)); #endif @@ -505,7 +555,7 @@ void verifyDescriptorPhis() { AgcDriver::ShaderMemory twoLaneMemory({}); const auto twoLaneCapture = twoLaneMemory.Capture(twoLane); twoLane.context.memory = twoLaneMemory.Regions(); - require(countOps(Recompile(twoLane, *twoLaneCapture)->spirv, OpImageSampleExplicitLod) == 4u, "descriptor Phi: the two-lane SPIR-V does not sample once per edge and half"); + require(countOps(Recompile(twoLane, *twoLaneCapture)->spirv, OpImageSampleExplicitLod) == 4u, "descriptor Phi: the specialized two-lane SPIR-V does not sample once per edge and half"); auto dynamic = makeRequest(dynamicCode); expectFailure([&] { static_cast(GetResourcePlan(dynamic)); }, "GetSamplerResource dword 0 is not a valid runtime value", "descriptor Phi: an edge without an SRT slot was accepted"); @@ -529,6 +579,7 @@ void verifyProgramCounterRelativeData() { const auto codeAddress = reinterpret_cast(code.data()); RecompileRequest request{}; request.shader = {ShaderStage::Vertex, codeAddress, code, 0, {}}; + request.target = BufferTarget(); request.context.waveSize = 64; request.context.userDataBaseRegister = 8; request.context.vertex = ShaderVertexStageInfo{}; @@ -608,6 +659,125 @@ void verifyLanesOutsideHostSubgroup() { require(pixel(permlanex16, 32u) && pixel(permlane16, 8u), "host lanes: a v_permlane16_b32 row that the host subgroup may hold did not build"); } +void verifyHalfWaveReduction() { + using namespace ShaderRecompiler; + static constexpr std::array capabilities{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic}; + const auto pixel = [](std::span code, std::uint32_t subgroupSize, bool arithmetic) { + ShaderPixelStageInfo info{}; + info.inputAddr = PixelInputBit(PixelInput::PositionX); + info.posX = true; + info.targetOutputMode[0] = 9u; + info.targetExportMapping.fill(0xe4u); + RecompileRequest request{}; + request.shader = {ShaderStage::Fragment, 0x30000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.pixel = info; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = subgroupSize; + if (arithmetic) request.target.supportedCapabilities = capabilities; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + return Recompile(request).spirv; + }; + const auto reductions = [](const std::vector& spirv, spv::Op opcode) { + std::uint32_t count = 0; + for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) { + if ((spirv[cursor] & 0xffffu) == static_cast(opcode) && (spirv[cursor] >> 16u) == 6u && spirv[cursor + 4u] == spv::GroupOperationReduce) ++count; + } + return count; + }; + const auto halves = [](const std::vector& spirv, spv::Op opcode, std::uint32_t identity) { + std::map definitions; + std::vector reduces; + for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) { + const auto op = spirv[cursor] & 0xffffu; + if (op == spv::OpConstant || op == spv::OpSelect || op == spv::OpUGreaterThanEqual) definitions[spirv[cursor + 2u]] = cursor; + if (op == static_cast(opcode) && (spirv[cursor] >> 16u) == 6u && spirv[cursor + 4u] == spv::GroupOperationReduce) reduces.push_back(cursor); + } + const auto defined = [&](std::uint32_t id, spv::Op op) -> std::size_t { + const auto found = definitions.find(id); + return found != definitions.end() && (spirv[found->second] & 0xffffu) == static_cast(op) ? found->second : 0u; + }; + const auto constant = [&](std::uint32_t id, std::uint32_t value) { + const auto at = defined(id, spv::OpConstant); + return at != 0u && spirv[at + 3u] == value; + }; + std::uint32_t lower = 0; + std::uint32_t upper = 0; + std::vector keys; + for (const auto reduce : reduces) { + const auto select = defined(spirv[reduce + 5u], spv::OpSelect); + const auto compare = select != 0u ? defined(spirv[select + 3u], spv::OpUGreaterThanEqual) : 0u; + if (compare == 0u || !constant(spirv[compare + 4u], 32u)) continue; + if (constant(spirv[select + 4u], identity)) { + ++lower; + keys.push_back(spirv[select + 5u]); + } else if (constant(spirv[select + 5u], identity)) { + ++upper; + keys.push_back(spirv[select + 4u]); + } + } + return reduces.size() == 2u && lower == 1u && upper == 1u && keys[0] == keys[1]; + }; + const auto scan = [](std::uint32_t identity, std::uint32_t vector, std::uint32_t scalar) { + std::vector code{0xbe98047eu, 0x7e160f00u, 0x8786187eu, 0xbeea287eu, 0xd501000du}; + if (identity == 0u || identity == 0xffffffffu) code.push_back(identity == 0u ? 0x001a1680u : 0x001a16c1u); + else code.insert(code.end(), {0x001a16ffu, identity}); + for (const auto control : {0xff01110du, 0xff01120du, 0xff01140du, 0xff01180du}) code.insert(code.end(), {vector | 0x001a1afau, control}); + code.insert(code.end(), {0xd778100cu, 0x0305830du, vector | 0x001a190du, 0xbefe046au, 0xd7600006u, 0x00013f0du, 0xd7600007u, 0x00017f0du, scalar | 0x00080706u, 0x7e020208u, 0xf800180fu, 0x01010101u, 0xbf810000u}); + return code; + }; + struct Reduction { + std::uint32_t identity; + std::uint32_t vector; + std::uint32_t scalar; + spv::Op reduce; + const char* message; + }; + static constexpr std::array scans{{ + {0xffffffffu, 0x26000000u, 0x83800000u, spv::OpGroupNonUniformUMin, "half-wave reduction: the wave64 UMin scan did not read lane 31 as a subgroup UMin on 32 lanes"}, + {0u, 0x28000000u, 0x84800000u, spv::OpGroupNonUniformUMax, "half-wave reduction: the wave64 UMax scan did not read lane 31 as a subgroup UMax on 32 lanes"}, + {0x7fffffffu, 0x22000000u, 0x83000000u, spv::OpGroupNonUniformSMin, "half-wave reduction: the wave64 SMin scan did not read lane 31 as a subgroup SMin on 32 lanes"}, + {0x80000000u, 0x24000000u, 0x84000000u, spv::OpGroupNonUniformSMax, "half-wave reduction: the wave64 SMax scan did not read lane 31 as a subgroup SMax on 32 lanes"}, + {0u, 0x4a000000u, 0x80000000u, spv::OpGroupNonUniformIAdd, "half-wave reduction: the wave64 IAdd scan did not read lane 31 as a subgroup IAdd on 32 lanes"}, + {0xffffffffu, 0x36000000u, 0x87000000u, spv::OpGroupNonUniformBitwiseAnd, "half-wave reduction: the wave64 AND scan did not read lane 31 as a subgroup AND on 32 lanes"}, + {0u, 0x38000000u, 0x88000000u, spv::OpGroupNonUniformBitwiseOr, "half-wave reduction: the wave64 OR scan did not read lane 31 as a subgroup OR on 32 lanes"}, + }}; + for (const auto& reduction : scans) require(reductions(pixel(scan(reduction.identity, reduction.vector, reduction.scalar), 32u, true), reduction.reduce) == 1u, reduction.message); + for (const auto& reduction : scans) { + const auto wide = "half-wave reduction: a wave64 scan reduced by SPIR-V opcode " + std::to_string(reduction.reduce) + " did not read lanes 31 and 63 as reductions of host invocations 0-31 and 32-63 on 64 lanes"; + require(halves(pixel(scan(reduction.identity, reduction.vector, reduction.scalar), 64u, true), reduction.reduce, reduction.identity), wide.c_str()); + } + const auto umin = scan(0xffffffffu, 0x26000000u, 0x83800000u); + const auto patched = [&umin](std::initializer_list> words) { + auto code = umin; + for (const auto& [index, word] : words) code[index] = word; + return code; + }; + require(reductions(pixel(patched({{3, 0xbeea25c1u}}), 32u, true), spv::OpGroupNonUniformUMin) == 1u, "half-wave reduction: the scan under s_or_saveexec_b64 -1 did not read lane 31 as a subgroup UMin"); + const char* lane63 = "v_readlane_b32 of lane 63 is outside the 32-lane host subgroup"; + const char* arithmetic = "v_readlane_b32 of lane 31 of a wave64 half-wave reduction scan needs subgroup arithmetic"; + expectFailure([&] { static_cast(pixel(umin, 32u, false)); }, arithmetic, "half-wave reduction: a device without subgroup arithmetic read lane 31 on 32 lanes"); + expectFailure([&] { static_cast(pixel(umin, 64u, false)); }, arithmetic, "half-wave reduction: a device without subgroup arithmetic read lane 31 on 64 lanes"); + expectFailure([&] { static_cast(pixel(umin, 128u, true)); }, "half-wave reduction scan on a 128-lane host subgroup, which is wider than the wave", "half-wave reduction: the scan built on a 128-lane subgroup"); + require(reductions(pixel(patched({{3, 0xbeea047eu}}), 64u, true), spv::OpGroupNonUniformUMin) == 0u, "half-wave reduction: a scan under the entry EXEC became a subgroup UMin on 64 lanes"); + expectFailure([&] { static_cast(pixel(patched({{3, 0xbeea047eu}}), 32u, true)); }, lane63, "half-wave reduction: a scan under the entry EXEC read lane 63"); + expectFailure([&] { static_cast(pixel(patched({{5, 0x001a1680u}}), 32u, true)); }, lane63, "half-wave reduction: a UMin scan of 0 outside the live lanes read lane 63"); + expectFailure([&] { static_cast(pixel(patched({{0, 0xbe980a7eu}, {2, 0xbe860418u}}), 32u, true)); }, lane63, "half-wave reduction: a scan of keys under an s_wqm_b64 mask read lane 63"); + expectFailure([&] { static_cast(pixel(patched({{7, 0xff09110du}}), 32u, true)); }, lane63, "half-wave reduction: a bound_ctrl row_shr:1 step read lane 63"); + expectFailure([&] { static_cast(pixel(patched({{15, 0x0305010du}}), 32u, true)); }, lane63, "half-wave reduction: a v_permlanex16_b32 of lane 0 read lane 63"); + expectFailure([&] { static_cast(pixel(patched({{16, 0x281a190du}}), 32u, true)); }, lane63, "half-wave reduction: a UMin row scan joined by v_max_u32 read lane 63"); + auto threeSteps = umin; + threeSteps.erase(threeSteps.begin() + 12, threeSteps.begin() + 14); + expectFailure([&] { static_cast(pixel(threeSteps, 32u, true)); }, lane63, "half-wave reduction: a scan without row_shr:8 read lane 63"); + auto compared = umin; + compared.erase(compared.begin() + 2); + compared.insert(compared.begin() + 3, {0xd4c50006u, 0x00021680u}); + expectFailure([&] { static_cast(pixel(compared, 32u, true)); }, lane63, "half-wave reduction: a scan selected by a compare under the all-lanes EXEC read lane 63"); + expectFailure([&] { static_cast(pixel(umin, 8u, true)); }, "v_permlanex16_b32 from lanes 16-31 is outside the 8-lane host subgroup", "half-wave reduction: the scan built on an 8-lane subgroup"); +} + void verifyMeshConfiguration() { using namespace ShaderRecompiler; ShaderMeshInputInfo list; @@ -678,7 +848,7 @@ void verifyPixelRequestSerialization() { value.hasPerspectiveCenterVgpr, value.perspectiveCentroid, value.posX, value.posY, value.posZ, value.posW, value.frontFace, value.ancillary, value.sampleShading, value.noPerspective, value.linearCentroid, value.pixelKillEnable, value.depthExportEnable, - value.sampleMaskExportEnable, value.earlyZ, value.executeOnNoop, value.conservativeZExport, + value.sampleMaskExportEnable, value.earlyZ, value.executeOnNoop, value.conservativeZExport, value.orderedPixelShader, value.targetOutputMode, value.targetExportMapping); }; const std::array code{0xbf810000u}; @@ -718,6 +888,7 @@ void verifyPixelRequestSerialization() { .earlyZ = true, .executeOnNoop = true, .conservativeZExport = ConservativeZExport::GreaterThanZ, + .orderedPixelShader = true, .targetOutputMode = {1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u} }; for (std::uint32_t i = 0; i < pixel.interpolatorSettings.size(); ++i) pixel.interpolatorSettings[i] = 0x10101010u + i; @@ -743,7 +914,7 @@ void verifyPixelRequestSerialization() { auto changed = request; changed.context.pixel->targetExportMapping[i] ^= 1u; RecompileCacheKey::Build(changed, replayKey); - require(key != replayKey && RecompileCacheKey::ContextHash(request) != RecompileCacheKey::ContextHash(changed), "shader identity ignored a pixel export mapping"); + require(key == replayKey && RecompileCacheKey::ContextHash(request) == RecompileCacheKey::ContextHash(changed), "runtime pixel export mapping changed the static shader identity"); } } request.context.pixel.reset(); @@ -756,19 +927,19 @@ void verifyPixelRequestSerialization() { minimal.context.waveSize = 64; minimal.context.pixel = ShaderPixelStageInfo{}; const auto encoded = serializer.Serialize(minimal); - require(requestPrefix(encoded, 8u) == "NVNQQQkAAAA=", "new requests did not use serialization version 9"); - constexpr std::size_t mappingOffset = 8u + 37u + 18u + 162u; + require(requestPrefix(encoded, 8u) == "NVNQQQ0AAAA=", "new requests did not use serialization version 13"); + constexpr std::size_t mappingOffset = 8u + 37u + 18u + 163u; for (std::size_t bytes = 0; bytes < 8u; ++bytes) { - expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated version-8 pixel mapping was accepted"); + expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated version-12 pixel mapping was accepted"); } - for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQQoAAAA="}) { + for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQQ4AAAA="}) { expectFailure([&] { static_cast(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted"); } } void verifyLegacyPixelRequests() { using namespace ShaderRecompiler; - static constexpr std::array legacyPixelRequests{ + static constexpr std::array legacyPixelRequests{ "NVNQQQEAAAAFAAADAAAAAAABAAAAAAAAAAAAgb8AAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAA" "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAkAAAAAAAAAAAAA" @@ -809,6 +980,16 @@ void verifyLegacyPixelRequests() { "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAgkAAAAAAAAA" "AAAAAAAAAAAAAAAAAAAAAAAAEEAAAAMBAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" "AAsAAAAAAAAAgAAAAAAAAQ==", + "NVNQQQkAAAAFAAADAAAAAAABAAAAAAAAAAAAgb8AAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAgkAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAEEAAAAMBAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAsAAAAAAAAAgAAAAAAAAQAAAAA=", + "NVNQQQoAAAAFAAADAAAAAAABAAAAAAAAAAAAgb8AAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAgkAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAEEAAAAMBAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAsAAAAAAAAAgAAAAAAAAQAAAAA=", }; const RequestSerializer serializer; for (std::size_t index = 0; index < legacyPixelRequests.size(); ++index) { @@ -820,9 +1001,10 @@ void verifyLegacyPixelRequests() { require(pixel.targetExportMapping == std::array{}, "legacy pixel mapping no longer defaults to zero"); require(pixel.inputAddr == 2u && pixel.hasPerspectiveCenterVgpr && pixel.targetOutputMode[0] == 9u, "legacy pixel layout was misread"); require(pixel.conservativeZExport == (version >= 7u ? ConservativeZExport::GreaterThanZ : ConservativeZExport::AnyZ), "legacy conservative Z layout was misread"); + require(!pixel.orderedPixelShader, "a legacy request became a primitive-ordered pixel shader"); require(request.shader.stage == ShaderStage::Fragment && request.shader.code.size() == 1u && request.shader.code[0] == 0xbf810000u && !request.context.vertex.has_value() && request.context.memory.empty(), "legacy guest context was misaligned"); require(request.target.vulkanVersion == 0x00401000u && request.target.spirvVersion == 0x00010300u && request.target.subgroupSize == 64u && request.layout.firstBinding == 11u && request.layout.pushConstantSizeBytes == 128u, "legacy target or binding layout was misaligned"); - require(request.useCache == (version == 1u) && request.target.nonConstantImageOffsets == (version >= 6u) && request.target.srgbDecodeFormats == 0u, "legacy request trailer was misread"); + require(request.useCache == (version == 1u) && request.target.nonConstantImageOffsets == (version >= 6u) && request.target.srgbDecodeFormats == 0u && !request.target.narrowSubgroupClock, "legacy request trailer was misread"); const auto upgraded = serializer.Deserialize(serializer.Serialize(request)); require(upgraded.request.context.pixel->targetExportMapping == pixel.targetExportMapping && RecompileCacheKey::ContextHash(upgraded.request) == RecompileCacheKey::ContextHash(request), "upgrading a legacy capture changed its pixel mapping"); } @@ -949,6 +1131,8 @@ ShaderRecompiler::RecompileResult recompileSlots(std::initializer_list capabilities{spv::CapabilityFloat64}; + request.target.supportedCapabilities = capabilities; request.layout.pushConstantSizeBytes = 128; request.useCache = false; return Recompile(request); @@ -1001,6 +1185,19 @@ void verifyPixelParameterSlots() { expectFailure([] { static_cast(recompileSlots({0x423u, 0x3u}, shared)); }, "passes its vertices through unchanged", "an interpolated pass-through input was accepted"); } +void verifyF16PixelParameterSlots() { + static constexpr std::array high{0xd7420002u, 0x00020100u, 0xd75a0003u, 0x040a0300u, 0xf800180fu, 0x03030303u, 0xbf810000u}; + static constexpr std::array low{0xd7420002u, 0x00020000u, 0xd75a0003u, 0x040a0200u, 0xf800180fu, 0x03030303u, 0xbf810000u}; + auto inputs = slotInputs({0x03080003u}, high); + require(inputs.size() == 1u && inputs[0].first == 3u && inputs[0].second, "a 16-bit interpolated input was not read per vertex at its slot"); + inputs = slotInputs({0x03080023u}, high); + require(inputs.size() == 1u && inputs[0].first == 3u && inputs[0].second, "an input with a defaulted low half was not read per vertex for its high half"); + require(slotInputs({0x03180023u}, high).empty(), "an input whose high half is defaulted was declared for a high-half read"); + require(slotInputs({0x03080023u}, low).empty(), "an input whose low half is defaulted was declared for a low-half read"); + inputs = slotInputs({0x03180003u}, low); + require(inputs.size() == 1u && inputs[0].first == 3u && inputs[0].second, "an input with a defaulted high half was not read per vertex for its low half"); +} + } void verifyComputedTexelOffsets() { @@ -1024,10 +1221,10 @@ void verifyComputedTexelOffsets() { }; const auto computed = program(0x100u, 0u); const auto constant = program(0xffu, 0x3fu | (1u << 8u)); - const std::array withGather{1u, static_cast(spv::CapabilityImageGatherExtended)}; const auto recompile = [&](const std::vector& code, bool offsets) { RecompileRequest request{}; request.shader = {ShaderStage::Compute, 0x30000u, code, 0, {}}; + request.target = BufferTarget(); request.context.waveSize = 32; request.context.userDataBaseRegister = 0; request.context.userData = userData; @@ -1035,7 +1232,6 @@ void verifyComputedTexelOffsets() { request.target.vulkanVersion = 0x00401000u; request.target.spirvVersion = 0x00010300u; request.target.subgroupSize = 32; - request.target.supportedCapabilities = withGather; request.target.fragmentShaderBarycentricEnabled = false; request.target.nonConstantImageOffsets = offsets; request.layout.pushConstantSizeBytes = 128; @@ -1067,6 +1263,103 @@ void verifyComputedTexelOffsets() { } } +void verifyShaderClockScopes() { + using namespace ShaderRecompiler; + static std::array output{}; + const auto outputBase = static_cast(reinterpret_cast(output.data())); + const std::array userData{static_cast(outputBase), static_cast((outputBase >> 32u) & 0xffffu), 64u, 0xfacu}; + const std::array capabilities{1u, static_cast(spv::CapabilityShaderClockKHR)}; + const std::array extensions{"SPV_KHR_shader_clock"}; + const auto scope = [&](std::uint32_t clockOpcode, bool narrow) { + const std::vector code{clockOpcode, 0x00000000u, 0xbf8cc07fu, 0x7e020204u, 0xe0700000u, 0x80000100u, 0xbf810000u}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x30000u, code, 0, {}}; + request.context.waveSize = 32; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32; + request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.narrowSubgroupClock = narrow; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + AgcDriver::ShaderMemory memory({}); + static_cast(memory.Capture(request)); + request.context.memory = memory.Regions(); + const auto words = Recompile(request).spirv; + std::map constants; + std::vector scopeIds; + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + require(count != 0 && count <= words.size() - cursor, "shader clock: truncated SPIR-V instruction"); + const auto op = words[cursor] & 0xffffu; + if (op == spv::OpConstant && count == 4u) constants[words[cursor + 2]] = words[cursor + 3]; + if (op == spv::OpReadClockKHR) scopeIds.push_back(words[cursor + 3]); + cursor += count; + } + require(scopeIds.size() == 1u && constants.contains(scopeIds[0]), "shader clock: expected one OpReadClockKHR with a constant scope"); + return constants[scopeIds[0]]; + }; + constexpr std::uint32_t Memtime = 0xf4900100u; + constexpr std::uint32_t Memrealtime = 0xf4940100u; + require(scope(Memtime, false) == spv::ScopeSubgroup, "shader clock: s_memtime does not read the subgroup clock"); + require(scope(Memtime, true) == spv::ScopeDevice, "shader clock: s_memtime reads the narrow subgroup clock"); + require(scope(Memrealtime, false) == spv::ScopeDevice && scope(Memrealtime, true) == spv::ScopeDevice, "shader clock: s_memrealtime does not read the device clock"); +} + +void verifyInt64AtomicCapabilities() { + using namespace ShaderRecompiler; + constexpr std::uint32_t Format32_32UInt = 62; + constexpr std::uint32_t Type2D = 9; + const std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; + alignas(256) static std::array buffer{}; + alignas(4096) static std::array texels{}; + const auto bufferAddress = static_cast(reinterpret_cast(buffer.data())); + const auto texelAddress = static_cast(reinterpret_cast(texels.data())); + const std::array userData{ + 0u, 0u, 0u, 0u, + static_cast(bufferAddress), static_cast((bufferAddress >> 32u) & 0xffffu), 256u, 0x01016facu, + static_cast(texelAddress >> 8u), static_cast((texelAddress >> 40u) & 0xffu) | (Format32_32UInt << 20u) | (3u << 30u), 7u | (7u << 14u), 0xfacu | (Type2D << 28u), + 0u, 0u, 0u, 0u}; + const std::vector bufferAtomic{0xe1705000u, 0x80012803u, 0xbf810000u}; + const std::vector imageAtomic{0xf03c0308u, 0x00020a08u, 0xbf810000u}; + const auto compile = [&](const std::vector& code, spv::Capability added) { + const std::array capabilities{spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, static_cast(added)}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x58000u, code, 0, {}}; + request.context.waveSize = 32; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32; + request.target.bdaAbiVersion = BdaAbi::Version; + request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.fragmentShaderBarycentricEnabled = false; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + return Recompile(request).spirv; + }; + const auto declares = [](const std::vector& words, spv::Capability capability) { + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + require(count != 0 && count <= words.size() - cursor, "64-bit atomics: truncated SPIR-V instruction"); + if ((words[cursor] & 0xffffu) == spv::OpCapability && words[cursor + 1] == static_cast(capability)) return true; + cursor += count; + } + return false; + }; + expectFailure([&] { static_cast(compile(bufferAtomic, spv::CapabilityShader)); }, "64-bit buffer atomics need shaderBufferInt64Atomics", "64-bit atomics: a buffer_atomic_inc_x2 compiled without shaderBufferInt64Atomics"); + require(declares(compile(bufferAtomic, spv::CapabilityInt64Atomics), spv::CapabilityInt64Atomics), "64-bit atomics: a buffer_atomic_inc_x2 does not declare Int64Atomics"); + expectFailure([&] { static_cast(compile(imageAtomic, spv::CapabilityShader)); }, "64-bit image atomics need VK_EXT_shader_image_atomic_int64", "64-bit atomics: an image_atomic_swap on a 32_32 image compiled without shaderImageInt64Atomics"); + require(declares(compile(imageAtomic, spv::CapabilityInt64ImageEXT), spv::CapabilityInt64ImageEXT), "64-bit atomics: an image_atomic_swap on a 32_32 image does not declare Int64ImageEXT"); +} + void verifyUnnormalizedSamplers() { using namespace ShaderRecompiler; constexpr std::uint32_t Format8888UNorm = 56; @@ -1103,7 +1396,7 @@ void verifyUnnormalizedSamplers() { request.context.waveSize = 32; request.context.userDataBaseRegister = 0; request.context.userData = data; - request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.context.compute = ShaderComputeStageInfo{{16u, 2u, 1u}, 0u, {false, false, false}, false, 1u}; request.target.vulkanVersion = 0x00401000u; request.target.spirvVersion = 0x00010300u; request.target.subgroupSize = 32; @@ -1201,7 +1494,8 @@ void verifyUnusedUnnormalizedSampler() { ResourceSnapshot snapshot; snapshot.images = {DescriptorValue{{0x00001000u, 0x03800000u, 0x0000c000u, 0x90000facu, 0u, 0u, 0u, 0u}, 8u}}; snapshot.samplers = {DescriptorValue{{0x00008092u, 0x00fff000u, 0x05500000u, 0u}, 4u}}; - const auto populate = [&](const ShaderInfo& shader) { + const auto populate = [&](ShaderInfo shader) { + shader.runtimeImageModes = {ResourceMaterializer::RuntimeImageModes(shader.images[0])}; BindingAllocationResult allocation; allocation.layout.descriptors = {{DescriptorBindingForImage(shader.images[0]), {0u}}, {DescriptorBindingKind::Samplers, {0u}}}; DescriptorBindingBuilder{}.Populate(allocation, shader, IrShaderStage::Compute, 0u, snapshot, {}); @@ -1283,9 +1577,12 @@ void verifyTwoLaneUniformValues() { const auto srtAddress = static_cast(reinterpret_cast(srt.data())); const std::array userData{static_cast(srtAddress), static_cast(srtAddress >> 32u)}; const std::array code{0xf4040080u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xbf8cc07fu, 0x93040302u, 0x4a020004u, 0xe0700000u, 0x80020100u, 0xbf810000u}; - const auto multiplies = [&](std::uint32_t subgroupSize) { + const auto multiplies = [&](std::uint32_t subgroupSize, bool multiply) { + auto shaderCode = code; + if (!multiply) shaderCode[5] = 0x80040302u; RecompileRequest request{}; - request.shader = {ShaderStage::Compute, 0x40000u, code, 0, {}}; + request.shader = {ShaderStage::Compute, 0x40000u, shaderCode, 0, {}}; + request.target = BufferTarget(); request.context.waveSize = 64; request.context.userDataBaseRegister = 0; request.context.userData = userData; @@ -1308,9 +1605,9 @@ void verifyTwoLaneUniformValues() { } return count; }; - const auto oneLane = multiplies(64u); - require(oneLane != 0u, "two-lane uniform values: the scalar multiply is missing from the module"); - require(multiplies(32u) == oneLane, "two-lane uniform values: a two-lane invocation computes a scalar value once per lane"); + for (const auto subgroupSize : {32u, 64u}) { + require(multiplies(subgroupSize, true) == multiplies(subgroupSize, false) + 1, "two-lane uniform values: the scalar multiply must be emitted once per invocation"); + } } void verifyBdaReadFallbackFunctions() { @@ -1525,9 +1822,191 @@ void verifyFunctionLdsBound() { require(unsized == FunctionLdsDwordLimit, "function LDS: an access without a known width must keep the full array"); } -int main() { +struct NestedRequest { + std::vector code; + std::array userData{}; + ShaderRecompiler::RecompileRequest request{}; + + NestedRequest(std::vector words, const void* srt) : code(std::move(words)) { + using namespace ShaderRecompiler; + const auto address = static_cast(reinterpret_cast(srt)); + userData = {static_cast(address), static_cast(address >> 32u)}; + request.shader = {ShaderStage::Compute, 0x60000u + static_cast(code.size()) * 4u, code, 0, {}}; + request.context.waveSize = 64; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target = BufferTarget(); + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32; + request.target.fragmentShaderBarycentricEnabled = false; + request.layout.pushConstantSizeBytes = 128; + } +}; + +bool RegionsCover(const std::vector& regions, const void* pointer, std::size_t bytes) { + auto address = static_cast(reinterpret_cast(pointer)); + const auto end = address + bytes; + while (address < end) { + const auto region = std::find_if(regions.begin(), regions.end(), [&](const ShaderRecompiler::MemoryRegion& candidate) { return address >= candidate.guestAddress && address < candidate.guestAddress + candidate.bytes.size(); }); + if (region == regions.end()) return false; + address = region->guestAddress + region->bytes.size(); + } + return true; +} + +bool RegionsTouch(const std::vector& regions, std::uint64_t address, std::size_t bytes) { + return std::any_of(regions.begin(), regions.end(), [&](const ShaderRecompiler::MemoryRegion& region) { return region.guestAddress < address + bytes && address < region.guestAddress + region.bytes.size(); }); +} + +bool GuardRecords(const ShaderRecompiler::ResourceCapture& capture) { + const auto& plan = *capture.plan; + const auto& flat = capture.snapshot.flattenedSrt; + const auto& poison = capture.snapshot.srtPoison; + const auto flags = plan.srtReads.size(); + const auto guards = plan.guardedSrtSlots.size(); + if (flat.size() != flags + guards + 3u * poison.size()) return false; + for (std::size_t guard = 0; guard < guards; ++guard) { + const auto record = std::find_if(poison.begin(), poison.end(), [&](const ShaderRecompiler::SrtReadPoison& entry) { return entry.slot == plan.guardedSrtSlots[guard]; }); + const auto expected = record == poison.end() ? 0u : static_cast(record - poison.begin()) + 1u; + if (flat[flags + guard] != expected) return false; + } + for (std::size_t record = 0; record < poison.size(); ++record) { + const auto* words = flat.data() + flags + guards + 3u * record; + if (words[0] != poison[record].pc || words[1] != static_cast(poison[record].address) || words[2] != static_cast(poison[record].address >> 32u) || flat.at(poison[record].slot) != 0u) return false; + } + return true; +} + +void verifyGuardedNullPointers() { + using namespace ShaderRecompiler; + constexpr std::size_t Reserved = 0x10000; + void* reserved = nullptr; +#ifdef _WIN32 + reserved = VirtualAlloc(nullptr, Reserved, MEM_RESERVE, PAGE_NOACCESS); +#else + reserved = mmap(nullptr, Reserved, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) reserved = nullptr; +#endif + require(reserved != nullptr, "guarded pointer: cannot reserve an inaccessible range"); + const auto unmapped = static_cast(reinterpret_cast(reserved)); + alignas(256) static std::array output{}; + alignas(256) static std::array root{}; + static std::uint32_t payload = 0x3f800000u; + static std::uint64_t indirect = 0; + const auto outputAddress = static_cast(reinterpret_cast(output.data())); + root = {static_cast(outputAddress), static_cast((outputAddress >> 32u) & 0xffffu), static_cast(sizeof(output)), 0x30027facu, 1u}; + const auto point = [&](std::uint64_t address) { + root[6] = static_cast(address); + root[7] = static_cast(address >> 32u); + }; + const auto payloadAddress = static_cast(reinterpret_cast(&payload)); + const std::vector branchCode{0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xbf068008u, 0xbf850003u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u}; + const std::vector chainCode{0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xbf068008u, 0xbf850006u, 0xf4040385u, 0xfa000000u, 0xbf8cc07fu, 0xf4000307u, 0xfa000000u, 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, + 0xe0701000u, 0x80010102u, 0xbf810000u}; + const std::vector descriptorCode{0xf4040280u, 0xfa000018u, 0xbf8cc07fu, 0xf4080105u, 0xfa000000u, 0xbf8cc07fu, 0x34040082u, 0xe0701000u, + 0x80010002u, 0xbf810000u}; + const auto slotPc = [](const IrResourcePlan& plan, std::uint32_t slot) { return plan.srtReads.at(slot).value->Flags().pc; }; + const auto faultBinding = [](const RecompileResult& result) { + return std::any_of(result.bindings.begin(), result.bindings.end(), [](const DescriptorBinding& binding) { return binding.role == DescriptorRole::FaultBuffer; }); + }; + const auto samePipeline = [](const RecompileResult& left, const RecompileResult& right) { + return left.variantId == right.variantId && left.specializationId == right.specializationId && left.PipelineVariantId() == right.PipelineVariantId() && left.spirv == right.spirv; + }; + + NestedRequest branch(branchCode, root.data()); + point(payloadAddress); + AgcDriver::ShaderMemory validMemory({}); + const auto valid = validMemory.Capture(branch.request); + require(valid->snapshot.srtPoison.empty() && valid->plan->guardedSrtSlots.size() == 1u && slotPc(*valid->plan, valid->plan->guardedSrtSlots[0]) == 0x2cu && GuardRecords(*valid), "guarded pointer: a mapped nested pointer was poisoned, or its read has no guard"); + require(valid->snapshot.flattenedSrt.at(valid->plan->guardedSrtSlots[0]) == payload, "guarded pointer: a mapped nested read does not hold its word"); + branch.request.context.memory = validMemory.Regions(); + const auto validResult = Recompile(branch.request, *valid); + require(validResult->poisonedSrtReads == 0u && faultBinding(*validResult) && validResult->bdaAbiVersion == BdaAbi::Version, "guarded pointer: a guarded program has no fault buffer"); + + for (const auto pointer : {std::uint64_t{0}, unmapped}) { + for (const auto condition : {1u, 0u}) { + point(pointer); + root[4] = condition; + branch.request.context.memory = {}; + AgcDriver::ShaderMemory memory({}); + const auto capture = memory.Capture(branch.request); + const auto& poison = capture->snapshot.srtPoison; + require(poison.size() == 1u && poison[0].pc == 0x2cu && poison[0].address == pointer && slotPc(*capture->plan, poison[0].slot) == 0x2cu, "guarded pointer: the nested read was not poisoned at its pc and address"); + require(GuardRecords(*capture), "guarded pointer: the poison record is not where the guard reads it"); + const auto regions = memory.Regions(); + require(!RegionsTouch(regions, pointer, 4u) && RegionsCover(regions, root.data() + 6, 8u), "guarded pointer: the captured regions are wrong"); + branch.request.context.memory = regions; + const auto result = Recompile(branch.request, *capture); + require(result->poisonedSrtReads == 1u && faultBinding(*result) && result->bdaAbiVersion == BdaAbi::Version, "guarded pointer: the poisoned capture does not report through the fault buffer"); + require(samePipeline(*result, *validResult), "guarded pointer: poison changed the variant, the specialization or the module"); + const auto replay = Recompile(branch.request); + require(replay.poisonedSrtReads == 1u && samePipeline(replay, *result), "guarded pointer: a replay of the captured regions did not reproduce the poison"); + verifyResult(*result, replay); +#if ANYPS5_ENABLE_SPIRV_TOOLS + static_cast(ValidateAndOptimizeSpirv(result->spirv, branch.request.target.vulkanVersion, branch.request.target.spirvVersion)); +#endif + } + } + root[4] = 1u; + + NestedRequest chain(chainCode, root.data()); + point(unmapped); + AgcDriver::ShaderMemory chainMemory({}); + const auto chained = chainMemory.Capture(chain.request); + const auto& chainPoison = chained->snapshot.srtPoison; + require(chainPoison.size() == 3u && GuardRecords(*chained), "guarded pointer: a pointer read through an unmapped pointer did not poison its two words and the read through it"); + for (const auto& poison : chainPoison) { + const auto own = slotPc(*chained->plan, poison.slot); + const bool pointerWord = own == 0x2cu && (poison.address == unmapped || poison.address == unmapped + 4u); + const bool throughPointer = own == 0x38u && poison.address == unmapped; + require(poison.pc == 0x2cu && (pointerWord || throughPointer), "guarded pointer: a derived poison does not name the inaccessible read"); + } + chain.request.context.memory = chainMemory.Regions(); + const auto chainResult = Recompile(chain.request, *chained); + indirect = 0; + point(static_cast(reinterpret_cast(&indirect))); + AgcDriver::ShaderMemory nullChainMemory({}); + const auto nullChain = nullChainMemory.Capture(chain.request); + require(nullChain->snapshot.srtPoison.size() == 1u && nullChain->snapshot.srtPoison[0].pc == 0x38u && nullChain->snapshot.srtPoison[0].address == 0u && GuardRecords(*nullChain), "guarded pointer: a null pointer read from memory did not poison the read through it"); + chain.request.context.memory = nullChainMemory.Regions(); + const auto nullChainResult = Recompile(chain.request, *nullChain); + indirect = payloadAddress; + AgcDriver::ShaderMemory mappedChainMemory({}); + const auto mappedChain = mappedChainMemory.Capture(chain.request); + require(mappedChain->snapshot.srtPoison.empty() && mappedChain->plan->guardedSrtSlots.size() == 3u && GuardRecords(*mappedChain), "guarded pointer: a mapped pointer chain was poisoned, or its reads have no guards"); + chain.request.context.memory = mappedChainMemory.Regions(); + const auto mappedChainResult = Recompile(chain.request, *mappedChain); + require(chainResult->poisonedSrtReads == 3u && nullChainResult->poisonedSrtReads == 1u && mappedChainResult->poisonedSrtReads == 0u, "guarded pointer: a chain result does not count its poisoned reads"); + require(samePipeline(*chainResult, *mappedChainResult) && samePipeline(*nullChainResult, *mappedChainResult), "guarded pointer: a chain's poison changed the variant, the specialization or the module"); + + NestedRequest descriptor(descriptorCode, root.data()); + point(0u); + AgcDriver::ShaderMemory descriptorMemory({}); + expectFailure([&] { static_cast(descriptorMemory.Capture(descriptor.request)); }, "null or misaligned address", "guarded pointer: a V# loaded through a null nested pointer was accepted"); + NestedRequest rootless(branchCode, nullptr); + AgcDriver::ShaderMemory rootlessMemory({}); + expectFailure([&] { static_cast(rootlessMemory.Capture(rootless.request)); }, "null or misaligned address", "guarded pointer: a null user-data pointer was accepted"); + point(0u); +#ifdef _WIN32 + VirtualFree(reserved, 0, MEM_RELEASE); +#else + munmap(reserved, Reserved); +#endif +} + +int main(int argc, char** argv) { try { using namespace ShaderRecompiler; + if (argc == 2 && std::string_view(argv[1]) == "--bindless") { + verifyBindlessTable(); + std::cout << "Bindless mapping, indexing capabilities and strict validation passed\n"; + return 0; + } + require(argc == 1, "unknown shader memory test arguments"); verifyRegisterSources(); verifyEvaluatedValues(); verifyPureFlatSlots(); @@ -1535,19 +2014,24 @@ int main() { verifyDescriptorPhis(); verifyProgramCounterRelativeData(); verifyLanesOutsideHostSubgroup(); + verifyHalfWaveReduction(); verifyMeshConfiguration(); verifyPixelInputs(); verifyPixelRequestSerialization(); verifyLegacyPixelRequests(); verifyPixelExportReplay(); verifyPixelParameterSlots(); + verifyF16PixelParameterSlots(); verifyComputedTexelOffsets(); + verifyShaderClockScopes(); + verifyInt64AtomicCapabilities(); verifyUnnormalizedSamplers(); verifyUnusedUnnormalizedSampler(); verifyWaveUniformValues(); verifyTwoLaneUniformValues(); verifyBdaReadFallbackFunctions(); verifyFunctionLdsBound(); + verifyGuardedNullPointers(); #if ANYPS5_ENABLE_SPIRV_TOOLS const std::vector minimalSpirv{ 0x07230203u, 0x00010000u, 0u, 5u, 0u, @@ -1581,6 +2065,7 @@ int main() { const std::array userData{static_cast(address), static_cast(address >> 32u)}; RecompileRequest request{}; request.shader = {ShaderStage::Vertex, 0x10000u, code, 0, {}}; + request.target = BufferTarget(); request.context.waveSize = 64; request.context.userDataBaseRegister = 8; request.context.userData = userData; @@ -1644,6 +2129,9 @@ int main() { auto srgb = uncached; srgb.target.srgbDecodeFormats = 3u; require(RequestSerializer{}.Deserialize(RequestSerializer{}.Serialize(srgb)).request.target.srgbDecodeFormats == 3u, "the sRGB formats decoded in the shader were lost in serialization"); + auto narrowClock = uncached; + narrowClock.target.narrowSubgroupClock = true; + require(RequestSerializer{}.Deserialize(RequestSerializer{}.Serialize(narrowClock)).request.target.narrowSubgroupClock, "the narrow subgroup clock was lost in serialization"); auto changedLayout = request; changedLayout.layout.pushConstantSizeBytes = 64; require(!Recompile(changedLayout).cacheHit, "binding layout change reused an incompatible variant"); @@ -1730,8 +2218,18 @@ int main() { expectFailure([&] { static_cast(PrepareResourceProgram(request)); }, "shader user data exceeds the scalar register bank", "user data overran scalar register bank"); request.context.userDataBaseRegister = 8; request.context.memory = {}; + AgcDriver::ShaderMemory guarded({}); + const auto poisoned = guarded.Capture(request); + require(poisoned->snapshot.srtPoison.size() == 1u && poisoned->snapshot.srtPoison[0].pc == 8u && poisoned->snapshot.srtPoison[0].address == 0u && GuardRecords(*poisoned), "null nested pointer was not poisoned at its read"); + const auto guardedRegions = guarded.Regions(); + request.context.memory = guardedRegions; + const auto poisonedVertex = Recompile(request, *poisoned); + require(poisonedVertex->poisonedSrtReads == 1u && poisonedVertex->bdaAbiVersion == BdaAbi::Version && poisonedVertex->PipelineVariantId() == first.PipelineVariantId() && std::any_of(poisonedVertex->bindings.begin(), poisonedVertex->bindings.end(), [](const DescriptorBinding& binding) { return binding.role == DescriptorRole::FaultBuffer; }), "a vertex shader reading through a null nested pointer has no fault buffer or another variant"); + request.context.memory = {}; + const std::array nullUserData{}; + request.context.userData = nullUserData; AgcDriver::ShaderMemory invalid({}); - expectFailure([&] { invalid.Capture(request); }, "null or misaligned address", "null nested pointer was accepted"); + expectFailure([&] { invalid.Capture(request); }, "null or misaligned address", "null user-data pointer was accepted"); std::cout << "Shader memory capture, strict validation and deterministic replay passed\n"; return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/SpirvNarrowConstantStore.cpp b/core/libs/prx/libSceAgcDriver/tests/SpirvNarrowConstantStore.cpp new file mode 100644 index 000000000..daa5ac849 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/SpirvNarrowConstantStore.cpp @@ -0,0 +1,108 @@ +#include "BdaAbi.hpp" +#include "NarrowConstantStoreFixture.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#if ANYPS5_ENABLE_SPIRV_TOOLS +#include +#endif + +namespace { + +using namespace ShaderRecompiler; + +void Check(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +void CheckStorageOnlyModule(const RecompileResult& result, std::string_view name) { + const auto& words = result.spirv.Words(); + Check(words.size() >= 5 && words[0] == spv::MagicNumber, std::string(name) + ": missing SPIR-V module"); + bool byteType = false; + bool store = false; + for (std::size_t offset = 5; offset < words.size();) { + const auto count = words[offset] >> spv::WordCountShift; + const auto opcode = words[offset] & spv::OpCodeMask; + Check(count != 0 && count <= words.size() - offset, std::string(name) + ": malformed SPIR-V instruction"); + if (opcode == spv::OpCapability && count == 2) { + Check(words[offset + 1] != spv::CapabilityInt8, std::string(name) + ": introduced unsupported Int8 arithmetic"); + } + if (opcode == spv::OpTypeInt && count == 4 && words[offset + 2] == 8) byteType = true; + if (opcode == spv::OpStore) store = true; + offset += count; + } + Check(byteType && store, std::string(name) + ": lost the byte storage regression fixture"); +#if ANYPS5_ENABLE_SPIRV_TOOLS + std::string diagnostics; + spvtools::SpirvTools validator(SPV_ENV_VULKAN_1_1); + validator.SetMessageConsumer([&](spv_message_level_t, const char*, const spv_position_t&, const char* message) { + if (message != nullptr) diagnostics += std::string(message) + '\n'; + }); + const bool valid = validator.Validate(words); + Check(valid, std::string(name) + ": emitted invalid SPIR-V:\n" + diagnostics); +#endif +} + +SpirvTarget StorageOnlyTarget() { + static constexpr std::array capabilities{ + spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, + }; + static constexpr std::array extensions{ + "SPV_EXT_descriptor_indexing", "SPV_KHR_8bit_storage", "SPV_KHR_float_controls", "SPV_KHR_physical_storage_buffer", + }; + return {0x00401000u, 0x00010300u, 64, BdaAbi::Version, capabilities, extensions, false, + {1024, 1024, 64}, 1024, 32768, {}, {}}; +} + +void CheckVertexReproducer() { + alignas(256) static constexpr std::array code{ + 0xbe801f00u, 0x800000ffu, 0x34u, 0xdc7519cbu, 0xb0020010u, 0x10005004u, 0xf4200100u, + 0xfa000000u, 0xbf8cc07fu, 0x7e0049c8u, 0xf80008cfu, 0u, 0xbf810000u, 0x3f800000u, + }; + const std::span words(code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(words)}}}; + RecompileRequest request{}; + request.shader = {ShaderStage::Vertex, reinterpret_cast(code.data()), words, 0, {}}; + request.target = StorageOnlyTarget(); + request.layout = {0, 0, 0, 128}; + request.context.waveSize = 64; + request.context.memory = memory; + request.context.userDataBaseRegister = 8; + request.context.vertex = ShaderVertexStageInfo{}; + request.useCache = false; + CheckStorageOnlyModule(Recompile(request), "issue #1276 vertex"); +} + +void CheckComputeStores(std::uint32_t waveSize) { + const std::span code(NarrowConstantStoreFixture::Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const std::array userData{0x10000u, 0u}; + const ShaderComputeStageInfo compute{{NarrowConstantStoreFixture::Threads, 1, 1}, 0, {false, false, false}, false, 1, {}}; + RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory}, + StorageOnlyTarget(), {0, 0, 0, 128}, std::nullopt, false, + }; + CheckStorageOnlyModule(Recompile(request), "constant stores wave" + std::to_string(waveSize)); +} + +} + +int main() { + try { + CheckVertexReproducer(); + CheckComputeStores(32); + CheckComputeStores(64); + std::puts("storage-only constant store compilation passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/SpirvOptimizer.cpp b/core/libs/prx/libSceAgcDriver/tests/SpirvOptimizer.cpp new file mode 100644 index 000000000..c511be056 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/SpirvOptimizer.cpp @@ -0,0 +1,294 @@ +#include "SpirvBackend/SpirvOptimizer.hpp" +#include "SpirvBackend/SpirvSpecialization.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int failures = 0; +constexpr std::uint32_t Vulkan11 = 0x00401000u; +constexpr std::uint32_t Spirv11 = 0x00010100u; +constexpr std::uint32_t Spirv13 = 0x00010300u; + +void check(bool condition, const std::string& message) { + if (!condition) { + std::fprintf(stderr, "%s\n", message.c_str()); + ++failures; + } +} + +struct NarrowType { + const char* name; + unsigned width; + bool signedInteger; + bool floating; + const char* arithmeticCapability; + const char* unrelatedCapability; +}; + +constexpr std::array Types{ + NarrowType{"u8", 8, false, false, "Int8", "Int16"}, + NarrowType{"i8", 8, true, false, "Int8", "Int16"}, + NarrowType{"u16", 16, false, false, "Int16", "Int8"}, + NarrowType{"i16", 16, true, false, "Int16", "Int8"}, + NarrowType{"f16", 16, false, true, "Float16", "Int8"}, +}; + +enum class ArithmeticSupport { StorageOnly, Explicit, ImpliedInt8 }; + +std::string moduleSource(const NarrowType& type, bool vector, ArithmeticSupport support) { + const auto width = std::to_string(type.width); + const std::string typeOp = type.floating ? "OpTypeFloat " : "OpTypeInt "; + const std::string signedness = type.floating ? "" : (type.signedInteger ? " 1" : " 0"); + const std::string conversion = type.floating ? "OpFConvert" : (type.signedInteger ? "OpSConvert" : "OpUConvert"); + const std::string wide = type.floating || type.signedInteger ? "%wide" : "%index"; + std::string text = "OpCapability Shader\nOpCapability "; + text += type.width == 8 ? "StorageBuffer8BitAccess\n" : "StorageBuffer16BitAccess\n"; + text += "OpCapability " + std::string(type.unrelatedCapability) + "\n"; + if (support == ArithmeticSupport::Explicit) text += "OpCapability " + std::string(type.arithmeticCapability) + "\n"; + if (support == ArithmeticSupport::ImpliedInt8) { + text += "OpCapability DotProductInput4x8BitKHR\nOpExtension \"SPV_KHR_integer_dot_product\"\n"; + } + text += type.width == 8 ? "OpExtension \"SPV_KHR_8bit_storage\"\n" : "OpExtension \"SPV_KHR_16bit_storage\"\n"; + text += R"(OpExtension "SPV_KHR_storage_buffer_storage_class" +OpMemoryModel Logical GLSL450 +OpEntryPoint GLCompute %main "main" +OpExecutionMode %main LocalSize 1 1 1 +OpDecorate %block Block +OpMemberDecorate %block 0 Offset 0 +OpDecorate %buffer DescriptorSet 0 +OpDecorate %buffer Binding 0 +%void = OpTypeVoid +%function = OpTypeFunction %void +%index = OpTypeInt 32 0 +%zero = OpConstant %index 0 +)"; + if (wide == "%wide") text += "%wide = " + typeOp + "32" + signedness + "\n"; + text += "%small = " + typeOp + width + signedness + "\n"; + text += "%zeroValue = OpConstant " + wide + " 0\n"; + text += "%value = OpConstant " + wide + " " + std::string(type.floating ? "1.5" : (type.signedInteger ? "-257" : "257")) + "\n"; + if (vector) { + text += "%wideValue = OpTypeVector " + wide + " 2\n%narrow = OpTypeVector %small 2\n"; + text += "%input = OpConstantComposite %wideValue %zeroValue %value\n"; + } + const std::string narrow = vector ? "%narrow" : "%small"; + text += "%block = OpTypeStruct " + narrow + "\n"; + text += "%blockPointer = OpTypePointer StorageBuffer %block\n"; + text += "%pointer = OpTypePointer StorageBuffer " + narrow + "\n"; + text += R"(%buffer = OpVariable %blockPointer StorageBuffer +%main = OpFunction %void None %function +%entry = OpLabel +%address = OpAccessChain %pointer %buffer %zero +%dead = OpIAdd %index %zero %zero +)"; + if (vector) { + text += "%converted = " + conversion + " " + narrow + " %input\n"; + text += "OpStore %address %converted\n"; + } else { + text += "%convertedZero = " + conversion + " " + narrow + " %zeroValue\n"; + text += "OpStore %address %convertedZero\n"; + text += "%converted = " + conversion + " " + narrow + " %value\n"; + text += "OpStore %address %converted\n"; + } + return text + "OpReturn\nOpFunctionEnd\n"; +} + +std::vector assemble(const std::string& source) { + spvtools::SpirvTools tools(SPV_ENV_VULKAN_1_1); + tools.SetMessageConsumer([](spv_message_level_t, const char*, const spv_position_t&, const char* message) { + std::fprintf(stderr, "SPIRV-Tools: %s\n", message); + }); + std::vector words; + check(tools.Assemble(source, &words), "test shader did not assemble"); + return words; +} + +std::size_t opcodeCount(const std::vector& words, spv::Op opcode) { + std::size_t count = 0; + for (std::size_t offset = 5; offset < words.size(); offset += words[offset] >> spv::WordCountShift) { + if ((words[offset] & spv::OpCodeMask) == static_cast(opcode)) ++count; + } + return count; +} + +std::set capabilities(const std::vector& words) { + std::set result; + for (std::size_t offset = 5; offset < words.size(); offset += words[offset] >> spv::WordCountShift) { + if ((words[offset] & spv::OpCodeMask) == spv::OpCapability) result.insert(words[offset + 1]); + } + return result; +} + +std::vector swapByteOrder(std::vector words) { + for (auto& word : words) { + word = (word >> 24u) | ((word & 0x00ff0000u) >> 8u) | + ((word & 0x0000ff00u) << 8u) | (word << 24u); + } + return words; +} + +void testNarrowConversions(bool optimizationEnabled) { + spvtools::SpirvTools validator(SPV_ENV_VULKAN_1_1); + for (const auto& type : Types) { + for (const bool vector : {false, true}) { + for (const auto support : {ArithmeticSupport::StorageOnly, ArithmeticSupport::Explicit, ArithmeticSupport::ImpliedInt8}) { + if (support == ArithmeticSupport::ImpliedInt8 && (type.width != 8 || !vector)) continue; + const bool arithmetic = support != ArithmeticSupport::StorageOnly; + const std::string name = std::string(type.name) + (vector ? " vector" : " scalar") + + (support == ArithmeticSupport::ImpliedInt8 ? " implied Int8" : (arithmetic ? " arithmetic" : " storage only")); + auto native = assemble(moduleSource(type, vector, support)); + for (const auto& [version, swapped] : {std::pair{Spirv13, false}, std::pair{Spirv11, false}, std::pair{Spirv11, true}}) { + native[1] = version; + const auto input = swapped ? swapByteOrder(native) : native; + const auto label = name + (version == Spirv13 ? " SPIR-V 1.3" : " SPIR-V 1.1") + (swapped ? " swapped" : " native"); + if (!validator.Validate(input)) { + check(false, label + ": invalid test input"); + continue; + } + try { + const auto output = ShaderRecompiler::ValidateAndOptimizeSpirv(input, Vulkan11, Spirv13, false); + if (!validator.Validate(output)) { + check(false, label + ": optimizer returned invalid SPIR-V"); + continue; + } + const auto normalized = output.front() == spv::MagicNumber ? output : swapByteOrder(output); + check(capabilities(normalized) == capabilities(native), label + ": optimizer changed device capabilities"); + check(opcodeCount(normalized, spv::OpStore) == (vector ? 1u : 2u), label + ": optimizer lost buffer stores"); + if (!optimizationEnabled) { + check(output == input, label + ": none mode changed the shader"); + } else { + check(opcodeCount(normalized, spv::OpIAdd) == 0, label + ": optimizer did not remove dead arithmetic"); + if (arithmetic && !type.floating) { + const auto conversion = type.signedInteger ? spv::OpSConvert : spv::OpUConvert; + check(opcodeCount(normalized, conversion) == 0, label + ": safe constant folding was disabled"); + } + } + } catch (const std::exception& error) { + check(false, label + ": " + error.what()); + } + } + } + } + } +} + +void expectRejected(const std::vector& words, std::uint32_t vulkanVersion, std::uint32_t spirvVersion, const char* reason, const char* diagnostic = nullptr) { + try { + static_cast(ShaderRecompiler::ValidateAndOptimizeSpirv(words, vulkanVersion, spirvVersion, false)); + check(false, std::string("accepted ") + reason); + } catch (const std::exception& error) { + if (diagnostic != nullptr) check(std::string(error.what()).starts_with(diagnostic), std::string(reason) + ": unexpected rejection: " + error.what()); + } +} + +void testSpecializedSelectionExit() { + for (const bool useSwitch : {false, true}) { + const std::string source = std::string(R"(OpCapability Shader +OpMemoryModel Logical GLSL450 +OpEntryPoint GLCompute %main "main" %index +OpExecutionMode %main LocalSize 8 1 1 +OpDecorate %index BuiltIn LocalInvocationIndex +%void = OpTypeVoid +%bool = OpTypeBool +%uint = OpTypeInt 32 0 +%input = OpTypePointer Input %uint +%private = OpTypePointer Private %uint +%function = OpTypeFunction %void +%zero = OpConstant %uint 0 +%true = OpConstantTrue %bool +%index = OpVariable %input Input +%output = OpVariable %private Private +%main = OpFunction %void None %function +%entry = OpLabel +%lane = OpLoad %uint %index +%condition = OpIEqual %bool %lane %zero +OpSelectionMerge %exit None +)") + (useSwitch ? "OpSwitch %zero %body 1 %dead\n" : "OpBranchConditional %true %body %dead\n") + R"(%body = OpLabel +OpBranchConditional %condition %exit %nested +%nested = OpLabel +OpSelectionMerge %join None +OpBranchConditional %condition %write %join +%write = OpLabel +OpStore %output %lane +OpBranch %join +%join = OpLabel +OpBranch %exit +%dead = OpLabel +OpStore %output %zero +OpBranch %exit +%exit = OpLabel +OpReturn +OpFunctionEnd +)"; + try { + const auto words = assemble(source); + static_cast(ShaderRecompiler::ValidateAndOptimizeSpirv(words, Vulkan11, Spirv13, false, false)); + const auto specialized = ShaderRecompiler::SpecializeSpirv(words); + static_cast(ShaderRecompiler::ValidateAndOptimizeSpirv(specialized, Vulkan11, Spirv13, false, false)); + check(opcodeCount(specialized, spv::OpStore) == 1u, "specialization retained the dead selection arm"); + check(ShaderRecompiler::SpecializeSpirv(specialized) == specialized, "selection exit specialization is not stable"); + } catch (const std::exception& error) { + check(false, std::string("specialized selection exit: ") + error.what()); + } + } +} + +void testInvalidInputIsRejected() { + const auto valid = assemble(moduleSource(Types.front(), false, ArithmeticSupport::StorageOnly)); + expectRejected(valid, 0x00400000u, Spirv13, "SPIR-V 1.3 with Vulkan 1.0"); + expectRejected(valid, Vulkan11, 0x00010200u, "a module newer than the requested SPIR-V version"); + expectRejected(valid, 0u, Spirv13, "an unsupported Vulkan version"); + auto invalid = moduleSource(Types.front(), false, ArithmeticSupport::StorageOnly); + invalid.insert(invalid.find("%main = OpFunction"), "%illegal = OpConstantNull %small\n"); + expectRejected(assemble(invalid), Vulkan11, Spirv13, "an illegal storage-only narrow constant even when unused"); + const auto rejectMalformed = [](const std::vector& words, const char* reason) { + auto native = words; + if (native.size() > 1) native[1] = Spirv11; + constexpr auto diagnostic = "SPIR-V validation before optimization failed:"; + expectRejected(native, Vulkan11, Spirv13, reason, diagnostic); + expectRejected(swapByteOrder(native), Vulkan11, Spirv13, (std::string(reason) + " (swapped)").c_str(), diagnostic); + }; + for (std::size_t length = 0; length < 5; ++length) { + rejectMalformed(std::vector(valid.begin(), valid.begin() + length), "an incomplete module header"); + } + auto malformed = valid; + malformed[0] = 0; + rejectMalformed(malformed, "an invalid module magic"); + malformed = valid; + malformed[5] = spv::OpCapability; + rejectMalformed(malformed, "a zero-length SPIR-V instruction"); + malformed[5] = (0xffffu << spv::WordCountShift) | spv::OpCapability; + rejectMalformed(malformed, "an instruction extending beyond the module"); + for (const auto& instruction : { + std::vector{(1u << spv::WordCountShift) | spv::OpCapability}, + std::vector{(3u << spv::WordCountShift) | spv::OpTypeInt, 1u, 8u}, + std::vector{(2u << spv::WordCountShift) | spv::OpTypeFloat, 1u}}) { + malformed.assign(valid.begin(), valid.begin() + 5); + malformed.insert(malformed.end(), instruction.begin(), instruction.end()); + rejectMalformed(malformed, "truncated capability or type operands"); + } +} + +} + +int main() { + const char* mode = std::getenv("APS5_SPIRV_OPT"); + const bool optimizationEnabled = mode == nullptr || std::string(mode) != "none"; + testNarrowConversions(optimizationEnabled); + testInvalidInputIsRejected(); + testSpecializedSelectionExit(); + if (failures != 0) { + std::fprintf(stderr, "%d optimizer check(s) failed\n", failures); + return 1; + } + std::puts("SPIR-V optimizer checks passed"); + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/SubgroupClock.cpp b/core/libs/prx/libSceAgcDriver/tests/SubgroupClock.cpp new file mode 100644 index 000000000..8adf998a9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/SubgroupClock.cpp @@ -0,0 +1,40 @@ +#include "prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp" +#include +#include + +namespace { + +struct Case { + VkDriverId driver; + std::string_view name; + bool narrow; +}; + +constexpr Case Cases[]{ + {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 6750 XT (RADV NAVI22)", true}, + {VK_DRIVER_ID_MESA_RADV, "AMD Ryzen 5 7600X 6-Core Processor (RADV RAPHAEL_MENDOCINO)", true}, + {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 7900 XTX (RADV NAVI31)", true}, + {VK_DRIVER_ID_MESA_RADV, "AMD Radeon 890M (RADV STRIX1)", true}, + {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 9070 XT (RADV GFX1201)", false}, + {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 5700 XT (RADV NAVI10)", false}, + {VK_DRIVER_ID_MESA_RADV, "AMD BC-250 (RADV GFX1013)", false}, + {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 6750 XT (RADV NAVI22", false}, + {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 6750 XT (RADV NAVI2)", false}, + {VK_DRIVER_ID_MESA_RADV, "", false}, + {VK_DRIVER_ID_AMD_PROPRIETARY, "AMD Radeon RX 6750 XT (RADV NAVI22)", false}, + {VK_DRIVER_ID_AMD_OPEN_SOURCE, "AMD Radeon RX 6750 XT", false}, + {VK_DRIVER_ID_NVIDIA_PROPRIETARY, "NVIDIA GeForce RTX 5070 Ti", false}, +}; + +} + +int main() { + int failures = 0; + for (const auto& item : Cases) { + if (AgcDriver::NarrowSubgroupClock(item.driver, item.name) != item.narrow) { + std::fprintf(stderr, "driver %d \"%.*s\": expected %s subgroup clock\n", static_cast(item.driver), static_cast(item.name.size()), item.name.data(), item.narrow ? "a narrow" : "a full"); + ++failures; + } + } + return failures == 0 ? 0 : 1; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp index 3419d7839..e774ae46e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "execution/VulkanTestDevice.hpp" #include "prx/libSceAgcDriver/Execution/include/QueueState.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" #include "prx/libc/include/Shutdown.hpp" @@ -15,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -113,6 +115,7 @@ void testClearState() { graphics.ClearContext(); check(graphics.context == AgcDriver::InitialContextRegisters(), "CLEAR_STATE retained context registers"); check(graphics.shader == shader && graphics.userConfig == userConfig, "CLEAR_STATE reset unrelated registers"); + check(graphics.context.count(0x1b3) == 1 && graphics.context.at(0x1b3) == 0 && graphics.context.count(0x1b4) == 1 && graphics.context.at(0x1b4) == 0, "CLEAR_STATE left SPI_PS_INPUT_ENA/ADDR unset"); graphics.context.emplace(0x10, 31); graphics.ClearContext(); check(graphics.context == AgcDriver::InitialContextRegisters(), "repeated CLEAR_STATE retained context registers"); @@ -193,6 +196,14 @@ void testEndOfPipeInterrupts() { check(sceAgcDriverSubmitDcb(&packet) == 0, "plain release submit failed"); AgcDriverWaitIdle_nid_postfix(); check(owner->GetTriggeredEvents(events.data(), 2) == 0, "release without INT_SEL raised an interrupt"); + alignas(8) static volatile std::uint64_t label = 0; + const auto labelAddress = reinterpret_cast(&label); + words = {0xc0064900, 0x528, (3u << 29u) | (3u << 24u) | (1u << 16u), static_cast(labelAddress), static_cast(static_cast(labelAddress) >> 32u), 0x89abcdefu, 0x01234567u, 0}; + check(sceAgcDriverSubmitDcb(&packet) == 0, "send-data release submit failed"); + AgcDriverWaitIdle_nid_postfix(); + check(label != 0, "send-data release did not write its label"); + check(owner->GetTriggeredEvents(events.data(), 2) == 0, "release with INT_SEL send data after write confirm raised an interrupt"); + words = {0xc0064900, 0, 1u << 24u, 0, 0, 0, 0, 0}; check(sceAgcDriverDeleteEqEvent(eq, 0) == 0, "graphics event deletion failed"); expectFailure([&] { sceAgcDriverDeleteEqEvent(eq, 0); }); words[2] = 1u << 24u; @@ -431,15 +442,28 @@ void testMultiSubmissions() { void testShaderHeaderAlignment() { alignas(256) static const std::array code{0xbf810000}; - alignas(8) static std::array storage{}; + struct Header { + Shader shader{}; + std::array registers{}; + ShaderSpecialRegs specials{}; + }; + alignas(8) static std::array storage{}; Shader shader{}; shader.file_header = 0x34333231; shader.version = 0x18; - shader.header_size = sizeof(Shader); + shader.header_size = sizeof(Header); shader.shader_size = sizeof(code); shader.code = code.data(); const auto at = [](std::size_t offset, const Shader& fields) { - std::memcpy(storage.data() + offset, &fields, sizeof(fields)); + Header header; + header.shader = fields; + const auto address = reinterpret_cast(fields.code); + header.registers = {{{0x20c, static_cast(address >> 8u)}, {0x20d, static_cast(address >> 40u)}, {0x207, 1}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 0}}}; + header.specials.dispatch_modifier = 0x8000u; + header.shader.sh_registers = reinterpret_cast(storage.data() + offset + offsetof(Header, registers)); + header.shader.num_sh_registers = header.registers.size(); + header.shader.specials = reinterpret_cast(storage.data() + offset + offsetof(Header, specials)); + std::memcpy(storage.data() + offset, &header, sizeof(header)); return reinterpret_cast(storage.data() + offset); }; for (const std::size_t offset : {0, 4, 1}) AgcDriverRegisterShader_nid_postfix(at(offset, shader)); @@ -457,6 +481,41 @@ void testShaderHeaderAlignment() { refused(expectFailure([&] { AgcDriverRegisterShader_nid_postfix(at(4, truncated)); }), "smaller than its fixed fields"); } +void testRegisteredFloatMode() { + using AgcDriver::DriverDetail::RegisteredFloatMode; + using AgcDriver::DriverDetail::ShaderSnapshot; + using AgcDriver::DriverDetail::RegisteredShaderState; + struct Stage { + std::uint8_t type; + std::uint32_t rsrc1; + std::uint32_t fp16OverflowBit; + }; + constexpr std::array stages{{{0, 0x212, 26}, {1, 0x00a, 29}, {2, 0x08a, 31}, {4, 0x08a, 31}, {6, 0x08a, 31}, {5, 0x10a, 30}, {7, 0x10a, 30}}}; + constexpr std::array otherBits{26, 29, 30, 31}; + for (const auto& stage : stages) { + const auto mode = [&](std::uint32_t value) { + ShaderSnapshot snapshot{0x20000, 0, stage.type, {}, {}}; + auto state = std::make_shared(); + state->shader.emplace(stage.rsrc1, value); + snapshot.registeredState = state; + return RegisteredFloatMode(snapshot); + }; + const auto name = "stage type " + std::to_string(stage.type); + const auto astro = mode((0xc0u << 12u) | (1u << 21u) | 0x3fu); + check(astro.has_value() && astro->floatMode == 0xc0u && astro->dx10Clamp && !astro->ieeeMode && !astro->fp16Overflow, (name + ": FLOAT_MODE 0xc0 with DX10_CLAMP decoded wrong").c_str()); + const auto ieee = mode(1u << 23u); + check(ieee && ieee->ieeeMode && ieee->floatMode == 0u && !ieee->dx10Clamp, (name + ": IEEE_MODE decoded wrong").c_str()); + check(mode(1u << stage.fp16OverflowBit)->fp16Overflow, (name + ": FP16_OVFL not read from bit " + std::to_string(stage.fp16OverflowBit)).c_str()); + for (const auto bit : otherBits) { + if (bit != stage.fp16OverflowBit) check(!mode(1u << bit)->fp16Overflow, (name + ": bit " + std::to_string(bit) + " read as FP16_OVFL").c_str()); + } + ShaderSnapshot missing{0x20000, 0, stage.type, {}, {}}; + check(!RegisteredFloatMode(missing).has_value(), (name + ": a snapshot without registered state has a float mode").c_str()); + missing.registeredState = std::make_shared(); + check(!RegisteredFloatMode(missing).has_value(), (name + ": a missing RSRC1 has a float mode").c_str()); + } +} + void testWorkerFailure() { std::array words{0xc0031500, 1, 1, 1, 0x41}; Packet packet{words.data(), static_cast(words.size()), 0, {}}; @@ -477,6 +536,8 @@ void testWorkerFailure() { int main() { try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; alignas(256) std::array rawCode{}; rawCode.fill(0xbf800000); rawCode[0] = 0xbe8003ff; @@ -555,6 +616,7 @@ int main() { testWaitFreeSubmissionTheCpuWaitsFor(); testMultiSubmissions(); testShaderHeaderAlignment(); + testRegisteredFloatMode(); testWorkerFailure(); check(expectFailure([] { LibcRunShutdown_nid_postfix(); }).find("required shader register") != std::string::npos, "shutdown lost worker failure"); std::puts("AGC driver submit tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp index 95b4b4c38..0db9767f1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp @@ -77,6 +77,19 @@ void requireThickAddresses(const GuestTextureResource& resource, std::uint32_t b } } +void requireBlockVolumeLayout(TextureTileMode tileMode, std::uint32_t format, std::uint32_t blockFormat, std::uint32_t width, std::uint32_t height, std::uint32_t depth, std::uint32_t mipCount, const std::string& what) { + const auto compressed = DescribeSurface(thickVolume(tileMode, format, width, height, depth, mipCount)); + const auto blocks = DescribeSurface(thickVolume(tileMode, blockFormat, (width + 3u) / 4u, (height + 3u) / 4u, depth, mipCount)); + Require(compressed.thick == blocks.thick && compressed.guestBytes == blocks.guestBytes && compressed.layerBytes == blocks.layerBytes && compressed.blockDepth == blocks.blockDepth, what + ": the volume is not tiled as a volume of its blocks"); + for (std::uint32_t level = 0; level < mipCount; ++level) { + const auto& mip = compressed.mips[level]; + const auto& block = blocks.mips[level]; + Require(mip.blocksPerRow == block.blocksPerRow && mip.pitchBytes == block.pitchBytes && mip.tiledOffset == block.tiledOffset && mip.tiledSize == block.tiledSize && mip.tail == block.tail && mip.tailX == block.tailX && mip.tailY == block.tailY, what + ": mip " + std::to_string(level) + " is not tiled as the same mip of a volume of its blocks"); + Require(mip.width == std::max((std::max(width >> level, 1u) + 3u) / 4u, 1u) && mip.height == std::max((std::max(height >> level, 1u) + 3u) / 4u, 1u), what + ": mip " + std::to_string(level) + " is not sized in blocks"); + Require(mip.linearSize == static_cast(mip.pitchBytes) * mip.height, what + ": mip " + std::to_string(level) + " linear region does not cover its block rows"); + } +} + void requireThinAddresses(TextureTileMode tileMode, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t mipCount, std::uint32_t slices, std::uint64_t guestBytes, std::uint64_t layerBytes, std::span expected, const std::string& what) { GuestTextureResource resource{}; resource.width = width; @@ -248,6 +261,9 @@ void RunTextureTilingTests() { { constexpr ElementAddress chainInTail[] = {{0, 0, 0, 0, 0x800}, {0, 7, 7, 7, 0xffc}, {0, 4, 2, 4, 0xe20}, {0, 2, 7, 0, 0x968}, {0, 7, 0, 7, 0xed4}, {1, 0, 0, 0, 0x300}, {1, 3, 3, 3, 0x3fc}, {1, 2, 1, 2, 0x3c8}, {1, 1, 3, 0, 0x32c}, {1, 3, 0, 3, 0x3d4}}; requireThickAddresses(thickVolume(TextureTileMode::kStandard4KB, 56, 8, 8, 8, 2), 4, 0, 4096, chainInTail, "SW_4KB_S 32 bpp 8x8x8, 2 levels"); + requireBlockVolumeLayout(TextureTileMode::kStandard4KB, 175, 71, 128, 128, 128, 8, "SW_4KB_S BC4 128x128x128, 8 levels"); + requireBlockVolumeLayout(TextureTileMode::kStandard64KB, 181, 77, 130, 66, 12, 6, "SW_64KB_S BC7 130x66x12, 6 levels"); + requireBlockVolumeLayout(TextureTileMode::kS64KBX, 169, 71, 60, 36, 20, 5, "SW_64KB_S_X BC1 60x36x20, 5 levels"); constexpr ElementAddress unevenTail[] = {{0, 0, 0, 0, 0x6000}, {0, 32, 19, 11, 0x1f0b8}, {0, 16, 6, 6, 0x85a0}, {0, 11, 19, 0, 0xc06c}, {0, 32, 0, 11, 0x1a090}, {1, 0, 0, 0, 0x3000}, {1, 15, 9, 5, 0x4e5c}, {1, 8, 3, 3, 0x40b8}, {1, 5, 9, 0, 0x3a0c}, {1, 15, 0, 5, 0x4654}, {2, 0, 0, 0, 0x1000}, {2, 7, 4, 2, 0x13c4}, {2, 4, 1, 1, 0x1218}, {2, 2, 4, 0, 0x1140}, {2, 7, 0, 2, 0x12c4}, {3, 0, 0, 0, 0x800}, {3, 3, 1, 0, 0x84c}, {3, 2, 0, 0, 0x840}, {3, 1, 1, 0, 0x80c}, {3, 3, 0, 0, 0x844}, {4, 0, 0, 0, 0x300}, {4, 1, 0, 0, 0x304}, {4, 0, 0, 0, 0x300}, {5, 0, 0, 0, 0x200}}; requireThickAddresses(thickVolume(TextureTileMode::kStandard4KB, 56, 33, 20, 12, 6), 4, 3, 131072, unevenTail, "SW_4KB_S 32 bpp 33x20x12, 6 levels"); constexpr ElementAddress wideTail[] = {{0, 0, 0, 0, 0x20000}, {0, 39, 23, 19, 0xb0bf8}, {0, 20, 8, 10, 0x2e280}, {0, 13, 23, 0, 0x41b28}, {0, 39, 0, 19, 0x902d8}, {1, 0, 0, 0, 0x10000}, {1, 19, 11, 9, 0x1e178}, {1, 10, 4, 5, 0x11c50}, {1, 6, 11, 0, 0x14360}, {1, 19, 0, 9, 0x1a058}, {2, 0, 0, 0, 0x8000}, {2, 9, 5, 4, 0x9c28}, {2, 5, 2, 2, 0x8388}, {2, 3, 5, 0, 0x8868}, {2, 9, 0, 4, 0x9408}, {3, 0, 0, 0, 0x4000}, {3, 4, 2, 1, 0x4310}, {3, 2, 1, 1, 0x4070}, {3, 1, 2, 0, 0x4108}, {3, 4, 0, 1, 0x4210}, {4, 0, 0, 0, 0x1000}, {4, 1, 0, 0, 0x1008}, {5, 0, 0, 0, 0xa00}}; diff --git a/core/libs/prx/libSceAgcDriver/tests/WaveControlDecode.cpp b/core/libs/prx/libSceAgcDriver/tests/WaveControlDecode.cpp new file mode 100644 index 000000000..e75cde7a9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/WaveControlDecode.cpp @@ -0,0 +1,137 @@ +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +bool isVector(const RdnaOperand& operand, std::uint32_t reg) { + return operand.kind == RdnaOperandKind::VectorRegister && operand.reg == reg; +} + +void verifyDecode() { + const std::vector code{ + 0xbf8d0000u, 0xbf8d0001u, 0xbf911234u, 0xbe802204u, + 0xd8661234u, 0x00000001u, 0xd86a0000u, 0x00000000u, 0xd86e0000u, 0x00000002u, + 0xd8720000u, 0x00000000u, 0xd8620000u, 0x00000000u, 0xd8760000u, 0x00000003u, + 0xd8fe0004u, 0x05000001u, + 0xbf9f0000u, 0xbf9f0000u, 0xffffffffu, + }; + const auto program = RdnaInstructionDecoder{}.Decode(code); + const auto& decoded = program.instructions; + require(decoded.size() == 12u, "decoding did not stop at the first s_code_end: " + std::to_string(decoded.size()) + " instructions"); + + require(decoded[0].op == RdnaOpcode::SSethalt && decoded[0].source0.value == 0u && decoded[0].sourceCount == 1u, "s_sethalt 0"); + require(decoded[1].op == RdnaOpcode::SSethalt && decoded[1].source0.value == 1u, "s_sethalt 1"); + require(decoded[2].op == RdnaOpcode::SSendmsghalt && decoded[2].source0.value == 0x1234u && decoded[2].sourceCount == 1u, "s_sendmsghalt 0x1234"); + require(decoded[3].op == RdnaOpcode::SRfeB64 && decoded[3].destination.kind == RdnaOperandKind::Null && decoded[3].sourceCount == 1u && + decoded[3].source0.kind == RdnaOperandKind::ScalarRegister && decoded[3].source0.reg == 4u, "s_rfe_b64 s[4:5]"); + + const std::array, 6> gws{{ + {RdnaOpcode::DsGwsInit, 1u, 0x1234u, "ds_gws_init v1 offset:4660 gds"}, + {RdnaOpcode::DsGwsSemaV, 0u, 0u, "ds_gws_sema_v gds"}, + {RdnaOpcode::DsGwsSemaBr, 1u, 0u, "ds_gws_sema_br v2 gds"}, + {RdnaOpcode::DsGwsSemaP, 0u, 0u, "ds_gws_sema_p gds"}, + {RdnaOpcode::DsGwsSemaReleaseAll, 0u, 0u, "ds_gws_sema_release_all gds"}, + {RdnaOpcode::DsGwsBarrier, 1u, 0u, "ds_gws_barrier v3 gds"}, + }}; + const std::array dataRegisters{1u, 0u, 2u, 0u, 0u, 3u}; + for (std::size_t index = 0; index < gws.size(); ++index) { + const auto& [opcode, sources, offset, name] = gws[index]; + const auto& instruction = decoded[4u + index]; + require(instruction.op == opcode && instruction.gds && instruction.sourceCount == sources && instruction.memoryOffset == offset && + instruction.destination.kind == RdnaOperandKind::None && instruction.wordCount == 2u, name); + if (sources != 0u) require(isVector(instruction.source0, dataRegisters[index]), std::string(name) + " reads the wrong data register"); + } + + const auto& ordered = decoded[10]; + require(ordered.op == RdnaOpcode::DsOrderedCount && ordered.gds && ordered.memoryOffset == 4u && isVector(ordered.destination, 5u) && + isVector(ordered.source0, 1u) && ordered.sourceCount == 1u, "ds_ordered_count v5, v1 offset:4 gds"); + require(decoded[11].op == RdnaOpcode::SCodeEnd, "s_code_end"); +} + +std::string recompile(std::span code) { + const std::array userData{0x10000000u, 0x00100000u, 0x40u, 0x00027facu}; + const std::array capabilities{1u, 61u}; + const std::array extensions{"SPV_KHR_storage_buffer_storage_class"}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x20000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target.vulkanVersion = 0x00403000u; + request.target.spirvVersion = 0x00010600u; + request.target.subgroupSize = 64u; + request.target.bdaAbiVersion = 1; + request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.maxWorkgroupSize = {1024u, 1024u, 64u}; + request.target.maxWorkgroupInvocations = 1024; + request.target.maxWorkgroupSharedMemoryBytes = 49152; + request.layout = {0, 0, 0, 128}; + request.useCache = false; + try { + const auto result = Recompile(request); + require(!result.spirv.empty(), "no SPIR-V"); + } catch (const std::exception& error) { + return error.what(); + } + return {}; +} + +std::vector withStore(std::initializer_list words) { + std::vector code(words); + code.insert(code.end(), {0x7e020281u, 0xe0700000u, 0x80000100u, 0xbf810000u}); + return code; +} + +void verifyTranslation() { + require(recompile(withStore({0xbf8d0000u})).empty(), "s_sethalt 0 is refused: " + recompile(withStore({0xbf8d0000u}))); + const std::vector trailingLoop{0x7e020281u, 0xbf820003u, 0xe0700000u, 0x80000100u, 0xbf810000u, 0xbf82fffcu, 0xbf9f0000u, 0xbf9f0000u, 0xffffffffu}; + require(recompile(trailingLoop).empty(), "a program padded with s_code_end after its last branch is refused: " + recompile(trailingLoop)); + + const std::vector, std::string_view>> refused{ + {"s_sethalt 1", withStore({0xbf8d0001u}), "s_sethalt 1 at pc 0 halts the wave"}, + {"s_sendmsghalt", withStore({0xbf910001u}), "s_sendmsghalt 1 at pc 0 halts the wave"}, + {"s_code_end", {0x7e020281u, 0xbf9f0000u}, "s_code_end at pc 4 is reached"}, + {"s_rfe_b64", withStore({0xbe802204u}), "s_rfe_b64 at pc 0 returns from a trap handler"}, + {"ds_gws_init", withStore({0xd8660000u, 0x00000001u}), "ds_gws_init at pc 0: global wave sync"}, + {"ds_gws_sema_v", withStore({0xd86a0000u, 0x00000000u}), "ds_gws_sema_v at pc 0: global wave sync"}, + {"ds_gws_sema_br", withStore({0xd86e0000u, 0x00000002u}), "ds_gws_sema_br at pc 0: global wave sync"}, + {"ds_gws_sema_p", withStore({0xd8720000u, 0x00000000u}), "ds_gws_sema_p at pc 0: global wave sync"}, + {"ds_gws_sema_release_all", withStore({0xd8620000u, 0x00000000u}), "ds_gws_sema_release_all at pc 0: global wave sync"}, + {"ds_gws_barrier", withStore({0xd8760000u, 0x00000003u}), "ds_gws_barrier at pc 0: global wave sync"}, + {"ds_ordered_count", withStore({0xd8fe0000u, 0x05000001u}), "ds_ordered_count at pc 0: GDS counters"}, + }; + for (const auto& [name, code, message] : refused) { + const auto error = recompile(code); + require(error.find(message) != std::string::npos, std::string(name) + ": expected a refusal with '" + std::string(message) + "', got '" + error + "'"); + } +} + +} + +int main() { + try { + verifyDecode(); + verifyTranslation(); + std::puts("wave control decode tests passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + } + return 1; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp index 72a7c0841..f72f451e8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp @@ -83,7 +83,7 @@ void CheckSharedBlock() { const auto beforeCpu = CollectWrites(base, 3 * Block); static_cast(memory)[Block + 8] = 0x22; CollectWritesUncached(base, 3 * Block); - Require(StoredOver(boundary, static_cast(sharedSecond), beforeCpu), "a CPU store in the shared block is not seen"); + Require(!StoredOver(boundary, static_cast(sharedSecond), beforeCpu), "a CPU store outside the shared range counts for it"); std::array unwatched{}; const auto outside = reinterpret_cast(unwatched.data()); @@ -117,6 +117,69 @@ void CheckOwnStore() { CollectWritesUncached(base, 2 * Block); Require(!UnchangedSince(base, 64, beforeCpu), "a CPU write after the driver store is not seen"); } + +void CheckUnwatch() { + void* memory = AllocateWatched(3 * Block); + const auto base = reinterpret_cast(memory); + const auto imported = base + Block; + std::memset(memory, 0x11, 3 * Block); + BumpCollectEpoch(); + const auto before = CollectWrites(base, 3 * Block); + Require(before != 0 && UnchangedSince(imported, Block, before), "the import range was not initially watched"); + Unwatch(imported, Block); + Require(!Watched(imported, Block) && !UnchangedSince(imported, Block, before), "unwatch kept trusting import stamps"); + Require(CollectWrites(base, 3 * Block) == 0 && CollectWritesUncached(imported, Block) == 0, "unwatch did not invalidate a cached collect"); + Require(Watched(base, Block) && Watched(base + 2 * Block, Block), "unwatch disabled unrelated memory"); + const auto beforeDriverWrite = TrackerGeneration(); + Require(MarkWritten(imported, Block) == 0, "an unwatched import still produces trusted stamps"); + Require(TrackerGeneration() > beforeDriverWrite, "an unwatched driver write did not invalidate byte comparisons"); + const auto beforeOwnWrite = TrackerGeneration(); + StoreOwnBytes(imported, 1, [&] { *reinterpret_cast(imported) = 0x42; }); + Require(TrackerGeneration() > beforeOwnWrite, "an unwatched CPU store did not invalidate byte comparisons"); + std::array generations{before}; + std::array changed{}; + Require(!ChangedBlocks(imported, Block, generations, changed), "an unwatched import still uses block stamps"); + Unwatch(imported, Block); + const auto adjacent = CollectWrites(base, Block); + Require(adjacent != 0, "unwatch broke the adjacent block's collect"); + static_cast(memory)[0] = 0x33; + CollectWritesUncached(base, Block); + Require(!UnchangedSince(base, Block, adjacent), "unwatch hid a CPU edit in unrelated memory"); +} + +#ifdef _WIN32 +void CheckPrivateMappingReuse() { + void* memory = AllocateWatched(3 * Block); + const auto base = reinterpret_cast(memory); + std::memset(memory, 0x11, 3 * Block); + BumpCollectEpoch(); + const auto before = CollectWrites(base, 3 * Block); + Require(before != 0, "the original backing was not collected"); + Unwatch(base + Block, Block); + Require(!Watched(base + Block, Block), "the imported block stayed watched"); + GuestArena::GuestArenaCommit_nid_postfix(memory, 3 * Block, PAGE_READWRITE, 3 * Block); + Require(!Watched(base + Block, Block), "a protection-only commit restored an imported block"); + const auto neighbour = CollectWrites(base, Block); + Require(neighbour != 0, "the neighbouring block was not collected before remapping"); + GuestArena::GuestArenaReset_nid_postfix(memory, 3 * Block); + GuestArena::GuestArenaRelease_nid_postfix(memory, 3 * Block); + auto* replacement = GuestArena::GuestArenaAllocateAtOrAbove_nid_postfix(base, 3 * Block, Block); + Require(replacement == memory, "the released arena address was not reused"); + GuestArena::GuestArenaCommit_nid_postfix(replacement, 3 * Block, PAGE_READWRITE, 3 * Block); + Require(Watched(base, 3 * Block), "fresh backing at an excluded address is not watched"); + Require(!UnchangedSince(base + Block, Block, before), "fresh backing reused an old import generation"); + Require(!UnchangedSince(base, Block, neighbour), "fresh backing reused an adjacent block's old generation"); + const auto after = CollectWrites(base, 3 * Block); + Require(after > before && UnchangedSince(base, 3 * Block, after), "fresh backing was not collected"); + static_cast(replacement)[Block + 8] = 0x22; + CollectWritesUncached(base, 3 * Block); + Require(!UnchangedSince(base + Block, Block, after), "a CPU edit in fresh backing was missed"); + Unwatch(base + Block, Block); + Require(!Watched(base + Block, Block) && CollectWrites(base + Block, Block) == 0, "a second import of fresh backing stayed watched"); + GuestArena::GuestArenaReset_nid_postfix(replacement, 3 * Block); + GuestArena::GuestArenaRelease_nid_postfix(replacement, 3 * Block); +} +#endif } int main() { @@ -127,6 +190,10 @@ int main() { } CheckSharedBlock(); CheckOwnStore(); + CheckUnwatch(); +#ifdef _WIN32 + CheckPrivateMappingReuse(); +#endif } catch (const std::exception& error) { std::cerr << "write tracking test failed: " << error.what() << "\n"; return 1; diff --git a/core/libs/prx/libSceAgcDriver/tests/WriteWatchCoverage.cpp b/core/libs/prx/libSceAgcDriver/tests/WriteWatchCoverage.cpp new file mode 100644 index 000000000..083a984de --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/WriteWatchCoverage.cpp @@ -0,0 +1,62 @@ +#include "prx/libSceAgcDriver/Execution/include/WriteWatchCoverage.hpp" +#include +#include +#include + +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +int main() { + try { + using AgcDriver::GuestMemory::WriteWatchCoverage; + constexpr std::uint64_t base = 0x200000000; + constexpr std::size_t block = 65536; + WriteWatchCoverage coverage; + require(!coverage.Covers(base, 1), "an uninitialized arena is watched"); + coverage.Initialize(base, 4 * block + 1); + require(coverage.Covers(base, 4 * block + 1), "the initialized arena is not watched"); + require(!coverage.Covers(base - 1, 1) && !coverage.Covers(base + 4 * block + 1, 1), "outside memory is watched"); + require(!coverage.Covers(base, 0) && !coverage.Covers(base, std::numeric_limits::max()), "an empty or overflowing range is watched"); + require(coverage.Exclude(base + block + 4096, 1), "an imported page was not excluded"); + require(!coverage.Covers(base + block, block), "the imported block is still watched"); + require(!coverage.Covers(base + block - 1, 2), "a range crossing into the import is watched"); + require(coverage.Covers(base, block) && coverage.Covers(base + 2 * block, 2 * block + 1), "unrelated blocks lost write watching"); + require(!coverage.Exclude(base + block, block), "excluding an imported block again reports a change"); + require(coverage.Exclude(base + 3 * block - 1, 2), "an import across a block boundary was not excluded"); + require(!coverage.Covers(base + 2 * block, block) && !coverage.Covers(base + 3 * block, block), "a boundary import left one block watched"); + require(coverage.Covers(base, block) && coverage.Covers(base + 4 * block, 1), "a boundary import excluded adjacent blocks"); + require(!coverage.Exclude(base - block, block) && !coverage.Exclude(base + 5 * block, block), "an outside import changed coverage"); + require(coverage.Exclude(base + 4 * block, std::numeric_limits::max()), "the last partial block was not excluded"); + require(!coverage.Covers(base + 4 * block, 1), "the last partial block is still watched"); + coverage.Initialize(base, 2 * block); + require(coverage.Exclude(base - 1, 2) && !coverage.Covers(base, block), "an import crossing the arena start was not clipped"); + require(coverage.Covers(base + block, block), "clipping an import excluded the rest of the arena"); + coverage.Initialize(base, 4 * block + 1); + coverage.Exclude(base, 4 * block + 1); + require(!coverage.Restore(base + 1, block - 1) && !coverage.Covers(base, block), "a partial replacement restored an excluded block"); + require(coverage.Restore(base + block, block) && coverage.Covers(base + block, block), "fresh private backing stayed excluded"); + require(!coverage.Restore(base + block, block), "restoring an already watched block reports a change"); + require(!coverage.Covers(base, block) && !coverage.Covers(base + 2 * block, block), "restoring one block restored its neighbours"); + require(coverage.Restore(base + block - 1, 2 * block + 2), "a full interior replacement was not restored"); + require(!coverage.Covers(base, block) && coverage.Covers(base + block, 2 * block) && !coverage.Covers(base + 3 * block, block), "a replacement restored partially covered boundary blocks"); + require(!coverage.Restore(base - block, block) && !coverage.Restore(base + 5 * block, block), "an outside replacement changed coverage"); + require(coverage.Restore(base + 3 * block, std::numeric_limits::max()) && coverage.Covers(base + 3 * block, block + 1), "an oversized replacement was not clipped to the arena"); + require(coverage.Restore(base - block, 2 * block) && coverage.Covers(base, block), "a replacement across the arena start was not clipped"); + coverage.Exclude(base + block + 4096, 1); + require(!coverage.Covers(base + block, block), "a re-import did not exclude fresh backing again"); + coverage.Initialize(base, 2 * block); + coverage.Exclude(base, block, 4); + require(!coverage.Restore(base, block, 3) && !coverage.Restore(base, block, 4), "a stale mapping notification restored a newer import"); + require(coverage.Restore(base, block, 5) && coverage.Covers(base, block), "a newer private mapping did not restore an older exclusion"); + coverage.Exclude(base, block, 5); + coverage.Exclude(base, block, 7); + require(!coverage.Restore(base, block, 6) && !coverage.Covers(base, block), "a delayed mapping notification erased a repeated import"); + require(coverage.Restore(base, block, 8), "a fresh replacement did not restore the latest import exclusion"); + std::cout << "write watch coverage passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp new file mode 100644 index 000000000..a990a210d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp @@ -0,0 +1,99 @@ +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using namespace ShaderRecompiler; + +struct alignas(4096) GuestData { + std::array, 4> heap{}; + std::array material{}; + std::array srt{}; + std::array output{}; +}; + +alignas(256) std::array, 2> textures{}; +std::array guest; +constexpr std::array code{0xbf068002u, 0xbf850013u, 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x93109010u, 0xf4200406u, 0x20000004u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u, 0xbf800000u}; + +std::array Buffer(const void* pointer, std::uint32_t stride, std::uint32_t records) { + const auto address = reinterpret_cast(pointer); + return {static_cast(address), static_cast(address >> 32u) | (stride << 16u), records, stride == 0u ? 0x31016facu : 0x01016facu}; +} + +void Run(AgcDriver::VulkanDevice& device) { + textures[0].fill(0x40000000u); + textures[1].fill(0x40400000u); + const auto threads = std::min(32u, device.Target().subgroupSize); + CompiledShaderArtifact artifact; + for (std::uint32_t iteration = 0u; iteration < guest.size(); ++iteration) { + auto& data = guest[iteration]; + const auto table = Buffer(data.heap.data(), 32u, 4u); + const auto material = Buffer(data.material.data(), 16u, 3u); + std::copy(table.begin(), table.end(), data.srt.begin()); + std::copy(material.begin(), material.end(), data.srt.begin() + 8u); + const auto srtAddress = reinterpret_cast(data.srt.data()); + const std::array userData{static_cast(srtAddress), static_cast(srtAddress >> 32u), iteration % 6u == 0u ? 0u : 1u}; + const std::array regions{MemoryRegion{reinterpret_cast(&data), std::as_bytes(std::span(&data, 1u))}}; + const auto key = iteration % 4u; + if (key < 2u) { + for (std::uint32_t texture = 0u; texture < textures.size(); ++texture) { + const auto address = reinterpret_cast(textures[texture].data()); + data.heap[texture] = {static_cast(address >> 8u), static_cast(address >> 40u) | ((texture == 0u ? 20u : 22u) << 20u) | (3u << 30u), 7u, 0x90000facu, 0u, 0u, 0u, 0u}; + } + } + data.material[1] = key == 3u ? 0xffffffffu : key; + data.material[5] = 0u; + data.material[9] = 1u; + data.output.fill(0xdeadbeefu); + const auto output = Buffer(data.output.data(), 0u, 256u); + std::copy(output.begin(), output.end(), data.srt.begin() + 12u); + const ShaderComputeStageInfo compute{{threads, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + RecompileRequest request{{ShaderStage::Compute, reinterpret_cast(code.data()), code, 0u, {}}, {32u, 0u, userData, compute, std::nullopt, std::nullopt, regions}, device.Target(), {0u, 0u, 0u, 128u}}; + const auto shader = Recompile(request); + if (artifact.variantId == 0u) artifact = shader; + else Require(shader.cacheHit && shader.variantId == artifact.variantId, "bindless table changed the compiled artifact"); + if (iteration == 1u) { + auto invalid = shader; + auto binding = std::ranges::find_if(invalid.bindings, [](const DescriptorBinding& value) { return value.role == DescriptorRole::ShaderData; }); + Require(binding != invalid.bindings.end(), "bindless shader has no runtime metadata"); + const auto offset = invalid.imageMetadataDword + invalid.runtimeImageResources.at(0) * (sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)) + offsetof(RuntimeAbi::ResourceMetadata, firstElement) / sizeof(std::uint32_t); + binding->guestDescriptor.at(offset) = RuntimeAbi::SampledHeapCapacity; + bool rejected = false; + try { + device.Dispatch(invalid, 1u, 1u, 1u); + } catch (const std::exception& error) { + rejected = std::string_view(error.what()).find("runtime metadata exceeds its bound heap") != std::string_view::npos; + } + Require(rejected, "driver accepted an out-of-range runtime heap index"); + } + device.Dispatch(shader, 1u, 1u, 1u); + device.SubmitRecorded(false); + } + device.WaitIdle(); + for (std::uint32_t iteration = 0u; iteration < guest.size(); ++iteration) { + const auto expected = iteration % 6u == 0u ? 0xdeadbeefu : iteration % 4u < 2u ? textures[iteration % 2u][0] : 0u; + Require(guest[iteration].output[0] == expected, "bindless resource selection or lifetime failed at " + std::to_string(iteration) + " value=" + std::to_string(guest[iteration].output[0]) + " expected=" + std::to_string(expected)); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + std::cout << "bindless image materialization tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp index aab4cf664..5a2a267f4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp @@ -79,7 +79,7 @@ std::uint32_t SubtractClamped(std::uint32_t memory, std::uint32_t subtrahend) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp index c07ca3567..550cbd7f0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp @@ -50,7 +50,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -120,6 +120,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityInt64Atomics)) { + std::puts("skipped, the device has no shaderBufferInt64Atomics"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("buffer atomics tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsFloat.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsFloat.cpp index df18dfbdb..99a286d6a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsFloat.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsFloat.cpp @@ -118,7 +118,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -168,6 +168,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityInt64Atomics)) { + std::puts("skipped, the device has no shaderBufferInt64Atomics"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("buffer atomics float tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsIncDec64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsIncDec64.cpp index c5f62c2a9..c407271e2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsIncDec64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsIncDec64.cpp @@ -110,7 +110,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -160,6 +160,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityInt64Atomics)) { + std::puts("skipped, the device has no shaderBufferInt64Atomics"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("buffer atomics inc dec64 tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsOrSwap64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsOrSwap64.cpp index be03cdc3b..fe84e93a1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsOrSwap64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicsOrSwap64.cpp @@ -281,6 +281,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityInt64Atomics)) { + std::puts("skipped, the device has no shaderBufferInt64Atomics"); + return VulkanTestSkipped; + } Run(*device, Wave32Code, 32, device->Target()); Check(Expected32, Names, "wave32"); Run(*device, Wave64Code, 64, device->Target()); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp index d5e82afdf..da69c6d9d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp @@ -288,6 +288,47 @@ void Run(AgcDriver::VulkanDevice& device, std::span code, s device.WaitIdle(); } +void CheckRuntimeDescriptors(AgcDriver::VulkanDevice& device, std::uint32_t waveSize) { + constexpr std::array code{0x34040082u, 0xe0000000u, 0x80000100u, 0xe0701000u, 0x80010102u, 0xbf810000u}; + struct Case { + std::uint32_t stride; + std::uint32_t records; + std::uint32_t format; + std::uint32_t selector; + bool swizzle; + }; + constexpr std::array cases{{{4u, 32u, 20u, 4u, false}, {8u, 32u, 5u, 4u, false}, {8u, 32u, 11u, 4u, true}, {4u, 32u, 18u, 5u, false}, {4u, 16u, 20u, 4u, false}, {4u, 64u, 20u, 4u, false}}}; + for (std::uint32_t index = 0; index < Input.size(); ++index) Input[index] = 0x3f000000u + index * 0x201u; + std::array userData{}; + const auto threads = waveSize * 2u; + const auto output = BufferDescriptor(Output.data(), threads * 4u); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const ShaderRecompiler::ShaderComputeStageInfo compute{{threads, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + ShaderRecompiler::RecompileRequest request{{ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {waveSize, 0u, userData, compute, std::nullopt, std::nullopt, {}}, device.Target(), {0u, 0u, 0u, 128u}}; + ShaderRecompiler::RecompileResult first; + for (const auto& value : cases) { + auto input = BufferDescriptor(Input.data(), value.records); + input[1] |= (value.stride << 16u) | (value.swizzle ? 0x80000000u : 0u); + input[3] = 0x11800000u | (value.format << 12u) | value.selector; + std::copy(input.begin(), input.end(), userData.begin()); + Output.fill(0xdeadbeefu); + const auto result = ShaderRecompiler::Recompile(request); + if (first.spirv.empty()) first = result; + else Require(result.cacheHit && result.variantId == first.variantId, "runtime V# changed the formatted shader artifact"); + device.Dispatch(result, 1, 1, 1); + device.WaitIdle(); + for (std::uint32_t thread = 0; thread < threads; ++thread) { + const auto lane = thread % waveSize; + const auto index = value.swizzle ? (lane / 8u) * 16u + lane % 8u : lane * value.stride / 4u; + auto expected = lane >= value.records ? 0u : Input[index]; + if (value.format == 5u) expected &= 0xffu; + if (value.format == 11u) expected &= 0xffffu; + if (value.format == 18u) expected = (expected >> 8u) & 0xffu; + Require(Output[thread] == expected, "runtime V# formatted load mismatch: wave" + std::to_string(waveSize) + " thread " + std::to_string(thread) + " format " + std::to_string(value.format) + " actual " + Hex(Output[thread]) + " expected " + Hex(expected)); + } + } +} + template void Check(const std::uint32_t (&expected)[Lanes][Checked], const char* const (&names)[Checked], const char* run) { for (std::uint32_t tid = 0; tid < Lanes; ++tid) { @@ -310,6 +351,8 @@ int main() { Check(Expected64, Names, "wave64"); Run(*device, Wave64Code, 64, device->ComputeTarget(32)); Check(Expected64, Names, "wave64 split"); + CheckRuntimeDescriptors(*device, 32u); + CheckRuntimeDescriptors(*device, 64u); std::puts("buffer load store tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferUnalignedBase.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferUnalignedBase.cpp new file mode 100644 index 000000000..dd78c229b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferUnalignedBase.cpp @@ -0,0 +1,260 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Lanes = 32; +constexpr std::uint32_t Slots = 8; +constexpr std::uint32_t Checked = 8; +constexpr std::uint32_t InputBytes = 256; +constexpr std::uint32_t ViewBytes = 60; +constexpr std::uint32_t ResultBytes = Lanes * Slots * 4; +constexpr std::uint32_t StoredBytes = 8 * 28; +constexpr std::uint32_t Fill = 0xdeadbeefu; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020082, 0x34040085, 0x34100081, 0x4a101082, 0x34140081, 0x4a141500, 0x4a141481, 0x34180083, + 0x4a1a00ff, 0x5a5abe00, 0x4a1c00ff, 0x7f7f7f91, 0x4a1e00ff, 0xa1b2c300, 0x4a2000bc, 0xe0301000, + 0x80000301, 0xe0301000, 0x80010401, 0xe0341004, 0x80010501, 0xe0281000, 0x80010708, 0xe0201000, + 0x8001090a, 0xe8281002, 0x80011101, 0xf4200402, 0xfa000006, 0xbf8c0000, 0x7e160210, 0xe0701000, + 0x80020302, 0xe0701004, 0x80020402, 0xe0701008, 0x80020502, 0xe070100c, 0x80020602, 0xe0701010, + 0x80020702, 0xe0701014, 0x80020902, 0xe0701018, 0x80020b02, 0xe070101c, 0x80021102, 0xe0701000, + 0x80030f0c, 0xe0601004, 0x80030e0c, 0xe0681006, 0x80030d0c, 0xe82c1005, 0x8003100c, 0xbf810000, +}; + +alignas(256) constexpr std::array PairCode{ + 0x34040083, 0x7e0a0281, 0x7e0c0280, 0xe0345000, 0x80010300, 0xe1480100, 0x80020500, 0xbf8c0000, + 0xe0741000, 0x80020302, 0xbf810000, +}; + +alignas(256) constexpr std::array AtomicCode{0xe0c81000, 0x80000100, 0xbf810000}; + +constexpr std::array FormattedLoadCode(std::uint32_t format, std::uint32_t offset) { + return {0x34020082, 0xe8001000u | (format << 19u) | offset, 0x80000201, 0xbf8c0000, 0xe0701000, 0x80010201, 0xbf810000}; +} + +constexpr std::array FormattedStoreCode(std::uint32_t format, std::uint32_t offset) { + return {0x34020082, 0x4a0400ff, 0x0000c300, 0xe8041000u | (format << 19u) | offset, 0x80000201, 0xbf810000}; +} + +constexpr std::uint32_t Format16UInt = 11; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t Format16_16UInt = 27; +alignas(256) constexpr auto Load16Code = FormattedLoadCode(Format16UInt, 2); +alignas(256) constexpr auto Load32Code = FormattedLoadCode(Format32UInt, 0); +alignas(256) constexpr auto Load16_16Code = FormattedLoadCode(Format16_16UInt, 0); +alignas(256) constexpr auto Store16Code = FormattedStoreCode(Format16UInt, 0); + +constexpr const char* Names[Checked] = { + "aligned dword", "dword", "dwordx2[0]", "dwordx2[1]", "ushort", "ubyte", "s_buffer_load_dword", "tbuffer_load_format_x 8_UINT", +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +std::uint32_t ViewDword(const std::uint8_t* view, std::uint32_t dword) { + if (dword >= ViewBytes / 4) return 0; + std::uint32_t value = 0; + std::memcpy(&value, view + dword * 4u, sizeof(value)); + return value; +} + +std::uint32_t ViewField(const std::uint8_t* view, std::uint32_t byte, std::uint32_t mask) { + return (ViewDword(view, byte / 4u) >> ((byte % 4u) * 8u)) & mask; +} + +std::uint8_t FillByte(std::size_t offset) { + return static_cast(Fill >> ((offset % 4u) * 8u)); +} + +ShaderRecompiler::RecompileRequest Request(std::span code, const std::vector& userData, const ShaderRecompiler::SpirvTarget& target, std::span memory) { + const ShaderRecompiler::ShaderComputeStageInfo compute{{Lanes, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {Lanes, 0, userData, compute, std::nullopt, std::nullopt, memory}, + target, + {0, 0, 0, 128} + }; + request.useCache = false; + return request; +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t viewOffset, std::uint32_t storeOffset) { + for (std::uint32_t index = 0; index < InputBytes; ++index) Input[index] = static_cast(index * 37u + 11u + (index >> 3u)); + Output.fill(Fill); + auto* output = reinterpret_cast(Output.data()); + const auto* view = Input.data() + viewOffset; + std::vector userData; + for (const auto& descriptor : {BufferDescriptor(Input.data(), InputBytes), BufferDescriptor(view, ViewBytes), BufferDescriptor(Output.data(), ResultBytes), BufferDescriptor(output + ResultBytes + storeOffset, StoredBytes)}) { + userData.insert(userData.end(), descriptor.begin(), descriptor.end()); + } + const std::array memory{{{reinterpret_cast(Code.data()), std::as_bytes(std::span(Code))}}}; + const auto result = ShaderRecompiler::Recompile(Request(Code, userData, device.Target(), memory)); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(Code.data())); + device.WaitIdle(); + const auto run = "view +" + std::to_string(viewOffset) + ", stores +" + std::to_string(storeOffset); + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + std::uint32_t aligned = 0; + std::memcpy(&aligned, Input.data() + lane * 4u, sizeof(aligned)); + std::uint32_t scalar = 0; + std::memcpy(&scalar, Input.data() + (viewOffset & ~3u) + 4u, sizeof(scalar)); + const std::uint32_t expected[Checked] = {aligned, ViewDword(view, lane), ViewDword(view, lane + 1u), ViewDword(view, lane + 2u), ViewField(view, lane * 2u + 2u, 0xffffu), ViewField(view, lane * 3u + 1u, 0xffu), scalar, ViewField(view, lane * 4u + 2u, 0xffu)}; + for (std::uint32_t slot = 0; slot < Checked; ++slot) { + const auto actual = Output[lane * Slots + slot]; + Require(actual == expected[slot], "buffer unaligned base " + run + ": lane " + std::to_string(lane) + " " + Names[slot] + " is " + Hex(actual) + ", expected " + Hex(expected[slot])); + } + } + for (std::size_t offset = ResultBytes; offset < Output.size() * 4u; ++offset) { + const auto relative = static_cast(offset) - static_cast(ResultBytes + storeOffset); + auto expected = FillByte(offset); + if (relative >= 0 && relative < StoredBytes) { + const auto lane = static_cast(relative / 8); + const auto byte = static_cast(relative % 8); + if (byte < 4u) expected = static_cast((0xa1b2c300u + lane) >> (byte * 8u)); + else if (byte == 4u) expected = static_cast(0x91u + lane); + else if (byte == 5u) expected = static_cast(0x3cu + lane); + else if (byte >= 6u) expected = static_cast((0xbe00u + lane) >> ((byte - 6u) * 8u)); + } + Require(output[offset] == expected, "buffer unaligned base " + run + ": stored byte " + std::to_string(offset - ResultBytes) + " is " + Hex(output[offset]) + ", expected " + Hex(expected)); + } +} + +void RunPair(AgcDriver::VulkanDevice& device, std::uint32_t viewOffset) { + for (std::uint32_t index = 0; index < InputBytes; ++index) Input[index] = static_cast(index * 53u + 7u + (index >> 2u)); + Output.fill(Fill); + const auto* view = Input.data() + viewOffset; + std::vector userData; + for (const auto& descriptor : {BufferDescriptor(Input.data(), InputBytes), BufferDescriptor(view, ViewBytes), BufferDescriptor(Output.data(), ResultBytes)}) { + userData.insert(userData.end(), descriptor.begin(), descriptor.end()); + } + const std::array memory{{{reinterpret_cast(PairCode.data()), std::as_bytes(std::span(PairCode))}}}; + const auto result = ShaderRecompiler::Recompile(Request(PairCode, userData, device.Target(), memory)); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(PairCode.data())); + device.WaitIdle(); + const auto run = "buffer unaligned base glc dwordx2, view +" + std::to_string(viewOffset); + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + for (std::uint32_t half = 0; half < 2u; ++half) { + const auto actual = Output[lane * 2u + half]; + const auto expected = ViewDword(view, lane / 4u + half); + Require(actual == expected, run + ": lane " + std::to_string(lane) + " dword " + std::to_string(half) + " is " + Hex(actual) + ", expected " + Hex(expected)); + } + } + Require(Output[Lanes * 2u] == Fill + Lanes && Output[Lanes * 2u + 1u] == Fill, run + ": the 64-bit atomic is " + Hex(Output[Lanes * 2u + 1u]) + Hex(Output[Lanes * 2u])); +} + +auto RecompileFormatted(AgcDriver::VulkanDevice& device, std::span code, const std::uint8_t* view, std::uint32_t viewBytes) { + std::vector userData; + for (const auto& descriptor : {BufferDescriptor(view, viewBytes), BufferDescriptor(Output.data(), ResultBytes)}) { + userData.insert(userData.end(), descriptor.begin(), descriptor.end()); + } + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + return ShaderRecompiler::Recompile(Request(code, userData, device.Target(), memory)); +} + +void RunFormattedFault(AgcDriver::VulkanDevice& device, const std::string& name, std::span code, std::uint32_t viewOffset) { + const auto run = "buffer unaligned base " + name + ", view +" + std::to_string(viewOffset); + try { + static_cast(RecompileFormatted(device, code, Input.data() + viewOffset, ViewBytes)); + } catch (const std::exception& error) { + Require(std::string(error.what()).find("is a formatted buffer access through a V# whose base is not aligned to its element") != std::string::npos, run + ": unexpected error: " + error.what()); + return; + } + throw std::runtime_error(run + ": a formatted access the hardware faults on was recompiled"); +} + +void RunFormattedLoad(AgcDriver::VulkanDevice& device, std::uint32_t viewOffset) { + for (std::uint32_t index = 0; index < InputBytes; ++index) Input[index] = static_cast(index * 29u + 5u + (index >> 4u)); + Output.fill(Fill); + const auto* view = Input.data() + viewOffset; + const auto result = RecompileFormatted(device, Load16Code, view, ViewBytes); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(Load16Code.data())); + device.WaitIdle(); + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + const auto expected = ViewField(view, lane * 4u + 2u, 0xffffu); + Require(Output[lane] == expected, "buffer unaligned base tbuffer_load_format_x 16_UINT, view +" + std::to_string(viewOffset) + ": lane " + std::to_string(lane) + " is " + Hex(Output[lane]) + ", expected " + Hex(expected)); + } +} + +void RunFormattedStore(AgcDriver::VulkanDevice& device, std::uint32_t viewOffset) { + Output.fill(Fill); + auto* output = reinterpret_cast(Output.data()); + const auto result = RecompileFormatted(device, Store16Code, output + viewOffset, Lanes * 4u); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(Store16Code.data())); + device.WaitIdle(); + for (std::size_t offset = 0; offset < Output.size() * 4u; ++offset) { + const auto relative = static_cast(offset) - static_cast(viewOffset); + auto expected = FillByte(offset); + if (relative >= 0 && relative < Lanes * 4 && relative % 4 < 2) expected = static_cast((0xc300u + static_cast(relative / 4)) >> ((relative % 4) * 8)); + Require(output[offset] == expected, "buffer unaligned base tbuffer_store_format_x 16_UINT, view +" + std::to_string(viewOffset) + ": byte " + std::to_string(offset) + " is " + Hex(output[offset]) + ", expected " + Hex(expected)); + } +} + +void RunAtomic(AgcDriver::VulkanDevice& device) { + std::vector userData; + const auto descriptor = BufferDescriptor(reinterpret_cast(Output.data()) + 2u, 64u); + userData.insert(userData.end(), descriptor.begin(), descriptor.end()); + const std::array memory{{{reinterpret_cast(AtomicCode.data()), std::as_bytes(std::span(AtomicCode))}}}; + try { + const auto result = ShaderRecompiler::Recompile(Request(AtomicCode, userData, device.Target(), memory)); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(AtomicCode.data())); + device.WaitIdle(); + } catch (const std::exception& error) { + Require(std::string(error.what()).find("buffer atomic on a V# whose base is not DWORD aligned") != std::string::npos, std::string("buffer unaligned base atomic: unexpected error: ") + error.what()); + return; + } + throw std::runtime_error("buffer unaligned base atomic: an atomic on a V# off a DWORD boundary ran"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, 2, 1); + Run(*device, 3, 3); + Run(*device, 1, 2); + RunPair(*device, 1); + RunPair(*device, 2); + RunPair(*device, 3); + RunAtomic(*device); + RunFormattedLoad(*device, 2); + RunFormattedStore(*device, 2); + for (const auto view : {1u, 3u}) { + RunFormattedFault(*device, "tbuffer_load_format_x 16_UINT", Load16Code, view); + RunFormattedFault(*device, "tbuffer_store_format_x 16_UINT", Store16Code, view); + } + for (const auto view : {1u, 2u, 3u}) { + RunFormattedFault(*device, "tbuffer_load_format_x 32_UINT", Load32Code, view); + RunFormattedFault(*device, "tbuffer_load_format_x 16_16_UINT", Load16_16Code, view); + } + std::puts("buffer unaligned base tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp index d0a599877..788ff56ff 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp @@ -230,7 +230,7 @@ void FillInput(std::uint64_t nodeBias) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array BvhDescriptor(std::uint64_t base, std::uint64_t lastNode) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ConservativeRasterization.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ConservativeRasterization.cpp new file mode 100644 index 000000000..4bc982a2b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ConservativeRasterization.cpp @@ -0,0 +1,160 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 16; +constexpr std::uint32_t WaveSize = 64; +constexpr std::byte Kept{0x40}; +alignas(256) std::array Pixels{}; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +alignas(256) constexpr std::array PixelCode{ + 0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000, +}; + +using Pixel = std::pair; + +constexpr std::array, 9> Corners{{ + {1.2f, 10.2f}, {4.6f, 10.2f}, {1.2f, 13.6f}, + {5.15f, 5.15f}, {5.4f, 5.15f}, {5.15f, 5.4f}, + {8.6f, 8.2f}, {9.4f, 8.2f}, {8.6f, 8.8f}, +}}; + +const std::set CentersCovered{{1, 10}, {2, 10}, {3, 10}, {1, 11}, {2, 11}, {1, 12}}; + +const std::set AreasTouched{ + {1, 10}, {2, 10}, {3, 10}, {4, 10}, {1, 11}, {2, 11}, {3, 11}, {1, 12}, {2, 12}, {1, 13}, + {5, 5}, {8, 8}, {9, 8}, +}; + +alignas(256) std::array, Corners.size()> Vertices{}; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +void Draw(AgcDriver::VulkanDevice& device, VkConservativeRasterizationModeEXT mode) { + Pixels.fill(Kept); + for (std::size_t index = 0; index < Corners.size(); ++index) { + Vertices[index] = {Corners[index][0] * 2.0f / Width - 1.0f, 1.0f - Corners[index][1] * 2.0f / Height, 0.5f, 1.0f}; + } + const auto target = device.Target(); + + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(Vertices.data(), 16u, static_cast(Vertices.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {WaveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData(8, 0u); + const std::array pixelMemory{{{reinterpret_cast(PixelCode.data()), std::as_bytes(std::span(PixelCode))}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, + {WaveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + {0, 0, vertexPush, 128 - vertexPush} + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, vertexPush} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, WaveSize, WaveSize, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.conservativeRasterization = mode; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Vertices.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); +} + +void Check(const std::set& covered, std::string_view what) { + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + const bool written = covered.contains({x, y}); + const auto expected = written ? std::byte{255} : Kept; + for (std::uint32_t channel = 0; channel < 4; ++channel) { + const auto actual = Pixels[(y * Width + x) * 4u + channel]; + Require(actual == expected, std::string(what) + ": pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") channel " + std::to_string(channel) + " is " + std::to_string(static_cast(actual)) + ", expected " + std::to_string(static_cast(expected)) + (written ? " (covered)" : " (not covered)")); + } + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Draw(*device, VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT); + Check(CentersCovered, "conservative rasterization off"); + if (!device->ConservativeRasterization()) { + try { + Draw(*device, VK_CONSERVATIVE_RASTERIZATION_MODE_OVERESTIMATE_EXT); + } catch (const std::exception& error) { + Require(std::string_view(error.what()).find("requires VK_EXT_conservative_rasterization") != std::string_view::npos, std::string("unexpected rejection: ") + error.what()); + std::puts("skipped, the device has no VK_EXT_conservative_rasterization overestimation within 1/256 pixel that rasterizes degenerate triangles"); + return VulkanTestSkipped; + } + Require(false, "overestimation was drawn without device support"); + } + Draw(*device, VK_CONSERVATIVE_RASTERIZATION_MODE_OVERESTIMATE_EXT); + Check(AreasTouched, "overestimation"); + Draw(*device, VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT); + Check(CentersCovered, "conservative rasterization off after overestimation"); + std::puts("conservative rasterization tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ConstantStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ConstantStore.cpp new file mode 100644 index 000000000..c9b16efb5 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ConstantStore.cpp @@ -0,0 +1,100 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "../NarrowConstantStoreFixture.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using namespace NarrowConstantStoreFixture; + +constexpr std::size_t BlockBytes = 65536; + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); +#endif + Require(block != nullptr, "constant store: cannot allocate the guest block"); + Clear(); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + void Clear() const { std::memset(block, Fill, BlockBytes); } + const std::uint8_t* Data() const { return block; } + +private: + std::uint8_t* block = nullptr; +}; + +void Run(AgcDriver::VulkanDevice& device, const GuestBlock& guest, std::uint32_t waveSize) { + using namespace ShaderRecompiler; + guest.Clear(); + const auto address = static_cast(reinterpret_cast(guest.Data())); + const std::array userData{ + static_cast(address), static_cast(address >> 32u), + }; + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1, {}}; + RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), {0, 0, 0, 128}, std::nullopt, false, + }; + const auto result = Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + for (std::size_t offset = 0; offset < BlockBytes; ++offset) { + const auto expected = offset < Threads * LaneBytes ? ExpectedLane[offset % LaneBytes] : Fill; + Require(guest.Data()[offset] == expected, "constant store: wave" + std::to_string(waveSize) + " byte " + std::to_string(offset) + " is " + + std::to_string(guest.Data()[offset]) + ", expected " + std::to_string(expected)); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const GuestBlock guest; + Run(*device, guest, 32); + Run(*device, guest, 64); + std::puts("constant store Vulkan readback passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/D16Memory.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/D16Memory.cpp index a4bf4e5b9..308b6b59f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/D16Memory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/D16Memory.cpp @@ -41,7 +41,7 @@ alignas(256) constexpr std::array D16Code{ std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DccKeysFirstWrite.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DccKeysFirstWrite.cpp index 0375be15f..c3c4bd5eb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DccKeysFirstWrite.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DccKeysFirstWrite.cpp @@ -71,15 +71,22 @@ std::uint64_t AddressOf(const void* data) { class GuestBlock { public: - explicit GuestBlock(bool watch) : watched(watch) { + explicit GuestBlock(bool watch, bool shared = false) : watched(watch) { #ifdef _WIN32 if (watched) { block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, 65536)); - if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes); + if (shared) { + Require(block != nullptr, "cannot reserve shared test memory"); + const auto section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE, 0, BlockBytes, nullptr); + Require(section != nullptr, "cannot create shared test memory"); + GuestArena::GuestArenaMap_nid_postfix(block, BlockBytes, section, 0, PAGE_READWRITE); + CloseHandle(section); + } else if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes); } else { block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); } #else + Require(!shared, "shared test memory requires Windows"); if (watched) { constexpr std::uintptr_t alignment = 65536; void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); @@ -146,7 +153,7 @@ std::array TextureDescriptor(const std::uint8_t* texels, const std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = AddressOf(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } void Dispatch(AgcDriver::VulkanDevice& device, std::span code, const std::vector& userData, bool wait = true) { @@ -184,6 +191,7 @@ void Run(AgcDriver::VulkanDevice& device, std::uint8_t* block) { std::memset(texels, 0x55, SurfaceBytes); std::memset(keys, 0x00, KeyBytes); std::memset(copied, 0xaa, KeyBytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); const auto texture = TextureDescriptor(texels, keys); std::vector writeData(16, 0u); std::copy(texture.begin(), texture.end(), writeData.begin() + 4); @@ -201,6 +209,12 @@ void Run(AgcDriver::VulkanDevice& device, std::uint8_t* block) { Require(image != nullptr && image->Descriptor().dccAddress == AddressOf(keys), "the written surface has no pending image under its keys"); Require(AgcDriver::Graphics::StorageImageServesKeys(*image, AddressOf(copied)), "the written image does not serve the copied keys"); Require(image->UploadedKeys() == DccKeys::Uncompressed && image->FilledKeys() == DccKeys::Uncompressed, "the written image still holds the clear it was uploaded under"); + const auto pendingVersion = image->Version(); + for (int repeat = 0; repeat < 3; ++repeat) { + Require(image->Refresh(), "an unchanged pending image was uploaded again"); + Require(image->Version() == pendingVersion, "an unchanged pending image changed version"); + Require(StorageTexture::FindPending(surface, SurfaceBytes) == image, "an unchanged refresh wrote the pending image back"); + } StorageTexture::FlushPending(surface, SurfaceBytes, nullptr, "test"); device.WaitIdle(); RequireKeys(keys, 0xff, "the keys after the write-back"); @@ -268,6 +282,24 @@ void Run(AgcDriver::VulkanDevice& device, std::uint8_t* block) { StorageTexture::FlushPending(surface, SurfaceBytes, nullptr, "test"); device.WaitIdle(); requireTexels("a first write followed by a kernel's 0000 store over half the keys"); + fillKeys(0x00, DccKeys::Clear0000); + image->Refresh(); + Require(image->UploadedKeys() == DccKeys::Clear0000 && image->FilledKeys() == DccKeys::Uncompressed, "the refresh after a 0000 key fill did not clear the image"); + fillKeys(0x40, DccKeys::Clear0001); + Require(image->FilledKeys() == DccKeys::Clear0001, "the 0001 key fill over the cleared image was not noted on it"); + Dispatch(device, ClearKeysCode, clearData); + const auto overFill = image->ProvedKeys(); + std::snprintf(message, sizeof(message), "a key read after a kernel's 0000 store over a 0001 key fill: the keys read %s and the image keeps a %s fill (expected 0000 and none)", AgcDriver::Graphics::DccKeysName(overFill), AgcDriver::Graphics::DccKeysName(image->FilledKeys())); + Require(overFill == DccKeys::Clear0000 && image->FilledKeys() == DccKeys::Uncompressed, message); + std::memset(copied, 0xaa, KeyBytes); + fillKeys(0x40, DccKeys::Clear0001); + Dispatch(device, ClearKeysCode, clearData); + Dispatch(device, WriteCode, writeData); + Dispatch(device, CopyKeysCode, copyData); + RequireKeys(copied, 0xff, "a kernel reading the keys after a 0001 key fill, a kernel's 0000 store over them and a write"); + StorageTexture::FlushPending(surface, SurfaceBytes, nullptr, "test"); + device.WaitIdle(); + requireTexels("a write after a 0001 key fill and a kernel's 0000 store over the keys"); const auto fill = [&](std::size_t offset, std::size_t bytes, std::uint8_t key) { const std::uint32_t word = key * 0x01010101u; const std::array pattern{word, word, word, word}; @@ -311,16 +343,106 @@ void Run(AgcDriver::VulkanDevice& device, std::uint8_t* block) { RequireKeys(keys, 0x40, "a write marked after a 0001 key fill over a cleared image, before a refresh saw the fill"); } +void RunUntrackedReuse(AgcDriver::VulkanDevice& device, std::uint8_t* block) { + const auto address = AddressOf(block); + std::memset(block, 0x55, BlockBytes); + auto descriptor = TextureDescriptor(block, nullptr); + descriptor[2] = ((Side - 1u) >> 2u) | ((2 * Side - 1u) << 14u); + descriptor[6] = 0; + descriptor[7] = 0; + std::vector userData(16, 0u); + std::copy(descriptor.begin(), descriptor.end(), userData.begin() + 4); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + Dispatch(device, WriteCode, userData); + const auto image = StorageTexture::FindPending(address, BlockBytes); + Require(image != nullptr && image->GuestBytes() == BlockBytes, "the two-block image has no pending results"); + Require(!AgcDriver::GuestMemory::Watched(address, BlockBytes), "the imported test image is still write-watched"); + AgcDriver::GuestMemory::BumpCollectEpoch(); + const auto version = image->Version(); + for (int repeat = 0; repeat < 3; ++repeat) { + Require(image->Refresh() && image->Version() == version, "an unchanged untracked image was uploaded again"); + Require(Count(block, BlockBytes, 0x55) == BlockBytes, "refresh wrote GPU results into unchanged guest memory"); + } + std::memset(block + SurfaceBytes, 0x33, SurfaceBytes); + AgcDriver::GuestMemory::BumpCollectEpoch(); + Require(!image->Refresh(), "an untracked CPU edit was missed"); + const auto editedVersion = image->Version(); + Require(image->Refresh() && image->Version() == editedVersion, "the CPU-edited image was uploaded twice"); + StorageTexture::FlushPending(address, BlockBytes, nullptr, "test untracked reuse"); + device.WaitIdle(); + Require(Count(block, SurfaceBytes, 0xff) == Threads * 4u, "refresh lost the unchanged block's GPU results"); + Require(Count(block, SurfaceBytes, 0x55) == SurfaceBytes - Threads * 4u, "refresh changed unrelated texels"); + Require(Count(block + SurfaceBytes, SurfaceBytes, 0x33) == SurfaceBytes, "write-back overwrote the CPU-edited block"); + + Dispatch(device, WriteCode, userData); + Require(image->Refresh(), "a freshly dispatched image did not retain its snapshot"); + std::memset(block, 0x77, SurfaceBytes); + image->WriteBack(); + device.WaitIdle(); + Require(Count(block, SurfaceBytes, 0x77) == SurfaceBytes, "write-back without refresh overwrote a CPU edit"); + image->Refresh(); + Require(image->Refresh(), "the image did not stabilize after write-back"); + AgcDriver::GuestMemory::StoreOwnBytes(address, SurfaceBytes, [&] { std::memset(block, 0x22, SurfaceBytes); }); + Require(!image->Refresh(), "a driver store within the comparison epoch was missed"); + Require(image->Refresh(), "the driver-edited image was uploaded twice"); + const std::array pattern{0x44444444u, 0x44444444u, 0x44444444u, 0x44444444u}; + Require(device.FillBuffer(address, SurfaceBytes, pattern), "the imported GPU fill was not recorded"); + Require(!image->Refresh(), "a pending GPU store within the comparison epoch was missed"); + device.WaitIdle(); + Require(Count(block, SurfaceBytes, 0x44) == SurfaceBytes, "the GPU fill did not reach guest memory"); + Require(image->Refresh(), "the GPU-edited image was uploaded twice"); +} + +#ifdef _WIN32 +void RunSharedImport(AgcDriver::VulkanDevice& device, std::uint8_t* block) { + using namespace AgcDriver::GuestMemory; + const auto address = AddressOf(block); + std::lock_guard gpu(GpuMutex()); + std::memset(block, 0x11, BlockBytes); + const std::array pattern{0x22222222u, 0x22222222u, 0x22222222u, 0x22222222u}; + Require(device.FillBuffer(address, SurfaceBytes, pattern), "shared memory was not imported"); + device.WaitIdle(); + Require(Watched(address, BlockBytes), "importing a separate alias disabled guest write tracking"); + auto* alias = static_cast(GuestArena::GuestArenaMapAlias_nid_postfix(address, BlockBytes)); + const auto beforeAliasWrite = CollectWritesUncached(address, BlockBytes); + alias[SurfaceBytes] = 0x11; + CollectWritesUncached(address, BlockBytes); + Require(UnchangedSince(address + SurfaceBytes, SurfaceBytes, beforeAliasWrite), "a driver alias write dirtied the protected guest view"); + GuestArena::GuestArenaUnmapAlias_nid_postfix(alias); + for (int repeat = 0; repeat < 8; ++repeat) { + const auto before = CollectWritesUncached(address, BlockBytes); + Require(device.FillBuffer(address, SurfaceBytes, pattern), "shared memory import was lost"); + device.WaitIdle(); + CollectWritesUncached(address, BlockBytes); + Require(UnchangedSince(address + SurfaceBytes, SurfaceBytes, before), "GPU access dirtied untouched guest pages"); + const auto current = CollectWritesUncached(address, BlockBytes); + block[SurfaceBytes] = static_cast(repeat); + CollectWritesUncached(address, BlockBytes); + Require(!UnchangedSince(address + SurfaceBytes, SurfaceBytes, current), "CPU write after GPU completion was missed"); + block[SurfaceBytes] = 0x11; + } +} +#endif + } int main() { try { std::optional block; std::optional watched; + std::optional untracked; + std::optional shared; const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; block.emplace(false); Run(*device, block->Data()); + untracked.emplace(false); + RunUntrackedReuse(*device, untracked->Data()); +#ifdef _WIN32 + shared.emplace(true, true); + RunSharedImport(*device, shared->Data()); + Run(*device, shared->Data()); +#endif if (AgcDriver::GuestMemory::WriteWatched()) { watched.emplace(true); Run(*device, watched->Data()); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DivFixupF16.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DivFixupF16.cpp index 175c75e45..13dcbe318 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DivFixupF16.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DivFixupF16.cpp @@ -108,7 +108,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -139,6 +139,7 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp index 001c96d22..39bf75b41 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp @@ -569,7 +569,7 @@ constexpr DivisionVector DivisionVectors[] = { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } class Shader { @@ -589,6 +589,7 @@ public: {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; result = ShaderRecompiler::Recompile(request); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DivisionOutputModifiers.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DivisionOutputModifiers.cpp index 69d530e1b..44c4c5299 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DivisionOutputModifiers.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DivisionOutputModifiers.cpp @@ -125,7 +125,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -177,6 +177,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityFloat64)) { + std::puts("skipped, the device has no shaderFloat64"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("division output modifiers tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DivisionResultModifiers.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DivisionResultModifiers.cpp index 8ea6352d8..f0a6a2922 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DivisionResultModifiers.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DivisionResultModifiers.cpp @@ -125,7 +125,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -181,6 +181,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityFloat64)) { + std::puts("skipped, the device has no shaderFloat64"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("division result modifiers tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16InactiveSource.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16InactiveSource.cpp index 1350e1ff2..c96044af4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16InactiveSource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16InactiveSource.cpp @@ -221,7 +221,7 @@ constexpr const char* Names[Checked] = {"mov_quad_perm", "mov_quad_perm_fi", "mo std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16Operands.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16Operands.cpp index 7c144a245..659c75b1a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16Operands.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16Operands.cpp @@ -117,7 +117,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16RowShareXmask.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16RowShareXmask.cpp index 0fd6a6861..2c65aa712 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16RowShareXmask.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Dpp16RowShareXmask.cpp @@ -223,7 +223,7 @@ constexpr const char* Names[Checked] = {"mov_row_share0", "mov_row_share5_fi", " std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Dpp8.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Dpp8.cpp index 61d69c415..7d59a352e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Dpp8.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Dpp8.cpp @@ -219,7 +219,7 @@ constexpr const char* Names[Checked] = {"mov", "mov_fi", "add_nc_u32", "add_f32" std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DsAppendConsume.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DsAppendConsume.cpp index 46ad1d7b6..90e7fc118 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DsAppendConsume.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DsAppendConsume.cpp @@ -59,7 +59,7 @@ alignas(256) constexpr auto Code = std::to_array({ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DsPermute.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DsPermute.cpp index 59177cf64..8e67595c5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DsPermute.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DsPermute.cpp @@ -108,7 +108,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DsSwizzle.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DsSwizzle.cpp index 4ef9a3949..bfae3c11c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DsSwizzle.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DsSwizzle.cpp @@ -223,7 +223,7 @@ constexpr const char* Names[Checked] = {"bitmask_identity", "bitmask_swap8", "bi std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DsWrxchg2.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DsWrxchg2.cpp index 9da6c4caf..b1dd9a3f0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DsWrxchg2.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DsWrxchg2.cpp @@ -55,7 +55,7 @@ alignas(256) constexpr auto DestinationOverflow = std::to_array({ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DsWrxchg2B64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DsWrxchg2B64.cpp index b07927ea4..5e2b9117e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DsWrxchg2B64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DsWrxchg2B64.cpp @@ -49,7 +49,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Dx10Clamp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Dx10Clamp.cpp new file mode 100644 index 000000000..d4910b991 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Dx10Clamp.cpp @@ -0,0 +1,243 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e160280u, 0x7e180280u, + 0x7e1a0280u, 0x7e1c0280u, 0x7e1e0280u, 0x7e200280u, 0x7e220280u, 0x7e240280u, 0x7e260280u, 0x7e280280u, + 0x7e2a0280u, 0x7e2c0280u, 0x7e2e0280u, 0x7e300280u, 0x7e320280u, 0xd503800au, 0x02020b04u, 0xd508800bu, + 0x02020b04u, 0xd54b800cu, 0x041a0b04u, 0xd510800du, 0x02020b04u, 0xd557800eu, 0x041a0b04u, 0xd5a0800fu, + 0x02010104u, 0xd5048010u, 0x02020d05u, 0xd5088011u, 0x0a020d04u, 0x2c3c0e90u, 0xd5328012u, 0x02023d07u, + 0xd5398013u, 0x02023d07u, 0xd74b9014u, 0x041e0f07u, 0xd58a8015u, 0x02010106u, 0xcc10d016u, 0x0a020f07u, + 0xcc0fd017u, 0x0a020f07u, 0xcc12d018u, 0x0a020f07u, 0xcc0ed019u, 0x0c1e0f07u, 0xe0781000u, 0x80010a03u, + 0xe0781010u, 0x80010e03u, 0xe0781020u, 0x80011203u, 0xe0781030u, 0x80011603u, 0xbf810000u, +}; + +constexpr std::uint32_t Rows[32][4] = { + {0x7fc00000u, 0xff812345u, 0xffc00000u, 0x7c007e00u}, + {0xffc00000u, 0x40400000u, 0xff800000u, 0x7e00fd23u}, + {0x7f800001u, 0xffc00000u, 0x00000000u, 0x0000fc00u}, + {0xff812345u, 0xbf800000u, 0x7fc00000u, 0xfc003800u}, + {0x7fc12345u, 0x3f800001u, 0x7f800000u, 0xfe008000u}, + {0x7f800000u, 0xff800000u, 0x40400000u, 0x80003c01u}, + {0xff800000u, 0xbe800000u, 0x3f800001u, 0x3c007c01u}, + {0x3f800000u, 0x7fc12345u, 0x7fc12345u, 0x7c017c00u}, + {0xbf800000u, 0x00000000u, 0x3f000000u, 0xb400bc00u}, + {0x3f000000u, 0x7f800001u, 0x3f7fffffu, 0xbc000000u}, + {0x40400000u, 0x3f000000u, 0xff812345u, 0xfd233bffu}, + {0x00000000u, 0x7fc00000u, 0xbf800000u, 0x3bfffe00u}, + {0x80000000u, 0x3f800000u, 0xbe800000u, 0x38007e55u}, + {0xbe800000u, 0x3f7fffffu, 0x7f800001u, 0x7e553c00u}, + {0x3f7fffffu, 0x7f800000u, 0x3f800000u, 0x3c014200u}, + {0x3f800001u, 0x80000000u, 0x80000000u, 0x4200b400u}, + {0x7fc00000u, 0xff812345u, 0xffc00000u, 0x7c007e00u}, + {0xffc00000u, 0x40400000u, 0xff800000u, 0x7e00fd23u}, + {0x7f800001u, 0xffc00000u, 0x00000000u, 0x0000fc00u}, + {0xff812345u, 0xbf800000u, 0x7fc00000u, 0xfc003800u}, + {0x7fc12345u, 0x3f800001u, 0x7f800000u, 0xfe008000u}, + {0x7f800000u, 0xff800000u, 0x40400000u, 0x80003c01u}, + {0xff800000u, 0xbe800000u, 0x3f800001u, 0x3c007c01u}, + {0x3f800000u, 0x7fc12345u, 0x7fc12345u, 0x7c017c00u}, + {0xbf800000u, 0x00000000u, 0x3f000000u, 0xb400bc00u}, + {0x3f000000u, 0x7f800001u, 0x3f7fffffu, 0xbc000000u}, + {0x40400000u, 0x3f000000u, 0xff812345u, 0xfd233bffu}, + {0x00000000u, 0x7fc00000u, 0xbf800000u, 0x3bfffe00u}, + {0x80000000u, 0x3f800000u, 0xbe800000u, 0x38007e55u}, + {0xbe800000u, 0x3f7fffffu, 0x7f800001u, 0x7e553c00u}, + {0x3f7fffffu, 0x7f800000u, 0x3f800000u, 0x3c014200u}, + {0x3f800001u, 0x80000000u, 0x80000000u, 0x4200b400u} +}; +constexpr std::uint32_t ExpectedDx10[32][16] = { + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003800u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x38003800u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x3c003c00u, 0x3c003c00u, 0x00003c00u}, + {0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003800u, 0x34003400u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3f000000u, 0x3f000000u, 0x00000000u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003bffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3bff3bffu, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003bffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3bff3bffu, 0x00000000u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00003800u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x38003800u, 0x00000000u}, + {0x3f3fffffu, 0x00000000u, 0x00000000u, 0x3f7fffffu, 0x00000000u, 0x3f400000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x00000000u}, + {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f7fffffu, 0x3f800000u, 0x3f800000u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x34000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x3c003c00u, 0x3c000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003800u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x38003800u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x3c003c00u, 0x3c003c00u, 0x00003c00u}, + {0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003800u, 0x34003400u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3f000000u, 0x3f000000u, 0x00000000u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003bffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3bff3bffu, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003bffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3bff3bffu, 0x00000000u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00003800u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x38003800u, 0x00000000u}, + {0x3f3fffffu, 0x00000000u, 0x00000000u, 0x3f7fffffu, 0x00000000u, 0x3f400000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x00000000u}, + {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f7fffffu, 0x3f800000u, 0x3f800000u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x34000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x3c003c00u, 0x3c000000u} +}; +constexpr std::uint32_t ExpectedNoDx10[32][16] = { + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0xff812345u, 0x7fc00000u, 0x00007e00u, 0x00003c00u, 0x00007e00u, 0x0000fe00u, 0x7e007e00u, 0x7e007e00u, 0x3c003c00u, 0x7e007e00u}, + {0xffc00000u, 0xffc00000u, 0xffc00000u, 0x3f800000u, 0x00000000u, 0xffc00000u, 0x3f800000u, 0xffc00000u, 0x0000fd23u, 0x0000fd23u, 0x0000fd23u, 0x00000000u, 0x7e00fd23u, 0x7e00fd23u, 0x7e00fd23u, 0x7e00fd23u}, + {0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x00000000u, 0x7f800001u, 0xffc00000u, 0x7f800001u, 0x00000000u, 0x00000000u, 0x0000fe00u, 0x00000000u, 0xfe00fe00u, 0x00000000u, 0x00000000u, 0xfe00fe00u}, + {0xff812345u, 0xff812345u, 0xff812345u, 0x00000000u, 0x00000000u, 0xff812345u, 0xffc00000u, 0xff812345u, 0x00000000u, 0x00003800u, 0x00000000u, 0x00007e00u, 0x00000000u, 0x00000000u, 0x38003800u, 0x00000000u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x3f800000u, 0x3f800000u, 0x7fc12345u, 0x00000000u, 0x7fc12345u, 0x0000fe00u, 0x00000000u, 0x0000fe00u, 0x00003c00u, 0xfe00fe00u, 0xfe00fe00u, 0x00000000u, 0xfe00fe00u}, + {0xffc00000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0xffc00000u, 0x00000000u, 0x3f800000u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x3c003c00u, 0x3c003c00u, 0x00003c00u}, + {0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00007c01u, 0x00003c00u, 0x00007c01u, 0x00003c00u, 0x7c017c01u, 0x7c017c01u, 0x3c003c00u, 0x7c017c01u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x7fc12345u, 0x7fc12345u, 0x00007c01u, 0x00003c00u, 0x00007c01u, 0x00007e09u, 0x7c017c01u, 0x7c017c01u, 0x3c003c00u, 0x7c017c01u}, + {0x00000000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003800u, 0x34003400u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x7f800001u, 0x7f800001u, 0x7f800001u, 0x3f000000u, 0x3f000000u, 0x3f000000u, 0x7f800001u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x3f800000u, 0x3f800000u, 0xff812345u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x7f812345u, 0xff812345u, 0x0000fd23u, 0x00003bffu, 0x0000fd23u, 0x0000fe09u, 0xfd23fd23u, 0xfd23fd23u, 0x3bff3bffu, 0xfd23fd23u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u, 0x00000000u, 0x0000fe00u, 0x00003bffu, 0x0000fe00u, 0x00000000u, 0xfe00fe00u, 0xfe00fe00u, 0x3bff3bffu, 0xfe00fe00u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00007e55u, 0x00003800u, 0x00007e55u, 0x00000000u, 0x7e557e55u, 0x7e557e55u, 0x38003800u, 0x7e557e55u}, + {0x3f3fffffu, 0x00000000u, 0x7f800001u, 0x3f7fffffu, 0x00000000u, 0x3f400000u, 0xff800001u, 0x7f800001u, 0x00007e55u, 0x00003c00u, 0x00007e55u, 0x00007e00u, 0x7e557e55u, 0x7e557e55u, 0x3c003c00u, 0x7e557e55u}, + {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f7fffffu, 0x3f800000u, 0x3f800000u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x34000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x3c003c00u, 0x3c000000u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0xff812345u, 0x7fc00000u, 0x00007e00u, 0x00003c00u, 0x00007e00u, 0x0000fe00u, 0x7e007e00u, 0x7e007e00u, 0x3c003c00u, 0x7e007e00u}, + {0xffc00000u, 0xffc00000u, 0xffc00000u, 0x3f800000u, 0x00000000u, 0xffc00000u, 0x3f800000u, 0xffc00000u, 0x0000fd23u, 0x0000fd23u, 0x0000fd23u, 0x00000000u, 0x7e00fd23u, 0x7e00fd23u, 0x7e00fd23u, 0x7e00fd23u}, + {0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x00000000u, 0x7f800001u, 0xffc00000u, 0x7f800001u, 0x00000000u, 0x00000000u, 0x0000fe00u, 0x00000000u, 0xfe00fe00u, 0x00000000u, 0x00000000u, 0xfe00fe00u}, + {0xff812345u, 0xff812345u, 0xff812345u, 0x00000000u, 0x00000000u, 0xff812345u, 0xffc00000u, 0xff812345u, 0x00000000u, 0x00003800u, 0x00000000u, 0x00007e00u, 0x00000000u, 0x00000000u, 0x38003800u, 0x00000000u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x3f800000u, 0x3f800000u, 0x7fc12345u, 0x00000000u, 0x7fc12345u, 0x0000fe00u, 0x00000000u, 0x0000fe00u, 0x00003c00u, 0xfe00fe00u, 0xfe00fe00u, 0x00000000u, 0xfe00fe00u}, + {0xffc00000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0xffc00000u, 0x00000000u, 0x3f800000u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x3c003c00u, 0x3c003c00u, 0x00003c00u}, + {0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00007c01u, 0x00003c00u, 0x00007c01u, 0x00003c00u, 0x7c017c01u, 0x7c017c01u, 0x3c003c00u, 0x7c017c01u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x7fc12345u, 0x7fc12345u, 0x00007c01u, 0x00003c00u, 0x00007c01u, 0x00007e09u, 0x7c017c01u, 0x7c017c01u, 0x3c003c00u, 0x7c017c01u}, + {0x00000000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003800u, 0x34003400u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x7f800001u, 0x7f800001u, 0x7f800001u, 0x3f000000u, 0x3f000000u, 0x3f000000u, 0x7f800001u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x3f800000u, 0x3f800000u, 0xff812345u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x7f812345u, 0xff812345u, 0x0000fd23u, 0x00003bffu, 0x0000fd23u, 0x0000fe09u, 0xfd23fd23u, 0xfd23fd23u, 0x3bff3bffu, 0xfd23fd23u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u, 0x00000000u, 0x0000fe00u, 0x00003bffu, 0x0000fe00u, 0x00000000u, 0xfe00fe00u, 0xfe00fe00u, 0x3bff3bffu, 0xfe00fe00u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00007e55u, 0x00003800u, 0x00007e55u, 0x00000000u, 0x7e557e55u, 0x7e557e55u, 0x38003800u, 0x7e557e55u}, + {0x3f3fffffu, 0x00000000u, 0x7f800001u, 0x3f7fffffu, 0x00000000u, 0x3f400000u, 0xff800001u, 0x7f800001u, 0x00007e55u, 0x00003c00u, 0x00007e55u, 0x00007e00u, 0x7e557e55u, 0x7e557e55u, 0x3c003c00u, 0x7e557e55u}, + {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f7fffffu, 0x3f800000u, 0x3f800000u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u}, + {0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x34000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c003c00u, 0x3c003c00u, 0x3c000000u} +}; +constexpr const char* Names[16] = { + "v_add_f32 clamp", + "v_mul_f32 clamp", + "v_fma_f32 clamp", + "v_max_f32 clamp", + "v_med3_f32 clamp", + "v_fract_f32 clamp", + "v_sub_f32 clamp", + "v_mul_f32 clamp mul:2", + "v_add_f16 clamp", + "v_max_f16 clamp", + "v_fma_f16 clamp", + "v_cvt_f16_f32 clamp", + "v_pk_mul_f16 clamp", + "v_pk_add_f16 clamp", + "v_pk_max_f16 clamp", + "v_pk_fma_f16 clamp" +}; + +void Fill(std::uint32_t tid, std::uint32_t* words) { + std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words); +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) { + Require(actual == expected, std::string("dx10 clamp: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); +} + +bool IsNan32(std::uint32_t value) { + return (value & 0x7fffffffu) > 0x7f800000u; +} + +bool IsNan16(std::uint32_t value) { + return (value & 0x7fffu) > 0x7c00u; +} + +void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = floatMode; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(const std::uint32_t (&expected)[32][16], const char* mode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t i = 0; i < 16; ++i) { + const std::string name = std::string(mode) + " " + Names[i]; + const bool driverNan = (i < 8u && IsNan32(expected[tid][i])) || (i >= 8u && i < 12u && IsNan16(expected[tid][i])); + if (driverNan) { + Require(i < 8u ? IsNan32(out[i]) : IsNan16(out[i]), std::string("dx10 clamp: lane ") + std::to_string(tid) + " " + name + " is " + Hex(out[i]) + ", expected a NaN"); + continue; + } + Expect(tid, i >= 8u && i < 12u ? out[i] & 0xffffu : out[i], i >= 8u && i < 12u ? expected[tid][i] & 0xffffu : expected[tid][i], name.c_str()); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, std::nullopt); + Check(ExpectedDx10, "no float mode"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + Check(ExpectedDx10, "DX10_CLAMP=1"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, false, false, false}); + Check(ExpectedNoDx10, "DX10_CLAMP=0"); + std::puts("dx10 clamp tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/EmulatedColorCompare.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/EmulatedColorCompare.cpp index c664f1d29..54514f210 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/EmulatedColorCompare.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/EmulatedColorCompare.cpp @@ -67,7 +67,7 @@ struct Sampler { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(std::uint32_t format) { @@ -166,7 +166,7 @@ float Expected(std::uint32_t tid, const Sampler& sampler, bool offsets) { return top * (1.0f - b) + bottom * b; } -ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::uint32_t format, const Sampler& sampler, std::span code = Code) { +ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::uint32_t format, const Sampler& sampler, std::span code = Code, bool useCache = false, bool nativeSampleOffsets = true) { std::vector userData(24, 0u); const auto input = BufferDescriptor(Input.data(), static_cast(sizeof(Input))); const auto output = BufferDescriptor(Output.data(), static_cast(sizeof(Output))); @@ -186,14 +186,19 @@ ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std:: device.Target(), {0, 0, 0, 128} }; - request.useCache = false; + request.useCache = useCache; + request.target.nonConstantImageOffsets = request.target.nonConstantImageOffsets && nativeSampleOffsets; return ShaderRecompiler::Recompile(request); } -void Run(AgcDriver::VulkanDevice& device, const Sampler& sampler, const char* name, bool offsets = false) { +void Run(AgcDriver::VulkanDevice& device, const Sampler& sampler, const char* name, bool offsets = false, bool useCache = false) { Output.fill(-1.0f); const std::span code = offsets ? std::span(OffsetCode) : std::span(Code); - const auto result = Compile(device, Format8888UNorm, sampler, code); + const auto result = Compile(device, Format8888UNorm, sampler, code, useCache); + for (const auto& binding : result.bindings) { + Require(binding.role != ShaderRecompiler::DescriptorRole::GuestSamplers, "emulated comparison retained a sampler binding"); + Require(std::all_of(binding.imageSamplers.begin(), binding.imageSamplers.end(), [](auto mask) { return mask == 0u; }), "emulated comparison retained an unused sampler association"); + } device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); for (std::uint32_t tid = 0; tid < Threads; ++tid) { @@ -209,9 +214,9 @@ bool BindsDepthCompare(const ShaderRecompiler::RecompileResult& result) { }); } -void Reject(AgcDriver::VulkanDevice& device, std::uint32_t format, const Sampler& sampler, std::string_view reason) { +void Reject(AgcDriver::VulkanDevice& device, std::uint32_t format, const Sampler& sampler, std::string_view reason, std::span code = Code, bool nativeSampleOffsets = true) { try { - static_cast(Compile(device, format, sampler)); + static_cast(Compile(device, format, sampler, code, false, nativeSampleOffsets)); } catch (const std::exception& error) { Require(std::string_view(error.what()).find(reason) != std::string_view::npos, std::string("unexpected rejection: ") + error.what()); return; @@ -240,6 +245,11 @@ int main() { Run(*device, {ClampEdge, FilterPoint}, "point, clamp to edge, offsets, bias and LOD clamp", true); Run(*device, {ClampWrap, FilterBilinear}, "bilinear, wrap, offsets, bias and LOD clamp", true); Run(*device, {ClampBorder, FilterBilinear, BorderWhite, 0u, 0x3f00u}, "bilinear, white border, offsets, bias and LOD clamp", true); + for (unsigned iteration = 0; iteration < 2; ++iteration) { + Run(*device, {ClampEdge, FilterPoint}, "cached point comparison", false, true); + Run(*device, {ClampWrap, FilterBilinear}, "cached bilinear comparison with offsets", true, true); + } + Reject(*device, Format32Float, {ClampEdge, FilterPoint}, "native comparison with a nonconstant texel offset requires", OffsetCode, false); Reject(*device, Format8888UInt, {ClampEdge, FilterPoint}, "unsupported format"); Reject(*device, Format8888UNorm, {ClampMirror, FilterPoint}, "wrap, clamp-to-edge or clamp-to-border"); Reject(*device, Format8888UNorm, {ClampHalfBorder, FilterPoint}, "wrap, clamp-to-edge or clamp-to-border"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ExecHostLanes.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ExecHostLanes.cpp new file mode 100644 index 000000000..9d1cbec5b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ExecHostLanes.cpp @@ -0,0 +1,182 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Vertices = 189; +constexpr std::uint32_t Threads = 192; +constexpr std::uint32_t Keys = 6; +constexpr std::uint32_t KeyStride = 440; +constexpr std::uint32_t PassLimit = 128; +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 64; +alignas(256) std::array LaneKeys{}; +alignas(256) std::array Table{}; +alignas(256) std::array Output{}; +alignas(256) std::array Pixels{}; + +alignas(256) constexpr std::array VertexCode{ + 0x8700ff03, 0x000000ff, 0x81ea00c0, 0x90fe6ac1, 0xe0302000, 0x80020c05, 0x7e3c0305, 0x36080a81, + 0x7e080d04, 0x7e2a02c1, 0xbea4040c, 0xbea6040e, 0xbea80380, 0xbf8c3f70, 0xbe96047e, 0xbe88047e, + 0xbeb0047e, 0x7c0808f9, 0x068698f2, 0xbf880023, 0x80288128, 0xbf0aff28, 0x00000080, 0xbf84001f, + 0xbeea1416, 0xd7600020, 0x0000d50c, 0x7d8418f9, 0x06869a20, 0xd5010004, 0x00690283, 0x7da40881, + 0xbf88000b, 0x7e0a0280, 0x87ea7e18, 0x8afe6a7e, 0x8ab06a30, 0xbf880006, 0x7e080283, 0xb82001b8, + 0xf4200792, 0x40000000, 0xbf8cc07f, 0x7e2a021e, 0xbefe0430, 0x7da40883, 0xbf880008, 0x7e0a0281, + 0x8a961a16, 0xbf128016, 0xbf800000, 0x85ea807e, 0x8ab06a30, 0x8afe6a7e, 0xbf82ffdc, 0xbefe0408, + 0x7e2c0228, 0x7e2e030c, 0x7e300305, 0xe0782000, 0x8004151e, 0x7e000280, 0x7e020280, 0x7e040280, + 0x7e0602f2, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +alignas(256) constexpr std::array ComputeCode{ + 0xbefe04c1, 0xe0302000, 0x80020c00, 0x7e3c0300, 0x36080081, 0x7e080d04, 0x7e2a02c1, 0xbea4040c, + 0xbea6040e, 0xbea80380, 0xbf8c3f70, 0xbe96047e, 0xbe88047e, 0xbeb0047e, 0x7c0808f9, 0x068698f2, + 0xbf880023, 0x80288128, 0xbf0aff28, 0x00000080, 0xbf84001f, 0xbeea1416, 0xd7600020, 0x0000d50c, + 0x7d8418f9, 0x06869a20, 0xd5010004, 0x00690283, 0x7da40881, 0xbf88000b, 0x7e0a0280, 0x87ea7e18, + 0x8afe6a7e, 0x8ab06a30, 0xbf880006, 0x7e080283, 0xb82001b8, 0xf4200792, 0x40000000, 0xbf8cc07f, + 0x7e2a021e, 0xbefe0430, 0x7da40883, 0xbf880008, 0x7e0a0281, 0x8a961a16, 0xbf128016, 0xbf800000, + 0x85ea807e, 0x8ab06a30, 0x8afe6a7e, 0xbf82ffdc, 0xbefe0408, 0x7e2c0228, 0x7e2e030c, 0x7e300305, + 0xe0782000, 0x8004151e, 0xbf810000, +}; + +alignas(256) constexpr std::array PixelCode{0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000}; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +std::uint32_t Entry(std::uint32_t key) { + return 0x5a000000u + key * 0x00010101u; +} + +std::vector Prepare(std::uint32_t lanes) { + for (std::uint32_t lane = 0; lane < Threads; ++lane) LaneKeys[lane] = (lane * 5u + lane / 7u) % Keys; + Table.fill(0u); + for (std::uint32_t key = 0; key < Keys; ++key) Table[key * KeyStride / 4u] = Entry(key); + Output.fill(0xdeadbeefu); + std::vector userData(12, 0u); + const auto keys = BufferDescriptor(LaneKeys.data(), 4u, lanes); + const auto table = BufferDescriptor(Table.data(), 0u, static_cast(Table.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), 16u, lanes); + std::copy(keys.begin(), keys.end(), userData.begin()); + std::copy(table.begin(), table.end(), userData.begin() + 4); + std::copy(output.begin(), output.end(), userData.begin() + 8); + return userData; +} + +void Draw(AgcDriver::VulkanDevice& device) { + const auto vertexUserData = Prepare(Vertices); + Pixels.fill(std::byte{0}); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {64, 8, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + device.Target(), + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX); + pixel.posX = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData(4, 0u); + const std::array pixelMemory{{{reinterpret_cast(PixelCode.data()), std::as_bytes(std::span(PixelCode))}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, + {64, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + device.Target(), + {0, 0, vertexPush, 128 - vertexPush} + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, vertexPush} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 64u, 64u, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, Vertices, 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); +} + +void Dispatch(AgcDriver::VulkanDevice& device) { + const auto userData = Prepare(Threads); + const std::span code(ComputeCode); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {64, 8, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(const std::string& name, std::uint32_t lanes) { + for (std::uint32_t lane = 0; lane < lanes; ++lane) { + const auto what = "exec host lanes: " + name + " " + std::to_string(lane); + const auto key = LaneKeys[lane]; + const bool inside = (lane & 1u) == 0u; + const auto* result = &Output[lane * 4u]; + Require(result[2] == key, what + " read key " + std::to_string(result[2]) + ", expected " + std::to_string(key)); + Require(result[1] >= 1u && result[1] <= Keys, what + " ran " + std::to_string(result[1]) + " passes of the waterfall loop over EXEC copied by s_mov_b64, expected 1 to " + std::to_string(Keys) + " (the loop stops at " + std::to_string(PassLimit) + ")"); + Require(result[3] == (inside ? 0u : 1u), what + " left the loop with v5 " + std::to_string(result[3]) + ", expected " + std::to_string(inside ? 0u : 1u)); + const auto expected = inside ? 0xffffffffu : Entry(key); + Require(result[0] == expected, what + " loaded " + std::to_string(result[0]) + " through its scalarized key, expected " + std::to_string(expected)); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Dispatch(*device); + Check("compute lane", Threads); + if (device->Target().subgroupSize < 32u) { + std::printf("vertex draw skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize); + } else { + Draw(*device); + Check("vertex", Vertices); + } + std::puts("exec host lane tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F32ArithmeticNan.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F32ArithmeticNan.cpp new file mode 100644 index 000000000..c78f91002 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F32ArithmeticNan.cpp @@ -0,0 +1,238 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e160280u, 0x7e180280u, + 0x7e1a0280u, 0x7e1c0280u, 0x7e1e0280u, 0x7e200280u, 0x7e220280u, 0x7e240280u, 0x7e260280u, 0x7e280280u, + 0x7e2a0280u, 0x7e2c0280u, 0x7e2e0280u, 0x7e300280u, 0x7e320280u, 0x06140b04u, 0x08160b04u, 0x0a180b04u, + 0x101a0b04u, 0xd54b000eu, 0x041a0b04u, 0x7e1e0306u, 0x561e0b04u, 0x0e200b04u, 0xd5400011u, 0x041a0b04u, + 0x58240b04u, 0x7f812345u, 0x5a260b04u, 0xffc12345u, 0xcc200014u, 0x041a0b04u, 0xd5030215u, 0x22020b04u, + 0xd5080016u, 0x0a020b04u, 0xd54b0017u, 0xa41a0b04u, 0xd5040118u, 0x42020b04u, 0xd5038019u, 0x02020b04u, + 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, 0xe0781020u, 0x80011203u, 0xe0781030u, 0x80011603u, + 0xbf810000u, +}; + +constexpr std::uint32_t Rows[32][4] = { + {0x7f812345u, 0x3f800000u, 0x3f800000u, 0x00000000u}, + {0x3f800000u, 0xffa00001u, 0x3f800000u, 0x00000000u}, + {0x3f800000u, 0x3f800000u, 0x7f812345u, 0x00000000u}, + {0x7fc00003u, 0x3f800000u, 0x3f800000u, 0x00000000u}, + {0x3f800000u, 0xffc12345u, 0x3f800000u, 0x00000000u}, + {0x3f800000u, 0x3f800000u, 0xffc12345u, 0x00000000u}, + {0x7f812345u, 0xffa00001u, 0x7fc00003u, 0x00000000u}, + {0x7fc00003u, 0xffa00001u, 0x7f812345u, 0x00000000u}, + {0xffc12345u, 0x7fc00003u, 0xffa00001u, 0x00000000u}, + {0xffa00001u, 0x7f812345u, 0xffc12345u, 0x00000000u}, + {0x7f800000u, 0x00000000u, 0x7fc00003u, 0x00000000u}, + {0x00000000u, 0xff800000u, 0x7f812345u, 0x00000000u}, + {0x7f800000u, 0x00000000u, 0x3f800000u, 0x00000000u}, + {0x7f800000u, 0xff800000u, 0x3f800000u, 0x00000000u}, + {0x7f800000u, 0x7f800000u, 0xff800000u, 0x00000000u}, + {0xff800000u, 0xff800000u, 0x3f800000u, 0x00000000u}, + {0x00000000u, 0x7f812345u, 0x3f800000u, 0x00000000u}, + {0xffa00001u, 0x00000000u, 0x3f800000u, 0x00000000u}, + {0x00000000u, 0xffc12345u, 0x7f812345u, 0x00000000u}, + {0x7fc00003u, 0x80000000u, 0xffc12345u, 0x00000000u}, + {0x7f800000u, 0x3f800000u, 0xff800000u, 0x00000000u}, + {0x3f800000u, 0x7f800000u, 0xff800000u, 0x00000000u}, + {0xbf800000u, 0x7f800000u, 0x7f800000u, 0x00000000u}, + {0x00000000u, 0x3f800000u, 0x7fc00003u, 0x00000000u}, + {0x80000000u, 0x80000000u, 0xffa00001u, 0x00000000u}, + {0x3f800000u, 0xbf800000u, 0x00000000u, 0x00000000u}, + {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x00000000u}, + {0xbf800000u, 0x3f800000u, 0x00000000u, 0x00000000u}, + {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x00000000u}, + {0x7f812345u, 0x7f812345u, 0x7f812345u, 0x00000000u}, + {0xffc12345u, 0xffc12345u, 0xffc12345u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u} +}; +constexpr std::uint32_t ExpectedIeee[32][16] = { + {0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x00000000u}, + {0xffe00001u, 0x7fe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0x7fc12345u, 0xffe00001u, 0xffe00001u, 0x7fe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0x00000000u}, + {0x40000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7fc12345u, 0x7fc12345u, 0x3f800000u, 0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x00000000u, 0x3f800000u, 0xffc12345u, 0x40000000u, 0x3f800000u}, + {0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x00000000u}, + {0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x00000000u}, + {0x40000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0xffc12345u, 0xffc12345u, 0x3f800000u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0x00000000u, 0x3f800000u, 0x7fc12345u, 0x40000000u, 0x3f800000u}, + {0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x00000000u}, + {0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x00000000u}, + {0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x00000000u}, + {0xffe00001u, 0xffe00001u, 0x7fe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0x7fe00001u, 0xffe00001u, 0x7fe00001u, 0x7fe00001u, 0x00000000u}, + {0x7f800000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x7fc00003u, 0x7fc12345u, 0xffc00000u, 0xffc00000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0xff800000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x7fc12345u, 0x7fc12345u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0xffc00000u, 0xff800000u, 0x00000000u}, + {0x7f800000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x3f800000u, 0x7fc12345u, 0xffc00000u, 0xffc00000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0xffc00000u, 0x7f800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0x7fc12345u, 0xffc12345u, 0xff800000u, 0xffc00000u, 0xff800000u, 0x7f800000u, 0xffc00000u, 0x00000000u}, + {0x7f800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0x7fc12345u, 0xffc12345u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0xff800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7fc12345u, 0xffc12345u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0x00000000u}, + {0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x00000000u, 0x3f800000u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x00000000u}, + {0xffe00001u, 0xffe00001u, 0x7fe00001u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0x00000000u, 0x3f800000u, 0xffe00001u, 0xffe00001u, 0xffe00001u, 0x7fe00001u, 0xffe00001u, 0x7fe00001u, 0x7fe00001u, 0x00000000u}, + {0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x00000000u, 0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x00000000u}, + {0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x00000000u, 0xffc12345u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x00000000u}, + {0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0x7fc12345u, 0xffc12345u, 0xffc00000u, 0xff800000u, 0x7f800000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0x7fc12345u, 0xffc12345u, 0xffc00000u, 0x7f800000u, 0x7f800000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0x7f800000u, 0xff800000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0xff800000u, 0xffc00000u, 0x7fc12345u, 0xffc12345u, 0xffc00000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0x3f800000u, 0xbf800000u, 0x3f800000u, 0x00000000u, 0x7fc00003u, 0x7fc00003u, 0x00000000u, 0x7fc00003u, 0x7fc12345u, 0xffc12345u, 0x7fc00003u, 0x3f800000u, 0x00000000u, 0xffc00003u, 0x3f800000u, 0x3f800000u}, + {0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffe00001u, 0xffe00001u, 0x00000000u, 0xffe00001u, 0x7fc12345u, 0xffc12345u, 0xffe00001u, 0x00000000u, 0x00000000u, 0x7fe00001u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x40000000u, 0xc0000000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0x7fc12345u, 0xffc12345u, 0xbf800000u, 0x00000000u, 0xbf800000u, 0x3f800000u, 0x00000000u, 0x00000000u}, + {0x40000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x40000000u, 0x40000000u, 0x3f800000u, 0x40000000u, 0x7fc12345u, 0xffc12345u, 0x40000000u, 0x00000000u, 0x3f800000u, 0xc0000000u, 0x40000000u, 0x3f800000u}, + {0x00000000u, 0xc0000000u, 0x40000000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0x7fc12345u, 0xffc12345u, 0xbf800000u, 0x40000000u, 0xbf800000u, 0x3f800000u, 0x40000000u, 0x00000000u}, + {0x7f800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7fc12345u, 0xffc12345u, 0x7f800000u, 0xffc00000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x3f800000u}, + {0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x00000000u}, + {0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc12345u, 0xffc12345u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u} +}; +constexpr std::uint32_t ExpectedNoIeee[32][16] = { + {0x7f812345u, 0x7f812345u, 0xff812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0xff812345u, 0x7f812345u, 0xff812345u, 0x7f812345u, 0x00000000u}, + {0xffa00001u, 0x7fa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0x7f812345u, 0xffa00001u, 0xffa00001u, 0x7fa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0x00000000u}, + {0x40000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f812345u, 0x7f812345u, 0x3f800000u, 0x7f812345u, 0x7f812345u, 0xffc12345u, 0x7f812345u, 0x00000000u, 0x40000000u, 0xff812345u, 0x40000000u, 0x3f800000u}, + {0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x00000000u}, + {0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x7f812345u, 0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x00000000u}, + {0x40000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0xffc12345u, 0xffc12345u, 0x3f800000u, 0xffc12345u, 0x7f812345u, 0xffc12345u, 0xffc12345u, 0x00000000u, 0x40000000u, 0x7fc12345u, 0x40000000u, 0x3f800000u}, + {0x7f812345u, 0x7f812345u, 0xff812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0xff812345u, 0x7f812345u, 0xff812345u, 0x7f812345u, 0x00000000u}, + {0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x00000000u}, + {0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x00000000u}, + {0xffa00001u, 0xffa00001u, 0x7fa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0x7fa00001u, 0xffa00001u, 0x7fa00001u, 0x7fa00001u, 0x00000000u}, + {0x7f800000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x7fc00003u, 0x7f812345u, 0xffc00000u, 0xffc00000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0xff800000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x7f812345u, 0x7f812345u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0xffc00000u, 0xff800000u, 0x00000000u}, + {0x7f800000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x3f800000u, 0x7f812345u, 0xffc00000u, 0xffc00000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0xffc00000u, 0x7f800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0x7f812345u, 0xffc12345u, 0xff800000u, 0xffc00000u, 0xff800000u, 0x7f800000u, 0xffc00000u, 0x00000000u}, + {0x7f800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0x7f812345u, 0xffc12345u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0xff800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f812345u, 0xffc12345u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0x00000000u}, + {0x7f812345u, 0xff812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x00000000u, 0x3f800000u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x00000000u}, + {0xffa00001u, 0xffa00001u, 0x7fa00001u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0x00000000u, 0x3f800000u, 0xffa00001u, 0xffa00001u, 0xffa00001u, 0x7fa00001u, 0xffa00001u, 0x7fa00001u, 0x7fa00001u, 0x00000000u}, + {0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x00000000u, 0x7f812345u, 0x7f812345u, 0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x00000000u}, + {0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0x00000000u, 0xffc12345u, 0x7fc00003u, 0x7fc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0xffc00003u, 0x7fc00003u, 0x00000000u}, + {0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0x7f812345u, 0xffc12345u, 0xffc00000u, 0xff800000u, 0x7f800000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0xffc00000u, 0x7f812345u, 0xffc12345u, 0xffc00000u, 0x7f800000u, 0x7f800000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0x7f800000u, 0xff800000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0xffc00000u, 0xff800000u, 0xffc00000u, 0x7f812345u, 0xffc12345u, 0xffc00000u, 0x7f800000u, 0xff800000u, 0xffc00000u, 0x7f800000u, 0x3f800000u}, + {0x3f800000u, 0xbf800000u, 0x3f800000u, 0x00000000u, 0x7fc00003u, 0x7fc00003u, 0x00000000u, 0x7fc00003u, 0x7f812345u, 0xffc12345u, 0x7fc00003u, 0x3f800000u, 0x00000000u, 0xffc00003u, 0x3f800000u, 0x3f800000u}, + {0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffa00001u, 0xffa00001u, 0x00000000u, 0xffa00001u, 0x7f812345u, 0xffc12345u, 0xffa00001u, 0x00000000u, 0x00000000u, 0x7fa00001u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x40000000u, 0xc0000000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0x7f812345u, 0xffc12345u, 0xbf800000u, 0x00000000u, 0xc0000000u, 0x3f800000u, 0x00000000u, 0x00000000u}, + {0x40000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x40000000u, 0x40000000u, 0x3f800000u, 0x40000000u, 0x7f812345u, 0xffc12345u, 0x40000000u, 0x00000000u, 0x40000000u, 0xc0000000u, 0x40000000u, 0x3f800000u}, + {0x00000000u, 0xc0000000u, 0x40000000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0x7f812345u, 0xffc12345u, 0xbf800000u, 0x40000000u, 0xc0000000u, 0x3f800000u, 0x40000000u, 0x00000000u}, + {0x7f800000u, 0xffc00000u, 0xffc00000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f812345u, 0xffc12345u, 0x7f800000u, 0xffc00000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x3f800000u}, + {0x7f812345u, 0x7f812345u, 0xff812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0xff812345u, 0x7f812345u, 0xff812345u, 0x7f812345u, 0x00000000u}, + {0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7f812345u, 0xffc12345u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u} +}; +constexpr const char* Names[16] = { + "v_add_f32", + "v_sub_f32", + "v_subrev_f32", + "v_mul_f32", + "v_fma_f32", + "v_fmac_f32", + "v_mul_legacy_f32", + "v_fma_legacy_f32", + "v_fmamk_f32", + "v_fmaak_f32", + "v_fma_mix_f32", + "v_add_f32 -v4 |v5|", + "v_mul_f32 mul:2", + "v_fma_f32 -v4 v5 -v6", + "v_sub_f32 |v4| -v5", + "v_add_f32 clamp" +}; + +void Fill(std::uint32_t tid, std::uint32_t* words) { + std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words); +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) { + Require(actual == expected, std::string("f32 arithmetic nan: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); +} + +bool IsNan32(std::uint32_t value) { + return (value & 0x7fffffffu) > 0x7f800000u; +} + +bool IsNan16(std::uint32_t value) { + return (value & 0x7fffu) > 0x7c00u; +} + +void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = floatMode; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(const std::uint32_t (&expected)[32][16], const char* mode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t i = 0; i < 16; ++i) { + const std::string name = std::string(mode) + " " + Names[i]; + Expect(tid, out[i], expected[tid][i], name.c_str()); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, std::nullopt); + Check(ExpectedNoIeee, "no float mode"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + Check(ExpectedNoIeee, "IEEE=0"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false}); + Check(ExpectedIeee, "IEEE=1"); + std::puts("f32 arithmetic nan tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp new file mode 100644 index 000000000..37427d269 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp @@ -0,0 +1,211 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x06140b04u, 0x08160905u, 0x10180b04u, + 0x0e1a0b04u, 0xd54b000eu, 0x041a0b04u, 0x7e1e0306u, 0x561e0b04u, 0x1e200b04u, 0xd5540011u, 0x041a0b04u, + 0xd5570012u, 0x041a0b04u, 0x7e264904u, 0x7e284505u, 0x7e2a8104u, 0x7c020b04u, 0xd5010016u, 0x01a90280u, + 0x7e2e1b04u, 0x7e307f05u, 0xd5080119u, 0x42020b04u, 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, + 0xe0781020u, 0x80011203u, 0xe0781030u, 0x80011603u, 0xbf810000u, +}; + +constexpr std::uint32_t Rows[32][4] = { + {0x00000001u, 0x3f800000u, 0x00000000u, 0x00000000u}, + {0x80400000u, 0x3f800000u, 0x00800000u, 0x00000000u}, + {0x3f800000u, 0x00012345u, 0x80000001u, 0x00000000u}, + {0x007fffffu, 0x007fffffu, 0x3f800000u, 0x00000000u}, + {0x3f7fffffu, 0x00800000u, 0x00000000u, 0x00000000u}, + {0xbf7fffffu, 0x00800000u, 0x80000000u, 0x00000000u}, + {0x3f7fffffu, 0x00800001u, 0x00000001u, 0x00000000u}, + {0x3f000000u, 0x01000000u, 0x80000001u, 0x00000000u}, + {0x00800000u, 0x80800001u, 0x00800000u, 0x00000000u}, + {0x00800001u, 0x00800000u, 0x3f800000u, 0x00000000u}, + {0x00000000u, 0x80000001u, 0x00000001u, 0x00000000u}, + {0x80000000u, 0x00000001u, 0x807fffffu, 0x00000000u}, + {0x7f800000u, 0x00000001u, 0x3f800000u, 0x00000000u}, + {0x00000001u, 0xff800000u, 0x00000000u, 0x00000000u}, + {0x7fc00000u, 0x00000001u, 0x00000001u, 0x00000000u}, + {0x40000000u, 0x00400000u, 0x80800000u, 0x00000000u}, + {0x3f800000u, 0x3f800000u, 0x80000001u, 0x00000000u}, + {0xbf800000u, 0x00000001u, 0x3f000000u, 0x00000000u}, + {0x34000000u, 0x34000000u, 0x00000000u, 0x00000000u}, + {0x1f800000u, 0x1f800000u, 0x00000000u, 0x00000000u}, + {0x1fffffffu, 0x1fffffffu, 0x00800000u, 0x00000000u}, + {0x20000000u, 0x1f000000u, 0x00000000u, 0x00000000u}, + {0x20000000u, 0x1f7fffffu, 0x00000000u, 0x00000000u}, + {0x20000001u, 0x1f7fffffu, 0x00000000u, 0x00000000u}, + {0xa0000000u, 0x1f7fffffu, 0x80000000u, 0x00000000u}, + {0x3f800000u, 0xbf800000u, 0x00000001u, 0x00000000u}, + {0x00800000u, 0x3f000000u, 0x00000000u, 0x00000000u}, + {0x00800000u, 0x3f7fffffu, 0x00000001u, 0x00000000u}, + {0x41200000u, 0x00000003u, 0x3dcccccdu, 0x00000000u}, + {0xc0490fdbu, 0x80000002u, 0x40490fdbu, 0x00000000u}, + {0x80000001u, 0x80000001u, 0x80000001u, 0x00000000u}, + {0x00000001u, 0x00000001u, 0x00000001u, 0x00000000u} +}; +constexpr std::uint32_t Expected[32][16] = { + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x80000000u}, + {0x3f800000u, 0x3f800000u, 0x80000000u, 0x00000000u, 0x00800000u, 0x00800000u, 0x80000000u, 0x3f800000u, 0x00800000u, 0x80000000u, 0x3f800000u, 0x80000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x80000000u}, + {0x3f800000u, 0xbf800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x80000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u}, + {0x3f7fffffu, 0xbf7fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x3f7fffffu, 0x00800000u, 0x00000000u, 0x3f800000u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0xffffff83u, 0x80000000u}, + {0xbf7fffffu, 0x3f7fffffu, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0xbf7fffffu, 0x00800000u, 0x80000000u, 0xbf800000u, 0x3f800000u, 0xbf7fffffu, 0x00000001u, 0xffffffffu, 0xffffff83u, 0x80000000u}, + {0x3f7fffffu, 0xbf7fffffu, 0x00800000u, 0x00800000u, 0x00800000u, 0x00800000u, 0x00800001u, 0x3f7fffffu, 0x00800001u, 0x00000000u, 0x3f800000u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0xffffff83u, 0x80800000u}, + {0x3f000000u, 0xbf000000u, 0x00800000u, 0x00800000u, 0x00800000u, 0x00800000u, 0x01000000u, 0x3f000000u, 0x01000000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffff84u, 0x80800000u}, + {0x80000000u, 0x81000000u, 0x80000000u, 0x80000000u, 0x00800000u, 0x00800000u, 0x80800001u, 0x00800000u, 0x00800000u, 0x00000000u, 0x80000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffff83u, 0x00000000u}, + {0x01000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00800000u, 0x3f800000u, 0x00800001u, 0x00000000u, 0x3f800000u, 0x3f000001u, 0x00000000u, 0x00000000u, 0xffffff83u, 0x80000000u}, + {0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u}, + {0x7f800000u, 0xff800000u, 0xffc00000u, 0x00000000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x7fffffffu, 0x00000000u, 0xffc00000u}, + {0xff800000u, 0xff800000u, 0xffc00000u, 0x00000000u, 0xffc00000u, 0xffc00000u, 0xff800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xff800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u}, + {0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0x00000000u, 0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u, 0x00000000u, 0x7fc00000u, 0x00000000u, 0x7fffffffu, 0x00000000u, 0x7fc00000u}, + {0x40000000u, 0xc0000000u, 0x00000000u, 0x00000000u, 0x80800000u, 0x80800000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000002u, 0x00000000u, 0x80000000u}, + {0x40000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0xbf800000u}, + {0xbf800000u, 0x3f800000u, 0x80000000u, 0x00000000u, 0x3f000000u, 0x3f000000u, 0xbf800000u, 0x3f000000u, 0x00000000u, 0xbf800000u, 0x00000000u, 0xbf000000u, 0x00000001u, 0xffffffffu, 0x00000000u, 0x80000000u}, + {0x34800000u, 0x00000000u, 0x28800000u, 0x28800000u, 0x28800000u, 0x28800000u, 0x34000000u, 0x34000000u, 0x34000000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffeau, 0xa8800000u}, + {0x20000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f800000u, 0x1f800000u, 0x1f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffc1u, 0x80000000u}, + {0x207fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00ffffffu, 0x00ffffffu, 0x1fffffffu, 0x1fffffffu, 0x1fffffffu, 0x00000000u, 0x3f800000u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0xffffffc1u, 0x80000000u}, + {0x20200000u, 0x9fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f000000u, 0x20000000u, 0x1f000000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffc0u, 0x80000000u}, + {0x20400000u, 0x9f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f7fffffu, 0x20000000u, 0x1f7fffffu, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffc0u, 0x80000000u}, + {0x20400001u, 0x9f800002u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f7fffffu, 0x20000001u, 0x1f7fffffu, 0x00000000u, 0x3f800000u, 0x3f000001u, 0x00000000u, 0x00000000u, 0xffffffc0u, 0x80000000u}, + {0x9f800000u, 0x20400000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0xa0000000u, 0x1f7fffffu, 0x80000000u, 0xbf800000u, 0x3f800000u, 0xbf000000u, 0x00000001u, 0xffffffffu, 0xffffffc0u, 0x80000000u}, + {0x00000000u, 0xc0000000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0xbf800000u, 0x3f000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x3f800000u}, + {0x3f000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x3f000000u, 0x00800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x80000000u}, + {0x3f7fffffu, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x3f7fffffu, 0x00800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x80000000u}, + {0x41200000u, 0xc1200000u, 0x00000000u, 0x00000000u, 0x3dcccccdu, 0x3dcccccdu, 0x00000000u, 0x41200000u, 0x3dcccccdu, 0x41200000u, 0x00000000u, 0x3f200000u, 0x00000000u, 0x0000000au, 0x00000000u, 0x80000000u}, + {0xc0490fdbu, 0x40490fdbu, 0x00000000u, 0x00000000u, 0x40490fdbu, 0x40490fdbu, 0xc0490fdbu, 0x40490fdbu, 0x80000000u, 0xc0800000u, 0x80000000u, 0xbf490fdbu, 0x00000001u, 0xfffffffcu, 0x00000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u} +}; +constexpr const char* Names[16] = { + "v_add_f32 v10, v4, v5", + "v_sub_f32 v11, v5, v4", + "v_mul_f32 v12, v4, v5", + "v_mul_legacy_f32 v13, v4, v5", + "v_fma_f32 v14, v4, v5, v6", + "v_fmac_f32 v15, v4, v5", + "v_min_f32 v16, v4, v5", + "v_max3_f32 v17, v4, v5, v6", + "v_med3_f32 v18, v4, v5, v6", + "v_floor_f32 v19, v4", + "v_ceil_f32 v20, v5", + "v_frexp_mant_f32 v21, v4", + "v_cmp_lt_f32 v4, v5", + "v_cvt_flr_i32_f32 v23, v4", + "v_frexp_exp_i32_f32 v24, v5", + "v_mul_f32 v25, |v4|, -v5" +}; + +void Fill(std::uint32_t tid, std::uint32_t* words) { + std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words); +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) { + Require(actual == expected, std::string("f32 denormal flush: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); +} + +bool IsNan(std::uint32_t value) { + return (value & 0x7fffffffu) > 0x7f800000u; +} + +void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = floatMode; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(const char* mode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t i = 0; i < 16; ++i) { + const bool floatResult = i < 12u || i == 15u; + if (floatResult && IsNan(Expected[tid][i])) { + Require(IsNan(out[i]), std::string("f32 denormal flush: lane ") + std::to_string(tid) + " " + mode + " " + Names[i] + " is " + Hex(out[i]) + ", expected a NaN"); + continue; + } + Expect(tid, out[i], Expected[tid][i], (std::string(mode) + " " + Names[i]).c_str()); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, std::nullopt); + Check("no float mode"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + Check("IEEE=0 f32 denormals flushed"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0x00u, false, true, false}); + Check("IEEE=1 all denormals flushed"); + for (const std::uint32_t mode : {0xd0u, 0xe0u}) { + bool refused = false; + try { + Run(*device, ShaderRecompiler::ShaderFloatMode{mode, true, false, false}); + } catch (const std::runtime_error& error) { + refused = std::string(error.what()).find("f32 denormal mode") != std::string::npos; + } + Require(refused, "f32 denormal flush: FLOAT_MODE " + Hex(mode) + " was not refused"); + } + std::puts("f32 denormal flush tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F32MinMaxNan.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F32MinMaxNan.cpp index b9e7244d8..d427b01f0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F32MinMaxNan.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F32MinMaxNan.cpp @@ -289,8 +289,139 @@ constexpr std::uint32_t Expected[128][7] = { {0x00000000u, 0x3f800000u, 0x80000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}, {0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u} }; +constexpr std::uint32_t ExpectedIeee[128][7] = { + {0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x80000000u, 0x00000000u}, + {0x00000000u, 0x3f800000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}, + {0xc0400000u, 0x00000000u, 0xc0400000u, 0x3f800000u, 0x00000000u, 0xc0400000u, 0x00000000u}, + {0x00000000u, 0x7f800000u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x00000000u, 0x7f800000u}, + {0xff800000u, 0x00000000u, 0xff800000u, 0x00000000u, 0xff800000u, 0xff800000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0xc0400000u, 0x00000000u, 0xc0400000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u}, + {0xffc12345u, 0xffc12345u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xffc12345u, 0xffc12345u}, + {0x80000000u, 0x00000000u, 0x80000000u, 0x7f800000u, 0x00000000u, 0x80000000u, 0x00000000u}, + {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u}, + {0x80000000u, 0x3f800000u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x80000000u, 0x3f800000u}, + {0xc0400000u, 0x80000000u, 0xff800000u, 0x80000000u, 0xc0400000u, 0xc0400000u, 0x80000000u}, + {0x80000000u, 0x7f800000u, 0x80000000u, 0x7f800000u, 0x3f800000u, 0x80000000u, 0x7f800000u}, + {0xff800000u, 0x80000000u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xff800000u, 0x80000000u}, + {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u}, + {0x80000000u, 0x80000000u, 0xc0400000u, 0x80000000u, 0xc0400000u, 0x80000000u, 0x80000000u}, + {0x7fc00001u, 0x7fc00001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00001u, 0x7fc00001u}, + {0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u}, + {0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}, + {0x80000000u, 0x3f800000u, 0x80000000u, 0x7f800000u, 0x3f800000u, 0x80000000u, 0x3f800000u}, + {0x3f800000u, 0x3f800000u, 0x80000000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u}, + {0xc0400000u, 0x3f800000u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0xc0400000u, 0x3f800000u}, + {0x3f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x3f800000u, 0x3f800000u, 0x7f800000u}, + {0xff800000u, 0x3f800000u, 0xff800000u, 0x3f800000u, 0x3f800000u, 0xff800000u, 0x3f800000u}, + {0x3f800000u, 0x3f800000u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x3f800000u, 0x3f800000u}, + {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u}, + {0x7fc00001u, 0x7fc00001u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0x7fc00001u, 0x7fc00001u}, + {0xffc12345u, 0xffc12345u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc12345u, 0xffc12345u}, + {0xc0400000u, 0x00000000u, 0xc0400000u, 0x00000000u, 0x00000000u, 0xc0400000u, 0x00000000u}, + {0xc0400000u, 0x80000000u, 0xc0400000u, 0x80000000u, 0xc0400000u, 0xc0400000u, 0x80000000u}, + {0xc0400000u, 0x3f800000u, 0xc0400000u, 0x7f800000u, 0x3f800000u, 0xc0400000u, 0x3f800000u}, + {0xc0400000u, 0xc0400000u, 0xc0400000u, 0x80000000u, 0xc0400000u, 0xc0400000u, 0xc0400000u}, + {0xc0400000u, 0x7f800000u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0xc0400000u, 0x7f800000u}, + {0xff800000u, 0xc0400000u, 0xff800000u, 0xc0400000u, 0xff800000u, 0xff800000u, 0xc0400000u}, + {0xc0400000u, 0xc0400000u, 0xc0400000u, 0x3f800000u, 0xc0400000u, 0xc0400000u, 0xc0400000u}, + {0xc0400000u, 0xc0400000u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xc0400000u, 0xc0400000u}, + {0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u}, + {0xffc12345u, 0xffc12345u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xffc12345u, 0xffc12345u}, + {0x00000000u, 0x7f800000u, 0xc0400000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x7f800000u}, + {0x80000000u, 0x7f800000u, 0x80000000u, 0x7f800000u, 0x00000000u, 0x80000000u, 0x7f800000u}, + {0x3f800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, 0x3f800000u, 0x3f800000u, 0x7f800000u}, + {0xc0400000u, 0x7f800000u, 0xc0400000u, 0x7f800000u, 0x7f800000u, 0xc0400000u, 0x7f800000u}, + {0x7f800000u, 0x7f800000u, 0x80000000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u}, + {0xff800000u, 0x7f800000u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0xff800000u, 0x7f800000u}, + {0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u}, + {0x7f800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, 0x7f800000u}, + {0x7fc00001u, 0x7fc00001u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x7fc00001u, 0x7fc00001u}, + {0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u}, + {0xff800000u, 0x00000000u, 0xff800000u, 0x00000000u, 0xff800000u, 0xff800000u, 0x00000000u}, + {0xff800000u, 0x80000000u, 0xff800000u, 0x80000000u, 0xc0400000u, 0xff800000u, 0x80000000u}, + {0xff800000u, 0x3f800000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0xff800000u, 0x3f800000u}, + {0xff800000u, 0xc0400000u, 0xff800000u, 0xc0400000u, 0xff800000u, 0xff800000u, 0xc0400000u}, + {0xff800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u}, + {0xff800000u, 0xff800000u, 0xff800000u, 0x80000000u, 0xff800000u, 0xff800000u, 0xff800000u}, + {0xff800000u, 0xff800000u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0xff800000u, 0xff800000u}, + {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u}, + {0x7fc00001u, 0x7fc00001u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x7fc00001u, 0x7fc00001u}, + {0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u}, + {0x00000000u, 0x00000000u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x00000000u, 0x00000000u}, + {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u}, + {0x3f800000u, 0x3f800000u, 0xc0400000u, 0x3f800000u, 0xc0400000u, 0x3f800000u, 0x3f800000u}, + {0xc0400000u, 0xc0400000u, 0xc0400000u, 0x00000000u, 0xc0400000u, 0xc0400000u, 0xc0400000u}, + {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u}, + {0xff800000u, 0xff800000u, 0xff800000u, 0x7f800000u, 0xff800000u, 0xff800000u, 0xff800000u}, + {0x7fc12345u, 0x7fc12345u, 0x80000000u, 0x80000000u, 0x80000000u, 0x7fc12345u, 0x7fc12345u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc12345u, 0x7fc12345u}, + {0x7fc00001u, 0x7fc00001u, 0xff800000u, 0xff800000u, 0xff800000u, 0x7fc00001u, 0x7fc00001u}, + {0xffc12345u, 0xffc12345u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xffc12345u, 0xffc12345u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x80000000u, 0x80000000u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x80000000u, 0x80000000u}, + {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u}, + {0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xc0400000u}, + {0x7f800000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x7f800000u}, + {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u}, + {0xffd54321u, 0xffd54321u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xffd54321u, 0xffd54321u}, + {0xffd54321u, 0xffd54321u, 0x80000000u, 0x80000000u, 0x80000000u, 0xffd54321u, 0xffd54321u}, + {0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u}, + {0xffc12345u, 0xffc12345u, 0xff800000u, 0xff800000u, 0xff800000u, 0xffc12345u, 0xffc12345u}, + {0x7fc00001u, 0x7fc00001u, 0xff800000u, 0xff800000u, 0xff800000u, 0x7fc00001u, 0x7fc00001u}, + {0x7fc00001u, 0x7fc00001u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x7fc00001u, 0x7fc00001u}, + {0x7fc00001u, 0x7fc00001u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x7fc00001u, 0x7fc00001u}, + {0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u}, + {0x7fc00001u, 0x7fc00001u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0x7fc00001u, 0x7fc00001u}, + {0x7fc00001u, 0x7fc00001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00001u, 0x7fc00001u}, + {0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u}, + {0x7fc00001u, 0x7fc00001u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7fc00001u, 0x7fc00001u}, + {0x7fc00001u, 0x7fc00001u, 0x80000000u, 0x80000000u, 0x80000000u, 0x7fc00001u, 0x7fc00001u}, + {0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u}, + {0xffc12345u, 0xffc12345u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0xffc12345u, 0xffc12345u}, + {0xffc12345u, 0xffc12345u, 0xff800000u, 0xff800000u, 0xff800000u, 0xffc12345u, 0xffc12345u}, + {0xffc12345u, 0xffc12345u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xffc12345u, 0xffc12345u}, + {0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u}, + {0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u}, + {0xffc12345u, 0xffc12345u, 0xc0400000u, 0xc0400000u, 0xc0400000u, 0xffc12345u, 0xffc12345u}, + {0xffc12345u, 0xffc12345u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc12345u, 0xffc12345u}, + {0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u}, + {0xffc12345u, 0xffc12345u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xffc12345u, 0xffc12345u}, + {0xffc12345u, 0xffc12345u, 0x80000000u, 0x80000000u, 0x80000000u, 0xffc12345u, 0xffc12345u}, + {0x3f800000u, 0x40200000u, 0x00000000u, 0x40200000u, 0x3f800000u, 0x3f800000u, 0x40200000u}, + {0x00800000u, 0x3f800000u, 0x00800000u, 0x3f800000u, 0x3f800000u, 0x00800000u, 0x3f800000u}, + {0xc0400000u, 0xbf800000u, 0xc0400000u, 0x00000000u, 0xbf800000u, 0xc0400000u, 0xbf800000u}, + {0xc0400000u, 0xbf800000u, 0xc0400000u, 0x3f800000u, 0xbf800000u, 0xc0400000u, 0xbf800000u}, + {0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u, 0x7fc00000u}, + {0xffc00000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u, 0xffc00000u}, + {0x7fc00001u, 0x7fc00001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00001u, 0x7fc00001u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0xbf800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0xff800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x00000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0x80000000u, 0x3f800000u, 0x00000000u, 0x80000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0xbf800000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0x80000000u, 0x7f800000u, 0x00000000u, 0x80000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0xff800000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x80000000u, 0x00000000u}, + {0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}, + {0x00000000u, 0x3f800000u, 0x80000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}, + {0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u}, +}; constexpr std::uint32_t Chunks = 4; std::uint32_t Chunk = 0; +bool Ieee = false; constexpr const char* Names[7] = { "v_min_f32 v10, v4, v5", "v_max_f32 v11, v4, v5", @@ -307,7 +438,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -317,7 +448,7 @@ std::string Hex(std::uint32_t value) { } void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) { - Require(actual == expected, std::string("f32 min max nan: row ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); + Require(actual == expected, std::string(Ieee ? "f32 min max nan, IEEE mode: row " : "f32 min max nan: row ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); } void Run(AgcDriver::VulkanDevice& device) { @@ -338,6 +469,7 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + if (Ieee) request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); @@ -348,7 +480,7 @@ void Check() { const std::uint32_t* in = &Input[tid * Inputs]; const std::uint32_t* out = &Output[tid * Results]; for (std::uint32_t i = 0; i < 7; ++i) { - Expect(Chunk * Threads + tid, out[i], Expected[Chunk * Threads + tid][i], Names[i]); + Expect(Chunk * Threads + tid, out[i], (Ieee ? ExpectedIeee : Expected)[Chunk * Threads + tid][i], Names[i]); } } } @@ -359,9 +491,12 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - for (Chunk = 0; Chunk < Chunks; ++Chunk) { - Run(*device); - Check(); + for (const bool ieee : {false, true}) { + Ieee = ieee; + for (Chunk = 0; Chunk < Chunks; ++Chunk) { + Run(*device); + Check(); + } } std::puts("f32 min max nan tests passed"); return 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F32OutputModifier.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F32OutputModifier.cpp index fe668ec80..fb6a90d04 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F32OutputModifier.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F32OutputModifier.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -100,6 +101,74 @@ constexpr std::uint32_t Expected[32][16] = { {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc12345u, 0x7fc12345u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc12345u}, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7f800000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f800000u} }; +constexpr std::uint32_t ExpectedKept[32][16] = { + {0x80000000u, 0x80000000u, 0x80000000u, 0x3f800000u, 0x3f800000u, 0xbf800000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0xcf000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x80800000u, 0x80800000u, 0x80800000u, 0x3f800000u, 0x3f800000u, 0xbf800000u, 0x81000000u, 0x81000000u, 0x80800000u, 0x80800000u, 0xbf800000u, 0xceff0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00ffffffu, 0x00ffffffu, 0x00ffffffu, 0x3f800000u, 0x3f800000u, 0xbf800000u, 0x00000000u, 0x00000000u, 0x00ffffffu, 0x00ffffffu, 0x00000000u, 0x4b7fffffu, 0x00ffffffu, 0x00ffffffu, 0x00000000u, 0x817fffffu}, + {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f7fffffu, 0x7f7fffffu, 0xff7fffffu, 0x7f800000u, 0x7f800000u, 0x40000000u, 0x7f7fffffu, 0x7f7fffffu, 0x4eff0000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xff7fffffu}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x4eff8247u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc12345u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x3f800000u, 0x7fc12345u, 0x7fc12345u, 0x4eff8247u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc12345u}, + {0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0xff800001u, 0x7f800001u, 0x7f800001u, 0x40000000u, 0x7f800001u, 0x7f800001u, 0x4eff0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xff800001u}, + {0xff812345u, 0xff812345u, 0xff812345u, 0xff812345u, 0xff812345u, 0x7f812345u, 0xff812345u, 0xff812345u, 0xbf800000u, 0xff812345u, 0xff812345u, 0xcafdb976u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7f812345u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0xff800000u, 0xff800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xff800000u}, + {0xff000000u, 0xff000000u, 0xff000000u, 0xff000000u, 0xff000000u, 0x7f000000u, 0xff000000u, 0xff000000u, 0xff000000u, 0x3f800000u, 0x3f800000u, 0x4e7e0000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f000000u}, + {0xbf000000u, 0xbf000000u, 0xbf000000u, 0xbf000000u, 0xbf000000u, 0x3f000000u, 0x7f800000u, 0x7f800000u, 0xbf800000u, 0x7f800000u, 0x00000000u, 0x4e7c0000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f800000u}, + {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0x7f800000u, 0x7fc00000u, 0x7fc00000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xcb000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u}, + {0x7f000000u, 0x7f000000u, 0x7f000000u, 0xff000000u, 0xff000000u, 0x7f000000u, 0x7f000000u, 0x7f000000u, 0xff000000u, 0xbf800000u, 0xff000000u, 0xcb800000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0xff000000u}, + {0x01000000u, 0x01000000u, 0x01000000u, 0x3f800000u, 0x3f800000u, 0xbf800000u, 0x3f000000u, 0x3f000000u, 0x01000000u, 0x3f800000u, 0x3f800000u, 0x4e7e0000u, 0x01000000u, 0x01000000u, 0x3f800000u, 0x3f000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x00800000u, 0x80800000u, 0x00800000u, 0x00800000u, 0x00000000u, 0x00800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x00800000u}, + {0x80000000u, 0x80000000u, 0x80000000u, 0x80800000u, 0x80800000u, 0x00800000u, 0x80800000u, 0x80800000u, 0x80800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80800000u}, + {0x7effffffu, 0x7effffffu, 0x7effffffu, 0x7f7fffffu, 0x7f7fffffu, 0xff7fffffu, 0x7effffffu, 0x7effffffu, 0x3f000000u, 0x7f7fffffu, 0x7f7fffffu, 0x4eff0000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xfeffffffu}, + {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x4eff0000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xff800000u}, + {0xfe800000u, 0xfe800000u, 0xfe800000u, 0xff000000u, 0xff000000u, 0x7f000000u, 0xfe800000u, 0xfe800000u, 0xff000000u, 0x3f000000u, 0x00000000u, 0x4e7c0000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x7e800000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0xcf000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u}, + {0xff800000u, 0xff800000u, 0xff800000u, 0x7efffffeu, 0x7efffffeu, 0xfefffffeu, 0x7fc00000u, 0x7fc00000u, 0xff000000u, 0xff000000u, 0xff000000u, 0xcb800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x01000000u, 0x81000000u, 0x01000000u, 0x01000000u, 0x00000000u, 0x01000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x01000000u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0x7fc00000u, 0x7f7fffffu, 0x00000000u, 0x7fc00000u, 0x4eff8000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u}, + {0x81000000u, 0x81000000u, 0x81000000u, 0xbf800000u, 0xbf800000u, 0x3f800000u, 0x81000000u, 0x81000000u, 0xbf800000u, 0x00000000u, 0xbf800000u, 0xce810000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x81000000u, 0x81000000u, 0x01000000u, 0x00000000u, 0x00000000u, 0x80800000u, 0x00000000u, 0xbf800000u, 0xceff0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x7f800001u, 0x7f800001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7f800001u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x7fc12345u, 0x7fc12345u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc12345u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x7f800000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f800000u} +}; +constexpr std::uint32_t ExpectedIeee[32][16] = { + {0x80000000u, 0x80000000u, 0x80000000u, 0x3f800000u, 0x3f800000u, 0xbf800000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0xcf000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x80800000u, 0x80800000u, 0x80800000u, 0x3f800000u, 0x3f800000u, 0xbf800000u, 0x81000000u, 0x81000000u, 0x80800000u, 0x80800000u, 0xbf800000u, 0xceff0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00ffffffu, 0x00ffffffu, 0x00ffffffu, 0x3f800000u, 0x3f800000u, 0xbf800000u, 0x00000000u, 0x00000000u, 0x00ffffffu, 0x00ffffffu, 0x00000000u, 0x4b7fffffu, 0x00ffffffu, 0x00ffffffu, 0x00000000u, 0x817fffffu}, + {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f7fffffu, 0x7f7fffffu, 0xff7fffffu, 0x7f800000u, 0x7f800000u, 0x40000000u, 0x7f7fffffu, 0x7f7fffffu, 0x4eff0000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xff7fffffu}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0xffc00001u, 0x7fc12345u, 0x4eff8247u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc12345u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0xffc12345u, 0x7fc12345u, 0x7fc12345u, 0x3f800000u, 0x7fc12345u, 0x7fc12345u, 0x4eff8247u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc12345u}, + {0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0xffc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x7fc00001u, 0x4eff0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00001u}, + {0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0x7fc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xffc12345u, 0xcafdb976u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc12345u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0xff800000u, 0xff800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xff800000u}, + {0xff000000u, 0xff000000u, 0xff000000u, 0xff000000u, 0xff000000u, 0x7f000000u, 0xff000000u, 0xff000000u, 0xff000000u, 0x3f800000u, 0x3f800000u, 0x4e7e0000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f000000u}, + {0xbf000000u, 0xbf000000u, 0xbf000000u, 0xbf000000u, 0xbf000000u, 0x3f000000u, 0x7f800000u, 0x7f800000u, 0xbf800000u, 0x7f800000u, 0x00000000u, 0x4e7c0000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f800000u}, + {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0x7f800000u, 0x7fc00000u, 0x7fc00000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xcb000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u}, + {0x7f000000u, 0x7f000000u, 0x7f000000u, 0xff000000u, 0xff000000u, 0x7f000000u, 0x7f000000u, 0x7f000000u, 0xff000000u, 0xbf800000u, 0xff000000u, 0xcb800000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0xff000000u}, + {0x01000000u, 0x01000000u, 0x01000000u, 0x3f800000u, 0x3f800000u, 0xbf800000u, 0x3f000000u, 0x3f000000u, 0x01000000u, 0x3f800000u, 0x3f800000u, 0x4e7e0000u, 0x01000000u, 0x01000000u, 0x3f800000u, 0x3f000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x00800000u, 0x80800000u, 0x00800000u, 0x00800000u, 0x00000000u, 0x00800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x00800000u}, + {0x80000000u, 0x80000000u, 0x80000000u, 0x80800000u, 0x80800000u, 0x00800000u, 0x80800000u, 0x80800000u, 0x80800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80800000u}, + {0x7effffffu, 0x7effffffu, 0x7effffffu, 0x7f7fffffu, 0x7f7fffffu, 0xff7fffffu, 0x7effffffu, 0x7effffffu, 0x3f000000u, 0x7f7fffffu, 0x7f7fffffu, 0x4eff0000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xfeffffffu}, + {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x4eff0000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xff800000u}, + {0xfe800000u, 0xfe800000u, 0xfe800000u, 0xff000000u, 0xff000000u, 0x7f000000u, 0xfe800000u, 0xfe800000u, 0xff000000u, 0x3f000000u, 0x00000000u, 0x4e7c0000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x7e800000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0xcf000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u}, + {0xff800000u, 0xff800000u, 0xff800000u, 0x7efffffeu, 0x7efffffeu, 0xfefffffeu, 0x7fc00000u, 0x7fc00000u, 0xff000000u, 0xff000000u, 0xff000000u, 0xcb800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x01000000u, 0x81000000u, 0x01000000u, 0x01000000u, 0x00000000u, 0x01000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x01000000u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0x7fc00000u, 0x7f7fffffu, 0x00000000u, 0x7fc00000u, 0x4eff8000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u}, + {0x81000000u, 0x81000000u, 0x81000000u, 0xbf800000u, 0xbf800000u, 0x3f800000u, 0x81000000u, 0x81000000u, 0xbf800000u, 0x00000000u, 0xbf800000u, 0xce810000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x81000000u, 0x81000000u, 0x01000000u, 0x00000000u, 0x00000000u, 0x80800000u, 0x00000000u, 0xbf800000u, 0xceff0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x7fc00001u, 0x7fc00001u, 0x00000000u, 0x7fc00001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00001u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x7fc12345u, 0x7fc12345u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc12345u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x7f800000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x7f800000u} +}; constexpr std::uint32_t Checked[32] = {0xffffu, 0xffffu, 0xffffu, 0x7f3fu, 0x7800u, 0x7b00u, 0x7b00u, 0x7b00u, 0xffffu, 0xffffu, 0xffffu, 0x7f3fu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0xffffu, 0x7f3fu, 0xffffu, 0xffffu, 0x7f3fu, 0xffffu, 0x7b00u, 0xffffu, 0x7f3fu, 0xffffu, 0xffffu, 0x7f3fu, 0x7f3fu, 0x7f3fu, 0x7f3fu, 0xffffu}; constexpr std::uint32_t AnyNan[32] = {0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x84ffu, 0x84ffu, 0x84ffu, 0x84ffu, 0x0000u, 0x0000u, 0x0000u, 0x80c0u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x80c0u, 0x0000u, 0x84ffu, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x80c0u, 0x80c0u, 0x80c0u, 0x80c0u, 0x0000u}; constexpr const char* Names[16] = { @@ -127,7 +196,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -140,7 +209,7 @@ void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, con Require(actual == expected, std::string("f32 output modifier: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); } -void Run(AgcDriver::VulkanDevice& device) { +void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); Output.fill(0xdeadbeefu); std::vector userData(8, 0u); @@ -158,17 +227,18 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = floatMode; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); } -void Check() { +void Check(const std::uint32_t (&expected)[32][16], const char* mode) { for (std::uint32_t tid = 0; tid < Threads; ++tid) { const std::uint32_t* in = &Input[tid * Inputs]; const std::uint32_t* out = &Output[tid * Results]; for (std::uint32_t i = 0; i < 16; ++i) { - if ((Checked[tid] >> i) & 1u) Expect(tid, out[i], Expected[tid][i], Names[i]); + if ((Checked[tid] >> i) & 1u) Expect(tid, out[i], expected[tid][i], (std::string(mode) + " " + Names[i]).c_str()); if ((AnyNan[tid] >> i) & 1u) Require((out[i] & 0x7fffffffu) > 0x7f800000u, std::string("f32 output modifier: lane ") + std::to_string(tid) + " " + Names[i] + " is " + Hex(out[i]) + ", expected a NaN"); } } @@ -180,8 +250,16 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - Run(*device); - Check(); + Run(*device, std::nullopt); + Check(Expected, "no float mode"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + Check(Expected, "IEEE=0 f32 denormals flushed"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xf0u, true, false, false}); + Check(ExpectedKept, "IEEE=0 f32 denormals kept"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false}); + Check(ExpectedIeee, "IEEE=1 f32 denormals flushed"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}); + Check(ExpectedIeee, "IEEE=1 f32 denormals kept"); std::puts("f32 output modifier tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F32ResultClamp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F32ResultClamp.cpp index 9833bcc22..5fb148587 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F32ResultClamp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F32ResultClamp.cpp @@ -126,7 +126,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64Arithmetic.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64Arithmetic.cpp index 8a3e6560c..c8e592d6f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64Arithmetic.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64Arithmetic.cpp @@ -106,7 +106,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -156,6 +156,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityFloat64)) { + std::puts("skipped, the device has no shaderFloat64"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("f64 arithmetic tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64Conversions.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64Conversions.cpp index bf52b9fa7..ab7877a62 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64Conversions.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64Conversions.cpp @@ -106,7 +106,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -156,6 +156,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityFloat64)) { + std::puts("skipped, the device has no shaderFloat64"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("f64 conversions tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64Division.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64Division.cpp index 634a3ed4e..84d1c6f68 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64Division.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64Division.cpp @@ -107,7 +107,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -161,6 +161,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityFloat64)) { + std::puts("skipped, the device has no shaderFloat64"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("f64 division tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64DivisionModifiers.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64DivisionModifiers.cpp index 645f47a10..16dcd8747 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64DivisionModifiers.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64DivisionModifiers.cpp @@ -107,7 +107,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -161,6 +161,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityFloat64)) { + std::puts("skipped, the device has no shaderFloat64"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("f64 division modifiers tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64NanConversions.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64NanConversions.cpp index f04b8b1b9..39d5cc579 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64NanConversions.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64NanConversions.cpp @@ -101,7 +101,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::copy(std::begin(Rows[t std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -153,6 +153,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityFloat64)) { + std::puts("skipped, the device has no shaderFloat64"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("f64 nan conversions tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64Rounding.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64Rounding.cpp index 4f717f846..8e3bff765 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64Rounding.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64Rounding.cpp @@ -106,7 +106,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -156,6 +156,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityFloat64)) { + std::puts("skipped, the device has no shaderFloat64"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("f64 rounding tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64Transcendental.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64Transcendental.cpp index f4e6cc55e..c3d8c21b3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64Transcendental.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64Transcendental.cpp @@ -121,7 +121,7 @@ void ExpectNear(std::uint32_t tid, const std::uint32_t* actual, const std::uint3 std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -179,6 +179,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityFloat64)) { + std::puts("skipped, the device has no shaderFloat64"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("f64 transcendental tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatCacheBits.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatCacheBits.cpp index 717f9f028..742faa65c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatCacheBits.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatCacheBits.cpp @@ -104,7 +104,7 @@ private: std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatD16Loads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatD16Loads.cpp index 3dfd1167e..abc38e965 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatD16Loads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatD16Loads.cpp @@ -505,7 +505,7 @@ private: std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoads.cpp index 6e3677e26..e96afcd62 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoads.cpp @@ -117,7 +117,7 @@ private: std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX2X3.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX2X3.cpp index 2642ebc78..c85f891fd 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX2X3.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX2X3.cpp @@ -116,7 +116,7 @@ private: std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX4.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX4.cpp index 008004a55..330114afb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX4.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX4.cpp @@ -116,7 +116,7 @@ private: std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16Dot.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16Dot.cpp index f7d519ee1..961afeb7c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Float16Dot.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16Dot.cpp @@ -109,7 +109,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -173,6 +173,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityFloat64)) { + std::puts("skipped, v_dot2_f32_f16 is computed in f64 and the device has no shaderFloat64"); + return VulkanTestSkipped; + } Run(*device); Check(); CheckRefused(*device, 0xcc13600au, "op_sel on the accumulator"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16Misc.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16Misc.cpp index bde645947..ddddf6d52 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Float16Misc.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16Misc.cpp @@ -303,7 +303,7 @@ constexpr std::array Names{ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16Modifiers.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16Modifiers.cpp index c53d40431..ad9657aed 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Float16Modifiers.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16Modifiers.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -127,7 +128,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -150,7 +151,7 @@ void ExpectHalf(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, Expect(tid, actual, expected, name); } -void Run(AgcDriver::VulkanDevice& device) { +void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); Output.fill(0xdeadbeefu); std::vector userData(8, 0u); @@ -168,6 +169,7 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = floatMode; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); @@ -187,8 +189,17 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - Run(*device); + Run(*device, std::nullopt); Check(); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}); + Check(); + bool refused = false; + try { + Run(*device, ShaderRecompiler::ShaderFloatMode{0x00u, true, false, false}); + } catch (const std::runtime_error& error) { + refused = std::string(error.what()).find("output modifier on an f16") != std::string::npos; + } + Require(refused, "float16 modifiers: an output modifier with f16 output denormals flushed was not refused"); std::puts("float16 modifiers tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16ResultModifiers.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16ResultModifiers.cpp index 45b992a20..ad48c4655 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Float16ResultModifiers.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16ResultModifiers.cpp @@ -114,7 +114,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16Rounding.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16Rounding.cpp index b740bcd5f..ec1c19913 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Float16Rounding.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16Rounding.cpp @@ -296,7 +296,7 @@ constexpr std::array Names{"v_cvt_f16_f32", "v_add_f16", "v_mul_ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16TernaryClamp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16TernaryClamp.cpp index 655d71fad..8872332f2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Float16TernaryClamp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16TernaryClamp.cpp @@ -118,7 +118,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FmaLegacy.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FmaLegacy.cpp index e9c867787..53af35045 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FmaLegacy.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FmaLegacy.cpp @@ -59,7 +59,7 @@ constexpr std::array Rows{{ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FmaRounding.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FmaRounding.cpp index f8be34b82..3699e8e0f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FmaRounding.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FmaRounding.cpp @@ -21,14 +21,15 @@ constexpr std::uint32_t Results = 16; alignas(256) std::array Input{}; alignas(256) std::array Output{}; -alignas(256) constexpr std::array FmaCode{ +alignas(256) constexpr std::array FmaCode{ 0x34020083, 0x34060084, 0xe0302000, 0x80000401, 0xe0302004, 0x80000501, 0xe0302008, 0x80000601, 0xe030200c, 0x80000701, 0xe0302010, 0x80000801, 0xbf8c3f70, 0xd54b000a, 0x041a0b04, 0x7e160306, - 0x56160b04, 0x5a180f04, 0x3f8ccccd, 0x581a1104, 0x3f8ccccd, 0xd541000e, 0x041a0b04, 0x7e1e0306, - 0x3e1e0b04, 0x42200f04, 0x3f8ccccd, 0x40221104, 0x3f8ccccd, 0x10240b04, 0x06240d12, 0x7e260306, - 0xd51f0013, 0x00020b04, 0xe0702000, 0x80010a03, 0xe0702004, 0x80010b03, 0xe0702008, 0x80010c03, - 0xe070200c, 0x80010d03, 0xe0702010, 0x80010e03, 0xe0702014, 0x80010f03, 0xe0702018, 0x80011003, - 0xe070201c, 0x80011103, 0xe0702020, 0x80011203, 0xe0702024, 0x80011303, 0xbf810000, + 0x56160b04, 0x5a180f04, 0x3f8ccccd, 0x581a1104, 0x3f8ccccd, 0xcc200014, 0x041a0b04, 0xd541000e, + 0x041a0b04, 0x7e1e0306, 0x3e1e0b04, 0x42200f04, 0x3f8ccccd, 0x40221104, 0x3f8ccccd, 0x10240b04, + 0x06240d12, 0x7e260306, 0xd51f0013, 0x00020b04, 0xe0702000, 0x80010a03, 0xe0702004, 0x80010b03, + 0xe0702008, 0x80010c03, 0xe070200c, 0x80010d03, 0xe0702010, 0x80011403, 0xe0702014, 0x80010e03, + 0xe0702018, 0x80010f03, 0xe070201c, 0x80011003, 0xe0702020, 0x80011103, 0xe0702024, 0x80011203, + 0xe0702028, 0x80011303, 0xbf810000, }; struct Row { @@ -105,7 +106,7 @@ constexpr std::array Rows{{ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { @@ -142,8 +143,8 @@ std::string Hex(std::uint32_t value) { } void Check() { - constexpr std::array names{ - "v_fma_f32", "v_fmac_f32", "v_fmaak_f32", "v_fmamk_f32", "v_mad_f32", "v_mac_f32", "v_madak_f32", "v_madmk_f32", "v_mul_f32 + v_add_f32", "v_mac_f32_e64", + constexpr std::array names{ + "v_fma_f32", "v_fmac_f32", "v_fmaak_f32", "v_fmamk_f32", "v_fma_mix_f32", "v_mad_f32", "v_mac_f32", "v_madak_f32", "v_madmk_f32", "v_mul_f32 + v_add_f32", "v_mac_f32_e64", }; bool hostFuses = false; for (std::uint32_t tid = 0; tid < Threads; ++tid) { @@ -154,10 +155,10 @@ void Check() { } for (std::uint32_t tid = 0; tid < Threads; ++tid) { const auto& row = Rows[tid]; - const std::array expected{ - row.fused, row.fused, row.fusedAk, row.fusedMk, row.separate, row.separate, row.separateAk, row.separateMk, row.separate, row.separate, + const std::array expected{ + row.fused, row.fused, row.fusedAk, row.fusedMk, row.fused, row.separate, row.separate, row.separateAk, row.separateMk, row.separate, row.separate, }; - for (std::uint32_t j = hostFuses ? 0u : 4u; j < expected.size(); ++j) { + for (std::uint32_t j = hostFuses ? 0u : 5u; j < expected.size(); ++j) { const auto actual = Output[tid * Results + j]; Require(actual == expected[j], std::string("fma rounding: lane ") + std::to_string(tid) + " " + names[j] + " is " + Hex(actual) + ", expected " + Hex(expected[j])); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16.cpp index 54a17ee50..635afe2fc 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16.cpp @@ -24,7 +24,7 @@ alignas(256) std::array Input{}; alignas(256) std::array Output{}; alignas(256) constexpr std::array Code{ - 0x34020084, 0x34060086, 0xbe900300, 0xbe910301, 0xbe920302, 0xbe9303ff, 0x01001244, 0x7e1402ff, + 0x34020084, 0x34060086, 0xbe900300, 0xbe910301, 0xbe920302, 0xbe9303ff, 0x31001244, 0x7e1402ff, 0xdead0000, 0x7e1602ff, 0xdead0000, 0x7e1802ff, 0xdead0000, 0x7e1a02ff, 0xdead0000, 0x7e1c02ff, 0xdead0000, 0x7e1e02ff, 0xdead0000, 0x7e2002ff, 0xdead0000, 0x7e2202ff, 0xdead0000, 0x7e2402ff, 0xdead0000, 0x7e2602ff, 0xdead0000, 0x7e2802ff, 0xbeefbeef, 0x7e2a02ff, 0xbeefbeef, 0x7e2c02ff, @@ -115,7 +115,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16Store.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16Store.cpp index 33dc26b02..126eb9bc2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16Store.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FormatD16Store.cpp @@ -112,7 +112,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStore.cpp index 8c8491eab..c608436a3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStore.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FormattedStore.cpp @@ -112,7 +112,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GdsAppendConsume.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GdsAppendConsume.cpp index 73faa7aad..0ee7d0c78 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GdsAppendConsume.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GdsAppendConsume.cpp @@ -58,7 +58,7 @@ constexpr std::array Workgroups{{{32, 32, 32}, {32, 32, 20}, {64, std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } std::vector Packet(std::uint32_t opcode, std::initializer_list payload) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicCsub.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicCsub.cpp index 7c84e3e8c..02ac733f2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicCsub.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicCsub.cpp @@ -137,7 +137,7 @@ std::vector Expected() { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics.cpp index 3409e7768..e36953f05 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics.cpp @@ -315,7 +315,7 @@ void Put(std::vector& image, std::uint32_t offset, std::uint64_t v std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint64_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics64.cpp index 541e26eb0..859f39c58 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics64.cpp @@ -279,7 +279,7 @@ void Put(std::vector& image, std::uint32_t offset, std::uint64_t v std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint64_t value) { @@ -389,6 +389,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityInt64Atomics)) { + std::puts("skipped, the device has no shaderBufferInt64Atomics"); + return VulkanTestSkipped; + } GuestBlock writable(true); RunAtomics(*device, writable, 32); std::puts("global atomics 64 tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsFloat.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsFloat.cpp index cf7b06501..800fb3947 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsFloat.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsFloat.cpp @@ -608,7 +608,7 @@ void Put(std::vector& image, std::uint32_t offset, std::uint64_t v std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint64_t value) { @@ -736,6 +736,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityInt64Atomics)) { + std::puts("skipped, the device has no shaderBufferInt64Atomics"); + return VulkanTestSkipped; + } GuestBlock writable(true); GuestBlock readOnly(false); RunAtomics(*device, writable, 32); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsLanes.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsLanes.cpp index 7629fb5af..d0bee4f75 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsLanes.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsLanes.cpp @@ -90,7 +90,7 @@ void Put(std::vector& image, std::uint32_t offset, std::uint64_t v std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint64_t value) { @@ -237,6 +237,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityInt64Atomics)) { + std::puts("skipped, the device has no shaderBufferInt64Atomics"); + return VulkanTestSkipped; + } GuestBlock writable(true); GuestBlock readOnly(false); RunAtomics(*device, writable, 32); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadAddtid.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadAddtid.cpp index c2a97f73e..e7875353c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadAddtid.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadAddtid.cpp @@ -49,7 +49,7 @@ constexpr std::array Workgroups{{{32, 32}, {128, 32}, {64, 64}, {1 std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoads.cpp index ad91fd604..26938a5d3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoads.cpp @@ -117,7 +117,7 @@ private: std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX2X3.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX2X3.cpp index 49ee77d74..878bf6426 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX2X3.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX2X3.cpp @@ -116,7 +116,7 @@ private: std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX4.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX4.cpp index d0bb6392d..6dc40863b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX4.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX4.cpp @@ -116,7 +116,7 @@ private: std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GuardedNullPointer.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GuardedNullPointer.cpp new file mode 100644 index 000000000..2f71f42a9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GuardedNullPointer.cpp @@ -0,0 +1,244 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#else +#include +#include +#endif + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Lanes = 64; +constexpr std::uint32_t Default = 0x11111111u; +constexpr std::uint32_t Payload = 0x3f800000u; + +alignas(256) std::array Output{}; +alignas(256) std::array Root{}; +alignas(256) std::array Data{Payload, 0u, 0u, 0u}; +alignas(256) std::array Indirect{}; + +alignas(256) constexpr std::array BranchCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xbf068008u, 0xbf850003u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, + 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +alignas(256) constexpr std::array ExecCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0x7da80008u, 0xbf880004u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, + 0x7e02020cu, 0xbefe04c1u, 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +alignas(256) constexpr std::array EmptyExecCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0x7da80008u, 0xbf800000u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, + 0x7e02020cu, 0xbefe04c1u, 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +alignas(256) constexpr std::array ChainCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xf4040385u, 0xfa000000u, 0xbf8cc07fu, 0xbf068008u, 0xbf850003u, 0xf4000307u, 0xfa000000u, + 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +class StderrCapture { +public: + StderrCapture() { + std::fflush(stderr); + file = std::tmpfile(); + Require(file != nullptr, "cannot open a temporary file for stderr"); +#ifdef _WIN32 + saved = _dup(_fileno(stderr)); + _dup2(_fileno(file), _fileno(stderr)); +#else + saved = dup(fileno(stderr)); + dup2(fileno(file), fileno(stderr)); +#endif + } + std::string Finish() { + std::fflush(stderr); +#ifdef _WIN32 + _dup2(saved, _fileno(stderr)); + _close(saved); +#else + dup2(saved, fileno(stderr)); + close(saved); +#endif + std::rewind(file); + std::string text; + char buffer[4096]; + for (std::size_t read = 0; (read = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, read); + std::fclose(file); + std::fwrite(text.data(), 1, text.size(), stderr); + return text; + } + +private: + std::FILE* file = nullptr; + int saved = -1; +}; + +struct Outcome { + std::vector words; + std::string log; + std::uint64_t variantId = 0; + std::uint64_t pipelineVariantId = 0; + std::uint32_t poisoned = 0; + bool cacheable = false; +}; + +template +Outcome Run(AgcDriver::VulkanDevice& device, const std::array& code, std::uint64_t pointer, std::uint32_t guard) { + const auto output = static_cast(reinterpret_cast(Output.data())); + Root = {}; + Root[0] = static_cast(output); + Root[1] = static_cast((output >> 32u) & 0xffffu); + Root[2] = static_cast(sizeof(Output)); + Root[3] = 0x30027facu; + Root[4] = guard; + Root[6] = static_cast(pointer); + Root[7] = static_cast(pointer >> 32u); + std::fill(Output.begin(), Output.end(), 0xdeadbeefu); + const auto root = static_cast(reinterpret_cast(Root.data())); + const std::array userData{static_cast(root), static_cast(root >> 32u)}; + const std::span span(code); + const ShaderRecompiler::ShaderComputeStageInfo compute{{Lanes, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(span.data()), span, 0, {}}, + {64, 0, userData, compute, std::nullopt, std::nullopt, {}}, + device.Target(), + {0, 0, 0, 128} + }; + AgcDriver::ShaderMemory memory({}); + const auto capture = memory.Capture(request); + const auto regions = memory.Regions(); + request.context.memory = regions; + const auto result = ShaderRecompiler::Recompile(request, *capture); + Outcome outcome; + outcome.variantId = result->variantId; + outcome.pipelineVariantId = result->PipelineVariantId(); + outcome.poisoned = result->poisonedSrtReads; + outcome.cacheable = AgcDriver::DriverDetail::CacheableResult(*result); + StderrCapture log; + try { + device.Dispatch(*result, 1, 1, 1, {}, reinterpret_cast(span.data())); + device.WaitIdle(); + } catch (...) { + static_cast(log.Finish()); + throw; + } + outcome.log = log.Finish(); + outcome.words.assign(Output.begin(), Output.end()); + return outcome; +} + +bool Faulted(const Outcome& outcome) { + return outcome.log.find("BDA access failed") != std::string::npos; +} + +std::string Fault(std::uint64_t address, std::uint32_t pc) { + char text[96]; + std::snprintf(text, sizeof(text), "address=0x%llx instruction=0x%x bytes=4", static_cast(address), pc); + return text; +} + +void RequireWords(const Outcome& outcome, std::uint32_t lanes, std::uint32_t inside, const std::string& what) { + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + const auto expected = lane < lanes ? inside : Default; + Require(outcome.words[lane] == expected, what + ": lane " + std::to_string(lane) + " wrote " + std::to_string(outcome.words[lane]) + ", expected " + std::to_string(expected)); + } +} + +void RunTests(AgcDriver::VulkanDevice& device) { + const auto valid = static_cast(reinterpret_cast(Data.data())); + void* reserved = nullptr; +#ifdef _WIN32 + reserved = VirtualAlloc(nullptr, 65536, MEM_RESERVE, PAGE_NOACCESS); +#else + reserved = mmap(nullptr, 65536, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) reserved = nullptr; +#endif + Require(reserved != nullptr, "cannot reserve an inaccessible range"); + const auto unmapped = static_cast(reinterpret_cast(reserved)); + + const auto taken = Run(device, BranchCode, valid, 1u); + Require(!Faulted(taken) && taken.poisoned == 0u && taken.cacheable, "guarded pointer: a mapped pointer faulted, was poisoned or left the caches:\n" + taken.log); + RequireWords(taken, Lanes, Payload, "guarded pointer: a mapped pointer under a taken branch"); + const auto skippedNull = Run(device, BranchCode, 0u, 0u); + Require(!Faulted(skippedNull) && skippedNull.poisoned == 1u && !skippedNull.cacheable, "guarded pointer: a null pointer behind a branch that skips its read faulted or stayed cacheable:\n" + skippedNull.log); + RequireWords(skippedNull, 0u, 0u, "guarded pointer: a skipped read through a null pointer"); + const auto takenNull = Run(device, BranchCode, 0u, 1u); + Require(takenNull.pipelineVariantId == skippedNull.pipelineVariantId && takenNull.pipelineVariantId == taken.pipelineVariantId && takenNull.variantId == taken.variantId, "guarded pointer: the poison or the branch condition changed the pipeline variant"); + Require(Faulted(takenNull) && takenNull.log.find(Fault(0u, 0x2cu)) != std::string::npos && takenNull.log.find("reason=1") != std::string::npos, "guarded pointer: a read through a null pointer that runs did not fault at its pc and address:\n" + takenNull.log); + Require(std::all_of(takenNull.words.begin(), takenNull.words.end(), [](std::uint32_t word) { return word == 0u || word == 0xdeadbeefu; }), "guarded pointer: a faulting read did not return zero"); + const auto takenUnmapped = Run(device, BranchCode, unmapped, 1u); + Require(Faulted(takenUnmapped) && takenUnmapped.log.find(Fault(unmapped, 0x2cu)) != std::string::npos, "guarded pointer: a read through an unmapped pointer that runs did not fault at its address:\n" + takenUnmapped.log); + Require(takenUnmapped.pipelineVariantId == taken.pipelineVariantId, "guarded pointer: an unmapped pointer changed the pipeline variant"); + + const auto someLanes = Run(device, ExecCode, valid, 5u); + Require(!Faulted(someLanes), "guarded pointer: a mapped pointer under EXEC faulted:\n" + someLanes.log); + RequireWords(someLanes, 5u, Payload, "guarded pointer: a mapped pointer read under EXEC"); + const auto noLanes = Run(device, ExecCode, 0u, 0u); + Require(!Faulted(noLanes) && noLanes.poisoned == 1u, "guarded pointer: a null pointer read skipped by s_cbranch_execz faulted:\n" + noLanes.log); + RequireWords(noLanes, 0u, 0u, "guarded pointer: a read skipped by s_cbranch_execz"); + const auto oneLane = Run(device, ExecCode, 0u, 1u); + Require(Faulted(oneLane) && oneLane.log.find(Fault(0u, 0x34u)) != std::string::npos, "guarded pointer: a null pointer read with one EXEC lane did not fault:\n" + oneLane.log); + const auto emptyMapped = Run(device, EmptyExecCode, valid, 0u); + Require(!Faulted(emptyMapped), "guarded pointer: a mapped pointer read with an empty EXEC faulted:\n" + emptyMapped.log); + RequireWords(emptyMapped, 0u, 0u, "guarded pointer: a mapped pointer read with an empty EXEC"); + const auto emptyNull = Run(device, EmptyExecCode, 0u, 0u); + Require(Faulted(emptyNull) && emptyNull.log.find(Fault(0u, 0x34u)) != std::string::npos, "guarded pointer: a null pointer read that runs with an empty EXEC did not fault:\n" + emptyNull.log); + Require(noLanes.pipelineVariantId == someLanes.pipelineVariantId && oneLane.pipelineVariantId == someLanes.pipelineVariantId && emptyNull.pipelineVariantId == emptyMapped.pipelineVariantId, "guarded pointer: the poison changed an EXEC program's pipeline variant"); + + const auto chainSkipped = Run(device, ChainCode, 0u, 0u); + Require(chainSkipped.poisoned == 3u && Faulted(chainSkipped) && chainSkipped.log.find(Fault(0u, 0x24u)) != std::string::npos, "guarded pointer: a pointer read through a null pointer, used only by a skipped read, did not fault at its own pc:\n" + chainSkipped.log); + const auto chainTaken = Run(device, ChainCode, 0u, 1u); + Require(Faulted(chainTaken) && chainTaken.log.find(Fault(0u, 0x24u)) != std::string::npos, "guarded pointer: a read through a pointer read through a null pointer did not fault at the first read:\n" + chainTaken.log); + Indirect[0] = valid; + const auto chainMapped = Run(device, ChainCode, static_cast(reinterpret_cast(Indirect.data())), 1u); + Require(!Faulted(chainMapped) && chainMapped.poisoned == 0u && chainMapped.cacheable && chainMapped.pipelineVariantId == chainTaken.pipelineVariantId && chainSkipped.pipelineVariantId == chainTaken.pipelineVariantId, "guarded pointer: a chain's poison changed its pipeline variant, or a mapped chain faulted:\n" + chainMapped.log); + RequireWords(chainMapped, Lanes, Payload, "guarded pointer: a read through a mapped pointer chain"); + +#ifdef _WIN32 + VirtualFree(reserved, 0, MEM_RELEASE); +#else + munmap(reserved, 65536); +#endif +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + RunTests(*device); + std::puts("guarded null pointer tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/HostImportTeardown.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/HostImportTeardown.cpp new file mode 100644 index 000000000..72095307c --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/HostImportTeardown.cpp @@ -0,0 +1,110 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; + +constexpr std::size_t BlockBytes = 65536; +constexpr std::size_t FillBytes = 4096; + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); +#endif + Require(block != nullptr, "host import teardown: cannot allocate the guest block"); + std::memset(block, 0, BlockBytes); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint8_t* Data() { return block; } + + std::uint64_t Address() const { return static_cast(reinterpret_cast(block)); } + +private: + std::uint8_t* block = nullptr; +}; + +bool Imported(VkDevice device, std::uint64_t address) { + AgcDriver::Graphics::Context probe{}; + probe.device = device; + probe.hostImportAlignment = 1; + return AgcDriver::Graphics::HostImportCovers(probe, address, BlockBytes); +} + +enum class Outcome : std::uint8_t { NoDevice, NoImport, Released }; + +Outcome FillAndDestroy(GuestBlock& guest, const std::array& pattern, const std::string& which) { + auto device = OpenVulkanTestDevice(); + if (!device) return Outcome::NoDevice; + bool filled = false; + { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + filled = device->FillBuffer(guest.Address(), FillBytes, pattern); + device->WaitIdle(); + } + if (!filled) return Outcome::NoImport; + for (std::size_t offset = 0; offset < FillBytes; offset += sizeof(pattern)) { + Require(std::memcmp(guest.Data() + offset, pattern.data(), sizeof(pattern)) == 0, "host import teardown: the " + which + "'s fill did not reach guest memory at offset " + std::to_string(offset)); + } + const std::array untouched{}; + Require(std::memcmp(guest.Data() + FillBytes, untouched.data(), untouched.size()) == 0, "host import teardown: the " + which + "'s fill wrote past its range"); + const auto handle = device->Device(); + Require(Imported(handle, guest.Address()), "host import teardown: the " + which + "'s fill left no import of the guest block"); + device.reset(); + Require(!Imported(handle, guest.Address()), "host import teardown: the guest block's import outlived the " + which); + return Outcome::Released; +} + +} + +int main() { + try { + GuestBlock guest; + const auto first = FillAndDestroy(guest, {0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}, "first device"); + if (first == Outcome::NoDevice) return VulkanTestSkipped; + if (first == Outcome::NoImport) { + std::printf("skipped, the device does not import guest memory\n"); + return VulkanTestSkipped; + } + Require(FillAndDestroy(guest, {0x55555555u, 0x66666666u, 0x77777777u, 0x88888888u}, "second device") == Outcome::Released, "host import teardown: the second device did not import the block the first one released"); + std::puts("host import teardown tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAddressDimension.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAddressDimension.cpp index 0910f962a..fe5bc89fc 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAddressDimension.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAddressDimension.cpp @@ -25,12 +25,15 @@ constexpr std::uint32_t Size = 16; constexpr std::uint32_t Levels = 5; constexpr std::uint32_t Format8888UNorm = 56; constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t Type3D = 10; +constexpr std::uint32_t Depth = 2; constexpr std::uint32_t ArrayInput = 0; constexpr std::uint32_t ArrayResult = 8; constexpr std::uint32_t PlainInput = 16; constexpr std::uint32_t PlainResult = 24; alignas(256) std::array Buffer{}; alignas(256) std::array Texels{}; +alignas(256) std::array Volume{}; alignas(256) constexpr std::array Code{ 0x34020087, 0xe0301000, 0x80000201, 0xe0301004, 0x80000301, 0xe0301008, 0x80000401, 0xe030100c, @@ -44,6 +47,12 @@ alignas(256) constexpr std::array Code{ 0xe0701074, 0x80000d01, 0xe0701078, 0x80000e01, 0xe070107c, 0x80000f01, 0xbf810000, }; +alignas(256) constexpr std::array VolumeCode{ + 0x34020087, 0xe030104c, 0x80001301, 0xe0301050, 0x80001401, 0xbf8c3f70, 0xf0001f08, 0x00010c13, + 0xe0701070, 0x80000c01, 0xe0701074, 0x80000d01, 0xe0701078, 0x80000e01, 0xe070107c, 0x80000f01, + 0xbf810000, +}; + alignas(256) constexpr std::array NarrowCode{ 0x34020087, 0xe0301040, 0x80001001, 0xe0301044, 0x80001101, 0xbf8c3f70, 0xf0900f00, 0x00610c10, 0xe0701070, 0x80000c01, 0xbf810000, @@ -100,7 +109,7 @@ void FillTexture() { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data) { @@ -117,14 +126,49 @@ std::array TextureDescriptor(const void* data) { }; } +std::array VolumeDescriptor() { + const auto address = static_cast(reinterpret_cast(Volume.data())); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (((Size - 1u) & 3u) << 30u), + ((Size - 1u) >> 2u) | ((Size - 1u) << 14u), + 0xfacu | (Type3D << 28u), + Depth - 1u, + 0u, + 0u, + 0u, + }; +} + +std::uint8_t VolumeSliceMarker(std::uint32_t z) { + return static_cast(z * 100u + 7u); +} + +void FillVolume() { + const auto geometry = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(VolumeDescriptor())); + Require(geometry.guestBytes <= Volume.size(), "image address dimension: the volume does not fit the texel storage"); + const auto& mip = geometry.mips.at(0); + Volume.fill(0xeeu); + for (std::uint32_t z = 0; z < Depth; ++z) { + for (std::uint32_t y = 0; y < Size; ++y) { + for (std::uint32_t x = 0; x < Size; ++x) { + auto* texel = &Volume[geometry.GuestLayerOffset(z) + mip.tiledOffset + static_cast(y) * mip.pitchBytes + x * 4u]; + texel[0] = VolumeSliceMarker(z); + texel[1] = static_cast(x); + texel[2] = static_cast(y); + texel[3] = 255u; + } + } + } +} + std::array SamplerDescriptor() { return {0u, 0xfffu << 12u, 1u << 26u, 0u}; } -void Run(AgcDriver::VulkanDevice& device, std::span code) { +void Run(AgcDriver::VulkanDevice& device, std::span code, const std::array& texture) { std::vector userData(16, 0u); const auto buffer = BufferDescriptor(Buffer.data(), static_cast(Buffer.size() * 4u)); - const auto texture = TextureDescriptor(Texels.data()); const auto sampler = SamplerDescriptor(); std::copy(buffer.begin(), buffer.end(), userData.begin()); std::copy(texture.begin(), texture.end(), userData.begin() + 4); @@ -158,10 +202,21 @@ void Check() { } } +void CheckVolume() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto sample = SampleOf(tid); + const std::array expected{VolumeSliceMarker(0), sample.x >> sample.level, sample.y >> sample.level, 255u}; + for (std::uint32_t component = 0; component < 4u; ++component) { + const float value = std::bit_cast(Buffer[tid * Words + PlainResult + 4u + component]) * 255.0f; + Require(std::lround(value) == static_cast(expected[component]), "image_load 2d on a 3D texture: thread " + std::to_string(tid) + " component " + std::to_string(component) + " is " + std::to_string(value) + ", expected " + std::to_string(expected[component])); + } + } +} + void RequireRefused(AgcDriver::VulkanDevice& device) { std::string refusal; try { - Run(device, NarrowCode); + Run(device, NarrowCode, TextureDescriptor(Texels.data())); } catch (const std::exception& error) { refusal = error.what(); } @@ -176,8 +231,11 @@ int main() { if (!device) return VulkanTestSkipped; FillInput(); FillTexture(); - Run(*device, Code); + Run(*device, Code, TextureDescriptor(Texels.data())); Check(); + FillVolume(); + Run(*device, VolumeCode, VolumeDescriptor()); + CheckVolume(); RequireRefused(*device); std::puts("image address dimension tests passed"); return 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics.cpp index f27314c9f..d60347995 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics.cpp @@ -70,7 +70,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data, std::uint32_t width, std::uint32_t height, std::uint32_t format) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics64.cpp index 79da0abe0..024365637 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics64.cpp @@ -93,7 +93,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data, std::uint32_t width, std::uint32_t height, std::uint32_t format) { @@ -175,6 +175,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityInt64ImageEXT)) { + std::puts("skipped, the device has no shaderImageInt64Atomics"); + return VulkanTestSkipped; + } FillInput(); for (const auto format : {Format32_32UInt, Format32_32SInt, Format32_32Float}) { Run(*device, format); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsBitwise.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsBitwise.cpp index 59fbc3c05..ca6796d7a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsBitwise.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsBitwise.cpp @@ -64,7 +64,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data, std::uint32_t width, std::uint32_t height) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsFloat.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsFloat.cpp index 1799bebc2..a7b492f8e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsFloat.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsFloat.cpp @@ -474,7 +474,7 @@ std::array ExpectedRow(std::uint32_t row) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data, std::uint32_t format) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageBy.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageBy.cpp new file mode 100644 index 000000000..fccae5c4f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageBy.cpp @@ -0,0 +1,193 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Width = 61; +constexpr std::uint32_t Height = 3; +constexpr std::uint32_t Levels = 2; +constexpr std::size_t Bytes = 16384; +constexpr std::uint32_t Sentinel = 0xcafe0000u; +constexpr std::uint32_t R8UInt = 5; +constexpr std::uint32_t R16UInt = 11; +constexpr std::uint32_t Rg8UInt = 18; +constexpr std::uint32_t R32UInt = 20; +alignas(256) std::array Texels{}; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +struct Format { + std::uint32_t id; + std::uint32_t bytes; + std::uint32_t components; +}; + +std::vector Code(std::uint32_t opcode, std::uint32_t dmask) { + std::vector code{0x34060084u, 0xe0381000u, 0x80031e03u, 0xbf8c3f70u}; + for (std::uint32_t index = 0; index < 4u; ++index) { + code.push_back(0x7e0002ffu | ((10u + index) << 17u)); + code.push_back(Sentinel + index); + } + code.push_back(0xf0000008u | (opcode << 18u) | (dmask << 8u)); + code.push_back(0x00010a1eu); + code.push_back(0xbf8c3f70u); + code.push_back(0xe0781000u); + code.push_back(0x80000a03u); + code.push_back(0xbf810000u); + return code; +} + +std::array Descriptor(std::uint32_t format, std::uint32_t swizzle) { + const auto address = reinterpret_cast(Texels.data()); + return {static_cast(address >> 8u), static_cast(address >> 40u) | (format << 20u) | (((Width - 1u) & 3u) << 30u), + ((Width - 1u) >> 2u) | ((Height - 1u) << 14u), swizzle | ((Levels - 1u) << 16u) | (9u << 28u), 0u, 1u << 4u, 0u, 0u}; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::uint32_t TexelValue(std::uint32_t level, std::uint32_t x, std::uint32_t y, std::uint32_t component, std::uint32_t bytes) { + const auto value = level * 0x9e37u + y * 0x51u + x * 0x13u + component * 0x2bu + 1u; + return bytes == 1u ? value & 0xffu : value & 0xffffu; +} + +void FillTexels(const Format& format) { + Texels.fill(0xeeu); + const auto descriptor = Descriptor(format.id, 0xfacu); + const auto surface = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(descriptor)); + Require(surface.mips.size() == Levels, "image BY mip count"); + for (std::uint32_t level = 0; level < Levels; ++level) { + const auto& mip = surface.mips[level]; + const auto width = std::max(Width >> level, 1u); + const auto height = std::max(Height >> level, 1u); + for (std::uint32_t y = 0; y < height; ++y) { + for (std::uint32_t x = 0; x < width; ++x) { + for (std::uint32_t component = 0; component < format.components; ++component) { + const auto value = TexelValue(level, x, y, component, format.bytes); + const auto offset = mip.tiledOffset + static_cast(y) * mip.pitchBytes + (x * format.components + component) * format.bytes; + Require(offset + format.bytes <= Texels.size(), "image BY texture size"); + std::memcpy(Texels.data() + offset, &value, format.bytes); + } + } + } + } +} + +void FillInput() { + constexpr std::array xs{0u, 1u, 2u, 3u, 5u, 6u, 7u, Width - 4u, Width - 3u, Width - 2u, Width - 1u, Width, 29u, 30u, 0xffffffffu, 0x80000000u}; + constexpr std::array ys{0u, 1u, 2u, Height, 0xffffffffu}; + constexpr std::array mips{0u, 1u, Levels}; + for (std::uint32_t thread = 0; thread < Threads; ++thread) { + Input[thread * 4u + 0u] = xs[thread % xs.size()]; + Input[thread * 4u + 1u] = ys[(thread * 3u) % ys.size()]; + Input[thread * 4u + 2u] = mips[(thread / 2u) % mips.size()]; + Input[thread * 4u + 3u] = 0u; + } +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t opcode, std::uint32_t format, std::uint32_t dmask, std::uint32_t swizzle = 0xfacu) { + const auto code = Code(opcode, dmask); + std::vector userData(16, 0u); + const auto output = BufferDescriptor(Output.data(), static_cast(sizeof(Output))); + const auto descriptor = Descriptor(format, swizzle); + const auto input = BufferDescriptor(Input.data(), static_cast(sizeof(Input))); + std::copy(output.begin(), output.end(), userData.begin()); + std::copy(descriptor.begin(), descriptor.end(), userData.begin() + 4); + std::copy(input.begin(), input.end(), userData.begin() + 12); + const std::span words(code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(words)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderRecompiler::ShaderStage::Compute, reinterpret_cast(code.data()), words, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, device.Target(), {0, 0, 0, 128}}; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(AgcDriver::VulkanDevice& device, std::uint32_t opcode, const Format& format, std::uint32_t dmask) { + const auto elements = (opcode & 1u) != 0u ? 4u : 2u; + const bool mip = (opcode & 8u) != 0u; + const auto dwords = static_cast(std::popcount(dmask)); + const auto name = "image BY opcode " + std::to_string(opcode) + " format " + std::to_string(format.id) + " dmask " + std::to_string(dmask); + FillTexels(format); + FillInput(); + const auto initial = Texels; + Output.fill(0xdeadbeefu); + Run(device, opcode, format.id, dmask); + for (std::uint32_t thread = 0; thread < Threads; ++thread) { + const auto x = Input[thread * 4u + 0u]; + const auto y = Input[thread * 4u + 1u]; + const auto level = mip ? Input[thread * 4u + 2u] : 0u; + const auto width = level < Levels ? std::max(Width >> level, 1u) : 0u; + const auto height = level < Levels ? std::max(Height >> level, 1u) : 0u; + const bool inside = level < Levels && x < width && width - x >= elements && y < height; + const auto first = x & ~(elements - 1u); + for (std::uint32_t index = 0; index < 4u; ++index) { + auto expected = Sentinel + index; + if (index < dwords) expected = inside ? TexelValue(level, first + index / format.components, y, index % format.components, format.bytes) : 0u; + const auto actual = Output[thread * 4u + index]; + Require(actual == expected, name + ": thread " + std::to_string(thread) + " (x " + std::to_string(x) + ", y " + std::to_string(y) + ", mip " + std::to_string(level) + ") v" + std::to_string(10u + index) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected)); + } + } + Require(Texels == initial, name + ": the load changed the texture"); +} + +void Refused(AgcDriver::VulkanDevice& device, std::uint32_t opcode, std::uint32_t format, std::uint32_t dmask, std::uint32_t swizzle, const char* reason) { + bool refused = false; + try { Run(device, opcode, format, dmask, swizzle); } + catch (const std::exception& error) { refused = std::string(error.what()).find(reason) != std::string::npos; } + Require(refused, "image BY opcode " + std::to_string(opcode) + " format " + std::to_string(format) + " was not refused"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + GuestAllocations::Mutation().Add(Texels.data(), Bytes, true, true); + constexpr Format r8{R8UInt, 1u, 1u}; + constexpr Format r16{R16UInt, 2u, 1u}; + constexpr Format rg8{Rg8UInt, 1u, 2u}; + for (const auto opcode : {0x42u, 0x4au}) { + Check(*device, opcode, r8, 0x3u); + Check(*device, opcode, r16, 0x3u); + Check(*device, opcode, rg8, 0xfu); + } + for (const auto opcode : {0x43u, 0x4bu}) Check(*device, opcode, r8, 0xfu); + constexpr auto layout = "elements fill one dword"; + Refused(*device, 0x42u, R32UInt, 0x3u, 0xfacu, layout); + Refused(*device, 0x42u, R8UInt, 0xfu, 0xfacu, layout); + Refused(*device, 0x43u, R16UInt, 0xfu, 0xfacu, layout); + Refused(*device, 0x43u, Rg8UInt, 0xfu, 0xfacu, layout); + Refused(*device, 0x42u, R16UInt, 0x3u, 0xf2eu, layout); + for (const auto opcode : {0x52u, 0x53u, 0x5au, 0x5bu}) Refused(*device, opcode, R8UInt, 0xfu, 0xfacu, "stores are not implemented"); + GuestAllocations::Mutation().Remove(Texels.data()); + std::cout << "image BY2/BY4 execution tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageCacheBits.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageCacheBits.cpp index ff6297c0a..f321df600 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageCacheBits.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageCacheBits.cpp @@ -200,7 +200,7 @@ constexpr std::array, Threads> SampleEx std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data, std::uint32_t format, std::uint32_t width, std::uint32_t height) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp index 376473101..a74578e4d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp @@ -34,6 +34,8 @@ constexpr std::uint32_t LoadGroups = LoadWidth / Threads; constexpr std::uint32_t Type2D = 9; constexpr std::uint32_t UnormFormat = 30; constexpr std::uint32_t UintFormat = 34; +constexpr std::uint32_t FloatFormat = 43; +constexpr std::size_t FloatOffset = 40960; constexpr std::uint32_t SwizzleXYZ1 = 0x3acu; constexpr std::uint32_t SwizzleXZY1 = 0x374u; constexpr std::uint32_t SwizzleZYX1 = 0x32eu; @@ -130,7 +132,7 @@ std::string Hex(std::uint32_t value) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data, std::uint32_t format, std::uint32_t swizzle, std::uint32_t width) { @@ -249,14 +251,85 @@ void CheckStore(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span } } -void RequireRefused(AgcDriver::VulkanDevice& device, std::span code, const std::uint8_t* texels, std::uint32_t swizzle, const std::array& extra, const std::string& reason, const std::string& what) { +constexpr std::array FloatValues{0.0f, 1.0f, 0.5f, 2.0f, 0.25f, 3.0f, 1.5f, 10.0f, 0.75f, 6.5f, 0.125f}; + +constexpr std::array, 7> FloatStoreEdges{{ + {0xbf800000u, 0x80000000u, 0x7fc00000u, 0xfc000000u}, + {0x7f800000u, 0x7149f2cau, 0x477fff00u, 0xf7feffe0u}, + {0x3f830000u, 0x3f818000u, 0x35800000u, 0x002f01e0u}, + {0x00000001u, 0xff800000u, 0xffc00000u, 0xfc000000u}, + {0x477c0000u, 0x4788b800u, 0x33000000u, 0x001effdfu}, + {0x36400000u, 0x477e0000u, 0x3dcccccdu, 0x5cdefc01u}, + {0x7f800001u, 0x7f810000u, 0x7fa00000u, 0xfa1f07e1u}, +}}; + +constexpr std::array, 3> FloatLoadEdges{{ + {0x3e0u | (0x7c0u << 10u) | (0x03fu << 21u), 0x7f800000u, 0x7f800000u, 0x387c0000u}, + {0x001u | (0x041u << 10u) | (0x7bfu << 21u), 0x36000000u, 0x38820000u, 0x477e0000u}, + {0x3dfu | (0x001u << 10u), 0x477c0000u, 0x35800000u, 0x00000000u}, +}}; + +std::uint32_t SmallFloat(float value, std::uint32_t mantissaBits) { + if (value == 0.0f) return 0u; + const auto bits = std::bit_cast(value); + const auto exponent = ((bits >> 23u) & 0xffu) - 127u + 15u; + return (exponent << mantissaBits) | ((bits & 0x7fffffu) >> (23u - mantissaBits)); +} + +std::uint32_t FloatTexel(std::uint32_t index) { + return SmallFloat(FloatValues[index % 11u], 5u) | (SmallFloat(FloatValues[(index + 4u) % 11u], 6u) << 10u) | (SmallFloat(FloatValues[(index + 8u) % 11u], 6u) << 21u); +} + +void CheckFloatLoad(AgcDriver::VulkanDevice& device, std::uint8_t* texels) { + for (std::uint32_t index = 0; index < LoadWidth; ++index) { + const auto texel = index < FloatLoadEdges.size() ? FloatLoadEdges[index][0] : FloatTexel(index); + std::memcpy(texels + index * 4u, &texel, 4u); + } + Run(device, LoadXyzw, TextureDescriptor(texels, FloatFormat, SwizzleXYZ1, LoadWidth), {}, LoadGroups); + for (std::uint32_t index = 0; index < LoadWidth; ++index) { + std::array expected{std::bit_cast(FloatValues[index % 11u]), std::bit_cast(FloatValues[(index + 4u) % 11u]), std::bit_cast(FloatValues[(index + 8u) % 11u]), std::bit_cast(1.0f)}; + if (index < FloatLoadEdges.size()) { + expected = {FloatLoadEdges[index][1], FloatLoadEdges[index][2], FloatLoadEdges[index][3], std::bit_cast(1.0f)}; + } + for (std::uint32_t component = 0; component < 4u; ++component) { + const auto actual = Output[index * 4u + component]; + Require(actual == expected[component], "image_load of R10_G11_B11_FLOAT: texel " + std::to_string(index) + " component " + std::to_string(component) + " is " + Hex(actual) + ", expected " + Hex(expected[component])); + } + } +} + +void CheckFloatStore(AgcDriver::VulkanDevice& device, std::uint8_t* texels) { + std::fill(texels, texels + Threads * 4u, static_cast(0xa5u)); + Input.fill(0u); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + Input[tid * 4u + 0u] = std::bit_cast(FloatValues[tid % 11u]); + Input[tid * 4u + 1u] = std::bit_cast(FloatValues[(tid + 4u) % 11u]); + Input[tid * 4u + 2u] = std::bit_cast(FloatValues[(tid + 8u) % 11u]); + if (tid < FloatStoreEdges.size()) { + for (std::uint32_t component = 0; component < 3u; ++component) { + Input[tid * 4u + component] = FloatStoreEdges[tid][component]; + } + } + } + Run(device, StoreXyz, TextureDescriptor(texels, FloatFormat, SwizzleXYZ1, Threads), BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)), 1); + AgcDriver::Graphics::StorageTexture::FlushPending(reinterpret_cast(texels), Threads * 4u, nullptr, "test"); + device.WaitIdle(); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + std::uint32_t actual = 0; + std::memcpy(&actual, texels + tid * 4u, 4u); + const auto expected = tid < FloatStoreEdges.size() ? FloatStoreEdges[tid][3] : FloatTexel(tid); + Require(actual == expected, "image_store of R10_G11_B11_FLOAT: texel " + std::to_string(tid) + " is " + Hex(actual) + ", expected " + Hex(expected)); + } +} + +void RequireRefused(AgcDriver::VulkanDevice& device, std::span code, const std::uint8_t* texels, std::uint32_t swizzle, const std::array& extra, const std::string& reason, const std::string& what, std::uint32_t format = UnormFormat) { std::string refusal; try { - Run(device, code, TextureDescriptor(texels, UnormFormat, swizzle, LoadWidth), extra, 1); + Run(device, code, TextureDescriptor(texels, format, swizzle, LoadWidth), extra, 1); } catch (const std::exception& error) { refusal = error.what(); } - Require(refusal.find(reason) != std::string::npos, what + " of a 10_11_11_UNORM image was not refused: " + refusal); + Require(refusal.find(reason) != std::string::npos, what + " of a converted image was not refused: " + refusal); } } @@ -279,9 +352,12 @@ int main() { CheckRoundTrip(*device, texels); CheckStore(*device, texels + StoreOffset, StoreXyz, 0x7u, "image_store dmask:0x7"); CheckStore(*device, texels + StoreOffset, StoreXz, 0x5u, "image_store dmask:0x5"); - RequireRefused(*device, SampleLz, texels, SwizzleXYZ1, PointSampler, "samples or gathers a converted unorm image", "image_sample_lz"); - RequireRefused(*device, Gather4Lz, texels, SwizzleXYZ1, PointSampler, "samples or gathers a converted unorm image", "image_gather4_lz"); - RequireRefused(*device, GetLod, texels, SwizzleXYZ1, PointSampler, "queries the level of detail of a converted unorm image", "image_get_lod"); + CheckFloatStore(*device, texels + FloatOffset); + CheckFloatLoad(*device, texels + FloatOffset); + RequireRefused(*device, SampleLz, texels + FloatOffset, SwizzleXYZ1, PointSampler, "samples or gathers a converted float image", "image_sample_lz of R10_G11_B11_FLOAT", FloatFormat); + RequireRefused(*device, SampleLz, texels, SwizzleXYZ1, PointSampler, "sampling, gathering or querying LOD of a converted unorm image", "image_sample_lz"); + RequireRefused(*device, Gather4Lz, texels, SwizzleXYZ1, PointSampler, "sampling, gathering or querying LOD of a converted unorm image", "image_gather4_lz"); + RequireRefused(*device, GetLod, texels, SwizzleXYZ1, PointSampler, "sampling, gathering or querying LOD of a converted unorm image", "image_get_lod"); RequireRefused(*device, LoadD16, texels, SwizzleXYZ1, {}, "converted unorm image with 16-bit data", "image_load d16"); RequireRefused(*device, StoreD16, texels + StoreOffset, SwizzleXYZ1, BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)), "converted unorm image with 16-bit data", "image_store d16"); RequireRefused(*device, LoadXyzw, texels, SwizzleXYZW, {}, "selects a channel the converted image format does not have", "image_load with DST_SEL X Y Z W"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageGatherOneDimensional.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageGatherOneDimensional.cpp index ac769d0b4..a15256a72 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageGatherOneDimensional.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageGatherOneDimensional.cpp @@ -48,7 +48,7 @@ struct Variant { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageGatherOneDimensionalArray.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageGatherOneDimensionalArray.cpp index a46fc19e1..e6badb991 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageGatherOneDimensionalArray.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageGatherOneDimensionalArray.cpp @@ -49,7 +49,7 @@ struct Variant { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageLoadPacked.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageLoadPacked.cpp index b0612df8b..93065073d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageLoadPacked.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageLoadPacked.cpp @@ -107,7 +107,7 @@ std::array RawTexel(const Format& format, std::uint32_t level, std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data, std::uint32_t format, std::uint32_t swizzle) { @@ -140,8 +140,10 @@ void Run(AgcDriver::VulkanDevice& device, std::uint32_t format, std::uint32_t sw device.Target(), {0, 0, 0, 128} }; - request.useCache = false; const auto result = ShaderRecompiler::Recompile(request); + static ShaderRecompiler::CompiledShaderArtifact first; + if (first.variantId == 0u) first = result; + else Require(result.cacheHit && result.variantId == first.variantId, "runtime image format changed the compiled artifact"); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageOneDimensionalArray.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageOneDimensionalArray.cpp index 28e3ddade..ee2a7c978 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageOneDimensionalArray.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageOneDimensionalArray.cpp @@ -37,7 +37,7 @@ alignas(256) constexpr std::array Code{ std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImagePackedHorizontalGather.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePackedHorizontalGather.cpp new file mode 100644 index 000000000..336a066bf --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePackedHorizontalGather.cpp @@ -0,0 +1,179 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Width = 16; +constexpr std::uint32_t Height = 4; +constexpr std::uint32_t Format8888UNorm = 56; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t IdentitySwizzle = 0xfacu; + +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; +alignas(4096) std::array Texels{}; + +alignas(256) std::array PckGather4h{ + 0x34020084, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xf1880f08, 0x00820804, 0xbf8c3f70, 0xe0781000, + 0x80010801, 0xbf810000, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::array TextureDescriptor() { + const auto address = static_cast(reinterpret_cast(Texels.data())); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (((Width - 1u) & 3u) << 30u), + ((Width - 1u) >> 2u) | ((Height - 1u) << 14u), + IdentitySwizzle | (Type2D << 28u), + 0u, 0u, 0u, 0u, + }; +} + +std::array SamplerDescriptor() { + return {(2u << 3u) | (2u << 6u), 0u, 0u, 0u}; +} + +void FillTexels() { + Texels.fill(0); + const auto descriptor = TextureDescriptor(); + const auto surface = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(descriptor)); + const auto& mip = surface.mips.at(0); + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + auto* texel = Texels.data() + mip.tiledOffset + static_cast(y) * mip.pitchBytes + static_cast(x) * 4u; + for (std::uint32_t c = 0; c < 4u; ++c) { + texel[c] = static_cast(((y * Width + x) * 4u + c + 1u) & 0xffu); + } + } + } +} + +void FillInput() { + constexpr std::array, 16> edges{{ + {0.03125f, 0.0f}, {0.09375f, 0.25f}, {0.53125f, 0.5f}, {0.96875f, 0.75f}, + {-0.25f, -0.3f}, {1.5f, 0.99f}, {0.4f, 1.0f}, {0.2f, 2.0f}, + {0.3f, 0.6f}, {0.7f, 0.1f}, {0.0f, -1.0f}, {1.0f, 0.5f}, + {0.0625f, 0.0f}, {0.15f, 0.375f}, {0.85f, 0.875f}, {-3.0f, 0.2f}, + }}; + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + float u = static_cast(tid % Width) / static_cast(Width); + float v = static_cast(tid % Height) / static_cast(Height); + if (tid >= Width) { + u = edges[tid - Width][0]; + v = edges[tid - Width][1]; + if (tid % 2u == 1u && tid < Width + 4u) u = std::nextafter(u, -1.0f); + } + Input[tid * 4u + 0u] = std::bit_cast(u); + Input[tid * 4u + 1u] = std::bit_cast(v); + Input[tid * 4u + 2u] = 0u; + Input[tid * 4u + 3u] = 0u; + } +} + +std::uint32_t TexelBits(std::int32_t x, std::int32_t y) { + const auto cx = static_cast(std::clamp(x, 0, static_cast(Width) - 1)); + const auto cy = static_cast(std::max(y, 0)); + if (cy >= Height) { + return 0u; + } + std::uint32_t bits = 0; + for (std::uint32_t c = 0; c < 4u; ++c) { + const auto byte = ((cy * Width + cx) * 4u + c + 1u) & 0xffu; + bits |= byte << (8u * c); + } + return bits; +} + +std::array Expected(std::uint32_t tid, std::uint32_t dmask) { + const float u = std::bit_cast(Input[tid * 4u + 0u]); + const float v = std::bit_cast(Input[tid * 4u + 1u]); + const auto anchor = static_cast(std::floor(u * static_cast(Width) - 0.5f)); + const auto row = static_cast(std::floor(v * static_cast(Height))); + std::uint32_t stream[8] = {}; + for (std::uint32_t element = 0; element < 4u; ++element) { + const auto bits = TexelBits(anchor + static_cast(element) - 1, row); + stream[element] = bits; + } + std::array selected{}; + std::uint32_t next = 0; + for (std::uint32_t bit = 0; bit < 4u; ++bit) { + if (((dmask >> bit) & 1u) != 0u) selected[next++] = stream[bit]; + } + return selected; +} + +ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device) { + std::vector userData(20, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(sizeof(Input))); + const auto output = BufferDescriptor(Output.data(), static_cast(sizeof(Output))); + const auto texture = TextureDescriptor(); + const auto sampler = SamplerDescriptor(); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + std::copy(texture.begin(), texture.end(), userData.begin() + 8); + std::copy(sampler.begin(), sampler.end(), userData.begin() + 16); + const std::array memory{{{reinterpret_cast(PckGather4h.data()), std::as_bytes(std::span(PckGather4h))}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(PckGather4h.data()), PckGather4h, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + return ShaderRecompiler::Recompile(request); +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t dmask) { + PckGather4h[4] = 0xf1880008u | (dmask << 8u); + Output.fill(0xdeadbeefu); + const auto result = Compile(device); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(PckGather4h.data())); + device.WaitIdle(); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto expected = Expected(tid, dmask); + for (std::uint32_t component = 0; component < static_cast(std::popcount(dmask)); ++component) { + const std::uint32_t actual = Output[tid * 4u + component]; + Require(actual == expected[component], std::string("packed horizontal gather: dmask ") + std::to_string(dmask) + " thread " + std::to_string(tid) + " register " + std::to_string(component) + " is 0x" + std::to_string(actual) + ", expected 0x" + std::to_string(expected[component])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + FillTexels(); + FillInput(); + for (const std::uint32_t dmask : {0xfu, 0x1u, 0x5u, 0xau, 0x8u}) Run(*device, dmask); + std::puts("image packed horizontal gather tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImagePckN.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePckN.cpp new file mode 100644 index 000000000..b16b20847 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePckN.cpp @@ -0,0 +1,196 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Width = 61; +constexpr std::uint32_t Height = 3; +constexpr std::uint32_t Levels = 2; +constexpr std::size_t Bytes = 16384; +constexpr std::uint32_t Sentinel = 0xcafe0000u; +constexpr std::uint32_t R8UInt = 5; +constexpr std::uint32_t R16UInt = 11; +constexpr std::uint32_t Rg8UInt = 18; +constexpr std::uint32_t R32UInt = 20; +alignas(256) std::array Texels{}; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +struct Format { + std::uint32_t id; + std::uint32_t bytes; + std::uint32_t components; +}; + +std::vector Code(std::uint32_t opcode, std::uint32_t dmask) { + std::vector code{0x34060084u, 0xe0381000u, 0x80031e03u, 0xbf8c3f70u}; + for (std::uint32_t index = 0; index < 4u; ++index) { + code.push_back(0x7e0002ffu | ((10u + index) << 17u)); + code.push_back(Sentinel + index); + } + code.push_back(0xf0000008u | (opcode << 18u) | (dmask << 8u)); + code.push_back(0x00010a1eu); + code.push_back(0xbf8c3f70u); + code.push_back(0xe0781000u); + code.push_back(0x80000a03u); + code.push_back(0xbf810000u); + return code; +} + +std::array Descriptor(std::uint32_t format, std::uint32_t swizzle) { + const auto address = reinterpret_cast(Texels.data()); + return {static_cast(address >> 8u), static_cast(address >> 40u) | (format << 20u) | (((Width - 1u) & 3u) << 30u), + ((Width - 1u) >> 2u) | ((Height - 1u) << 14u), swizzle | ((Levels - 1u) << 16u) | (9u << 28u), 0u, 1u << 4u, 0u, 0u}; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::uint32_t TexelValue(std::uint32_t level, std::uint32_t x, std::uint32_t y, std::uint32_t component, std::uint32_t bytes) { + const auto value = level * 0x9e37u + y * 0x51u + x * 0x13u + component * 0x2bu + 1u; + return bytes == 1u ? value & 0xffu : value & 0xffffu; +} + +void FillTexels(const Format& format) { + Texels.fill(0xeeu); + const auto descriptor = Descriptor(format.id, 0xfacu); + const auto surface = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(descriptor)); + Require(surface.mips.size() == Levels, "image PCK mip count"); + for (std::uint32_t level = 0; level < Levels; ++level) { + const auto& mip = surface.mips[level]; + const auto width = std::max(Width >> level, 1u); + const auto height = std::max(Height >> level, 1u); + for (std::uint32_t y = 0; y < height; ++y) { + for (std::uint32_t x = 0; x < width; ++x) { + for (std::uint32_t component = 0; component < format.components; ++component) { + const auto value = TexelValue(level, x, y, component, format.bytes); + const auto offset = mip.tiledOffset + static_cast(y) * mip.pitchBytes + (x * format.components + component) * format.bytes; + Require(offset + format.bytes <= Texels.size(), "image PCK texture size"); + std::memcpy(Texels.data() + offset, &value, format.bytes); + } + } + } + } +} + +void FillInput() { + constexpr std::array xs{0u, 1u, 2u, 3u, 5u, 6u, 7u, Width - 4u, Width - 3u, Width - 2u, Width - 1u, Width, 29u, 30u, 0xffffffffu, 0x80000000u}; + constexpr std::array ys{0u, 1u, 2u, Height, 0xffffffffu}; + constexpr std::array mips{0u, 1u, Levels}; + for (std::uint32_t thread = 0; thread < Threads; ++thread) { + Input[thread * 4u + 0u] = xs[thread % xs.size()]; + Input[thread * 4u + 1u] = ys[(thread * 3u) % ys.size()]; + Input[thread * 4u + 2u] = mips[(thread / 2u) % mips.size()]; + Input[thread * 4u + 3u] = 0u; + } +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t opcode, std::uint32_t format, std::uint32_t dmask, std::uint32_t swizzle = 0xfacu) { + const auto code = Code(opcode, dmask); + std::vector userData(16, 0u); + const auto output = BufferDescriptor(Output.data(), static_cast(sizeof(Output))); + const auto descriptor = Descriptor(format, swizzle); + const auto input = BufferDescriptor(Input.data(), static_cast(sizeof(Input))); + std::copy(output.begin(), output.end(), userData.begin()); + std::copy(descriptor.begin(), descriptor.end(), userData.begin() + 4); + std::copy(input.begin(), input.end(), userData.begin() + 12); + const std::span words(code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(words)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderRecompiler::ShaderStage::Compute, reinterpret_cast(code.data()), words, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, device.Target(), {0, 0, 0, 128}}; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(AgcDriver::VulkanDevice& device, std::uint32_t opcode, const Format& format, std::uint32_t dmask) { + const auto elements = opcode == 0x71u || opcode == 0x74u ? 4u : 2u; + const bool mip = opcode == 0x73u || opcode == 0x74u; + const auto texelBits = format.bytes * format.components * 8u; + const auto name = "image PCK opcode " + std::to_string(opcode) + " format " + std::to_string(format.id) + " dmask " + std::to_string(dmask); + FillTexels(format); + FillInput(); + const auto initial = Texels; + Output.fill(0xdeadbeefu); + Run(device, opcode, format.id, dmask); + for (std::uint32_t thread = 0; thread < Threads; ++thread) { + const auto x = Input[thread * 4u + 0u]; + const auto y = Input[thread * 4u + 1u]; + const auto level = mip ? Input[thread * 4u + 2u] : 0u; + const auto width = level < Levels ? std::max(Width >> level, 1u) : 0u; + const auto height = level < Levels ? std::max(Height >> level, 1u) : 0u; + const bool inside = level < Levels && x < width && width - x >= elements && y < height; + const auto first = x & ~(elements - 1u); + std::uint32_t packed = 0; + for (std::uint32_t element = 0; inside && element < elements; ++element) { + std::uint32_t texel = 0; + for (std::uint32_t component = 0; component < format.components; ++component) texel |= TexelValue(level, first + element, y, component, format.bytes) << (component * format.bytes * 8u); + packed |= texel << (element * texelBits); + } + for (std::uint32_t index = 0; index < 4u; ++index) { + const auto expected = index == 0u ? packed : Sentinel + index; + const auto actual = Output[thread * 4u + index]; + Require(actual == expected, name + ": thread " + std::to_string(thread) + " (x " + std::to_string(x) + ", y " + std::to_string(y) + ", mip " + std::to_string(level) + ") v" + std::to_string(10u + index) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected)); + } + } + Require(Texels == initial, name + ": the load changed the texture"); +} + +void Refused(AgcDriver::VulkanDevice& device, std::uint32_t opcode, std::uint32_t format, std::uint32_t dmask, std::uint32_t swizzle, const char* reason) { + bool refused = false; + try { Run(device, opcode, format, dmask, swizzle); } + catch (const std::exception& error) { refused = std::string(error.what()).find(reason) != std::string::npos; } + Require(refused, "image PCK opcode " + std::to_string(opcode) + " format " + std::to_string(format) + " was not refused"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + GuestAllocations::Mutation().Add(Texels.data(), Bytes, true, true); + constexpr Format r8{R8UInt, 1u, 1u}; + constexpr Format r16{R16UInt, 2u, 1u}; + constexpr Format rg8{Rg8UInt, 1u, 2u}; + for (const auto opcode : {0x70u, 0x73u}) { + Check(*device, opcode, r8, 0x1u); + Check(*device, opcode, r16, 0x1u); + Check(*device, opcode, rg8, 0x1u); + } + for (const auto opcode : {0x71u, 0x74u}) Check(*device, opcode, r8, 0x1u); + constexpr auto layout = "elements fill one dword"; + Refused(*device, 0x70u, R32UInt, 0x1u, 0xfacu, layout); + Refused(*device, 0x71u, R16UInt, 0x1u, 0xfacu, layout); + Refused(*device, 0x71u, Rg8UInt, 0x1u, 0xfacu, layout); + Refused(*device, 0x70u, R16UInt, 0x1u, 0xf2eu, "identity swizzle"); + for (const auto opcode : {0x76u, 0x77u, 0x79u, 0x7au}) Refused(*device, opcode, R8UInt, 0x1u, 0xfacu, "stores are not implemented"); + GuestAllocations::Mutation().Remove(Texels.data()); + std::cout << "image PCK2/PCK4 execution tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleBiasA16.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleBiasA16.cpp index ec41e7e3f..cdf84fdf1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleBiasA16.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleBiasA16.cpp @@ -27,7 +27,6 @@ constexpr std::uint32_t Levels = 5; constexpr std::uint32_t Clamps = 4; constexpr std::uint32_t Format8888UNorm = 56; constexpr std::uint32_t Type2D = 9; -constexpr std::uint32_t MinLodCapability = 42; alignas(256) std::array Buffer{}; alignas(256) std::array Texels{}; @@ -99,7 +98,7 @@ void FillTexture() { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data) { @@ -171,8 +170,7 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - const auto capabilities = device->Target().supportedCapabilities; - const bool clamped = std::find(capabilities.begin(), capabilities.end(), MinLodCapability) != capabilities.end(); + const bool clamped = TargetHasCapability(device->Target(), spv::CapabilityMinLod); FillInput(); FillTexture(); Run(*device, BiasCode); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleDerivatives.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleDerivatives.cpp index 34488c71a..51e039534 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleDerivatives.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleDerivatives.cpp @@ -289,7 +289,7 @@ void Run(AgcDriver::VulkanDevice& device, const std::array(Buffer.data()); - const std::array buffer{static_cast(address), static_cast((address >> 32u) & 0xffffu), static_cast(Buffer.size() * 4u), 0x01016facu}; + const std::array buffer{static_cast(address), static_cast((address >> 32u) & 0xffffu), static_cast(Buffer.size() * 4u), 0x31016facu}; const auto texture = TextureDescriptor(type, height, depth); const std::array sampler{0x92u, (4u * 256u) << 12u, (1u << 22u) | (1u << 26u), 0u}; std::vector userData(16, 0u); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleLodClamp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleLodClamp.cpp index dcdd05ce3..1a1fcffbc 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleLodClamp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleLodClamp.cpp @@ -27,7 +27,6 @@ constexpr std::uint32_t Gradients = 4; constexpr std::uint32_t Clamps = 4; constexpr std::uint32_t Format8888UNorm = 56; constexpr std::uint32_t Type2D = 9; -constexpr std::uint32_t MinLodCapability = 42; alignas(256) std::array Buffer{}; alignas(256) std::array Texels{}; @@ -92,7 +91,7 @@ void FillTexture() { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data) { @@ -166,8 +165,7 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - const auto capabilities = device->Target().supportedCapabilities; - if (std::find(capabilities.begin(), capabilities.end(), MinLodCapability) == capabilities.end()) { + if (!TargetHasCapability(device->Target(), spv::CapabilityMinLod)) { std::puts("skipped, the device has no shaderResourceMinLod"); return VulkanTestSkipped; } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleReduction.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleReduction.cpp index 648a88e52..d0418ce36 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleReduction.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleReduction.cpp @@ -71,7 +71,7 @@ struct Sampler { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(std::uint32_t format, const void* texels = Texels.data(), std::uint32_t levels = Levels) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleSharedSampler.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleSharedSampler.cpp new file mode 100644 index 000000000..d331ee4e9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleSharedSampler.cpp @@ -0,0 +1,153 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Size = 8; +constexpr std::uint32_t Format32SInt = 21; +constexpr std::uint32_t Format8888UNorm = 56; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t SwizzleX111 = 0x24cu; +constexpr std::uint32_t IdentitySwizzle = 0xfacu; +constexpr std::uint32_t SampleX = 3; +constexpr std::uint32_t SampleY = 5; +constexpr float Tolerance = 1.0f / 64.0f; +constexpr std::array LinearSampler{0x00000092u, 0x00fff000u, 0x00500000u, 0u}; + +alignas(256) std::array Output{}; +alignas(256) std::array Integers{}; +alignas(256) std::array Colors{}; + +alignas(256) constexpr std::array Code{ + 0x34060085u, 0x7e2802ffu, 0x3ef00000u, 0x7e2a02ffu, 0x3f300000u, 0xf09c0f08u, 0x00610a14u, 0xf09c0f08u, + 0x00640e14u, 0xbf8c3f70u, 0xe0781000u, 0x80000a03u, 0xe0781010u, 0x80000e03u, 0xbf810000u, 0xbf810000u, +}; + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +std::uint32_t PitchBytes(std::uint32_t format, std::size_t storage) { + const auto mips = AgcDriver::Graphics::ComputeMipLayout(AgcDriver::Graphics::TextureTileMode::kLinear, format, Size, Size, 1u); + Require(AgcDriver::Graphics::ComputeSurfaceSize(mips, 1) <= storage, "image sample shared sampler: a texture does not fit its storage"); + return mips[0].pitchBytes; +} + +std::int32_t Integer(std::uint32_t x, std::uint32_t y) { + return -static_cast(100u * x + 7u * y + 1u); +} + +void Fill() { + const auto integerPitch = PitchBytes(Format32SInt, Integers.size()); + const auto colorPitch = PitchBytes(Format8888UNorm, Colors.size()); + Integers.fill(0x5au); + Colors.fill(0x5au); + for (std::uint32_t y = 0; y < Size; ++y) { + for (std::uint32_t x = 0; x < Size; ++x) { + const auto bits = static_cast(Integer(x, y)); + auto* integer = &Integers[static_cast(y) * integerPitch + x * 4u]; + for (std::uint32_t byte = 0; byte < 4u; ++byte) integer[byte] = static_cast(bits >> (byte * 8u)); + auto* color = &Colors[static_cast(y) * colorPitch + x * 4u]; + color[0] = x > SampleX ? 255u : 0u; + color[1] = 0u; + color[2] = 0u; + color[3] = 255u; + } + } +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::array TextureDescriptor(const void* texels, std::uint32_t format, std::uint32_t swizzle) { + const auto address = static_cast(reinterpret_cast(texels)); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (format << 20u) | (((Size - 1u) & 3u) << 30u), + ((Size - 1u) >> 2u) | ((Size - 1u) << 14u), + swizzle | (Type2D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +void Run(AgcDriver::VulkanDevice& device) { + Output.fill(0xdeadbeefu); + std::vector userData(24, 0u); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + const auto integers = TextureDescriptor(Integers.data(), Format32SInt, SwizzleX111); + const auto colors = TextureDescriptor(Colors.data(), Format8888UNorm, IdentitySwizzle); + std::copy(output.begin(), output.end(), userData.begin()); + std::copy(integers.begin(), integers.end(), userData.begin() + 4); + std::copy(LinearSampler.begin(), LinearSampler.end(), userData.begin() + 12); + std::copy(colors.begin(), colors.end(), userData.begin() + 16); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {true, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(AgcDriver::VulkanDevice& device) { + Fill(); + Run(device); + const std::array integer{static_cast(Integer(SampleX, SampleY)), 1u, 1u, 1u}; + const std::array color{0.25f, 0.0f, 0.0f, 1.0f}; + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + for (std::uint32_t component = 0; component < 4u; ++component) { + const auto actual = Output[lane * 8u + component]; + Require(actual == integer[component], "32_SINT image_sample_lz through a bilinear sampler shared with 8_8_8_8_UNORM: lane " + std::to_string(lane) + " component " + std::to_string(component) + " is " + Hex(actual) + ", expected " + Hex(integer[component])); + } + for (std::uint32_t component = 0; component < 4u; ++component) { + const auto bits = Output[lane * 8u + 4u + component]; + const auto actual = std::bit_cast(bits); + Require(std::fabs(actual - color[component]) <= Tolerance, "8_8_8_8_UNORM image_sample_lz through a bilinear sampler shared with 32_SINT: lane " + std::to_string(lane) + " component " + std::to_string(component) + " is " + Hex(bits) + ", expected " + std::to_string(color[component])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Check(*device); + std::puts("image sample shared sampler tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp index dc7da4f9a..cc0385031 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp @@ -109,7 +109,7 @@ void FillTexture(std::span texels, std::uint32_t levels) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data, std::uint32_t lastLevel, std::uint32_t maxMip) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp index 87dd6105b..cb8958107 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp @@ -85,7 +85,7 @@ void Fill() { std::array BufferDescriptor(const void* data, std::uint32_t records, std::uint32_t stride = 0u) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), records, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), records, stride == 0u ? 0x31016facu : 0x01016facu}; } std::array TextureDescriptor(const void* texels, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t swizzle) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp index d03b3026e..560166355 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp @@ -119,7 +119,7 @@ std::uint32_t Linear(std::uint32_t code) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* texels, std::uint32_t format, std::uint32_t swizzle) { @@ -147,7 +147,7 @@ void Fill(std::uint32_t format) { } } -void Run(AgcDriver::VulkanDevice& device, std::span code, const void* texels, std::uint32_t format, std::uint32_t swizzle, std::uint32_t groups) { +void Run(AgcDriver::VulkanDevice& device, std::span code, const void* texels, std::uint32_t format, std::uint32_t swizzle, std::uint32_t groups, bool useCache = false) { Output.fill(0xdeadbeefu); std::vector userData(24, 0u); const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); @@ -163,7 +163,7 @@ void Run(AgcDriver::VulkanDevice& device, std::span code, c device.Target(), {0, 0, 0, 128} }; - request.useCache = false; + request.useCache = useCache; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, groups, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); @@ -203,13 +203,15 @@ void CheckStoredLoad(AgcDriver::VulkanDevice& device, GuestBlock& block, std::sp void RequireRefused(AgcDriver::VulkanDevice& device, std::span code, const std::string& what) { Fill(Srgb8_8); - std::string refusal; - try { - Run(device, code, Texels.data(), Srgb8_8, SwizzleXY01, 1); - } catch (const std::exception& error) { - refusal = error.what(); + for (const bool useCache : {false, true, true}) { + std::string refusal; + try { + Run(device, code, Texels.data(), Srgb8_8, SwizzleXY01, 1, useCache); + } catch (const std::exception& error) { + refusal = error.what(); + } + Require(refusal.find("samples or gathers an sRGB image the device cannot sample") != std::string::npos, what + " of an 8_8_SRGB image read through its UNORM view was not refused: " + refusal); } - Require(refusal.find("samples or gathers an sRGB image the device cannot sample") != std::string::npos, what + " of an 8_8_SRGB image read through its UNORM view was not refused: " + refusal); } } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp index be3291656..0a3a058a5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -179,8 +180,11 @@ void Run(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span artifacts; + auto& first = artifacts[reinterpret_cast(code.data())]; + if (first.variantId == 0u) first = result; + else Require(result.cacheHit && result.variantId == first.variantId, "runtime storage image changed the compiled artifact"); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); AgcDriver::Graphics::StorageTexture::FlushPending(reinterpret_cast(texels), TexelBytes, nullptr, "test"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp index 5fa74af2e..8f8e1752b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp @@ -197,7 +197,7 @@ std::vector Expected(const Format& format, std::span std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data, std::uint32_t format, std::uint32_t swizzle) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTexelStatus.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTexelStatus.cpp index d2ff375fe..319faa1a0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTexelStatus.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTexelStatus.cpp @@ -50,7 +50,7 @@ alignas(256) constexpr std::array SampleCode{ 0x5a7a000d, 0x7e1c02ff, 0x5a7a000e, 0x7e1e02ff, 0x5a7a000f, 0x7e2002ff, 0x5a7a0010, 0x7e2202ff, 0x5a7a0011, 0x7e2402ff, 0x5a7a0012, 0x7e2602ff, 0x5a7a0013, 0x7e2802ff, 0x5a7a0014, 0x7e2a02ff, 0x5a7a0015, 0x7e2c02ff, 0x5a7a0016, 0x7e2e02ff, 0x5a7a0017, 0x7e3002ff, 0x5a7a0018, 0x7e3202ff, - 0x5a7a0019, 0xf0810108, 0x00820a04, 0xf1010108, 0x00820c04, 0x7e540280, 0xf03b0308, 0x0002112a, + 0x5a7a0019, 0xf0830108, 0x00820a04, 0xf1030108, 0x00820c04, 0x7e540280, 0xf03b0308, 0x0002112a, 0xf0910108, 0x00821404, 0x34500081, 0x7e520281, 0xf0030108, 0x00021628, 0x7e300280, 0xf0010108, 0x80021828, 0xbf8c3f70, 0xe0701200, 0x80000a03, 0xe0701204, 0x80000b03, 0xe0701208, 0x80000c03, 0xe070120c, 0x80000d03, 0xe0701210, 0x80000e03, 0xe0701214, 0x80000f03, 0xe0701218, 0x80001003, @@ -59,9 +59,7 @@ alignas(256) constexpr std::array SampleCode{ 0xe070123c, 0x80001903, 0xbf810000, }; -alignas(256) constexpr std::array, 5> RefusedCode{{ - {0xf0820108u, 0x00820a04u, 0xbf810000u}, - {0xf1020108u, 0x00820a04u, 0xbf810000u}, +alignas(256) constexpr std::array, 3> RefusedCode{{ {0xf1820308u, 0x00820a04u, 0xbf810000u}, {0xf0452108u, 0x00020a08u, 0xbf810000u}, {0xf0010108u, 0x0002ff08u, 0xbf810000u}, @@ -209,7 +207,7 @@ constexpr std::array, Threads> SampleEx std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(const void* data, std::uint32_t format, std::uint32_t width, std::uint32_t height) { @@ -282,7 +280,7 @@ void RunSample(AgcDriver::VulkanDevice& device) { } void CheckRefused(AgcDriver::VulkanDevice& device) { - constexpr std::array names{"image_sample lwe", "image_gather4 lwe", "image_get_lod lwe", "image_atomic_add tfe", "image_load tfe status past v255"}; + constexpr std::array names{"image_get_lod lwe", "image_atomic_add tfe", "image_load tfe status past v255"}; std::vector userData(16, 0u); for (std::uint32_t i = 0; i < RefusedCode.size(); ++i) { const std::span code(RefusedCode[i]); @@ -321,7 +319,7 @@ void CheckStorage() { } void CheckSample() { - constexpr std::array names{"image_sample tfe", "image_sample tfe status", "image_gather4 tfe x", "image_gather4 tfe y", "image_gather4 tfe z", "image_gather4 tfe w", "image_gather4 tfe status", "image_get_resinfo tfe lwe width", "image_get_resinfo tfe lwe height", "image_get_resinfo tfe lwe status", "image_sample_l tfe", "image_sample_l tfe status", "image_load tfe lwe", "image_load tfe lwe status", "image_load tfe d16", "image_load tfe d16 status"}; + constexpr std::array names{"image_sample tfe lwe", "image_sample tfe lwe status", "image_gather4 tfe lwe x", "image_gather4 tfe lwe y", "image_gather4 tfe lwe z", "image_gather4 tfe lwe w", "image_gather4 tfe lwe status", "image_get_resinfo tfe lwe width", "image_get_resinfo tfe lwe height", "image_get_resinfo tfe lwe status", "image_sample_l tfe", "image_sample_l tfe status", "image_load tfe lwe", "image_load tfe lwe status", "image_load tfe d16", "image_load tfe d16 status"}; for (std::uint32_t tid = 0; tid < Threads; ++tid) { for (std::uint32_t j = 0; j < SampleResults; ++j) { Expect(names[j], tid, Buffer[Threads * Inputs + tid * Results + j], SampleExpected[tid][j]); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp index a7d2ce68c..3aaf8d19f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp @@ -43,7 +43,7 @@ alignas(256) constexpr std::array Code{ std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::array TextureDescriptor(std::uint32_t tileMode) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/IntegerDot.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/IntegerDot.cpp index dd9af9662..3cf02f136 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/IntegerDot.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/IntegerDot.cpp @@ -53,7 +53,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationF16.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationF16.cpp new file mode 100644 index 000000000..6cfadd21b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationF16.cpp @@ -0,0 +1,308 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::PixelInput; +using ShaderRecompiler::PixelInputBit; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Sentinel = 0xdeadbeefu; +constexpr std::uint32_t Width = 192; +constexpr std::uint32_t Height = 128; +constexpr std::uint32_t ResultDwords = 12; +constexpr std::uint32_t Fp16Input = 0x03080000u; +constexpr std::uint32_t Fp16DefaultHighInput = 0x01580000u; +alignas(256) std::array Pixels{}; +alignas(256) std::array Results{}; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000u, 0x80000005u, 0xe0382010u, 0x80000605u, 0xbf8c3f70u, 0xf800020fu, 0x09080706u, 0xf80008cfu, 0x03020100u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PixelCode{ + 0x7e080f02u, 0x7e0a0f03u, 0xd5430006u, 0x0411ff05u, 0x000000c0u, 0x7e300300u, 0x7e320301u, 0xd742001au, + 0x00020000u, 0x7e3602ffu, 0xabcd0000u, 0xd75a001bu, 0x046a0200u, 0xd7420008u, 0x00020100u, 0x7e3802ffu, + 0x12340000u, 0xd75a001cu, 0x04220300u, 0x7e1e02ffu, 0x4500bc00u, 0xd743001du, 0x043e0140u, 0x7e3c0280u, + 0xd75a001eu, 0x04760340u, 0x7e3e0280u, 0xd75a801fu, 0x046a0200u, 0xd7420020u, 0x00020181u, 0xd7420021u, + 0x000201c1u, 0xd7420022u, 0x40020000u, 0xd7430023u, 0x043e0000u, 0xe0782000u, 0x80011806u, 0xe0782010u, + 0x80011c06u, 0xe0782020u, 0x80012006u, 0x7e0e02f2u, 0xf800180fu, 0x07070707u, 0xbf810000u, +}; + +constexpr std::array LowX{0x3400u, 0x3f00u, 0xb800u}; +constexpr std::array HighX{0x4200u, 0xc100u, 0x4700u}; +constexpr std::array LowY{0x3800u, 0x3a00u, 0x3000u}; +constexpr std::array HighY{0x6c00u, 0x3c00u, 0xc800u}; + +struct Vertex { + std::array position; + std::array parameter; +}; + +float HalfToFloat(std::uint32_t half) { + const auto sign = (half & 0x8000u) != 0u ? -1.0f : 1.0f; + const auto exponent = static_cast((half >> 10u) & 0x1fu); + const auto mantissa = static_cast(half & 0x3ffu); + if (exponent == 0) return sign * std::ldexp(mantissa, -24); + if (exponent == 31) return mantissa == 0.0f ? sign * INFINITY : NAN; + return sign * std::ldexp(1024.0f + mantissa, exponent - 25); +} + +std::uint32_t FloatToHalf(float value) { + const auto bits = std::bit_cast(value); + const auto sign = (bits >> 16u) & 0x8000u; + const auto magnitude = bits & 0x7fffffffu; + if (magnitude > 0x7f800000u) return sign | 0x7e00u; + if (magnitude >= 0x477ff000u) return sign | 0x7c00u; + if (magnitude < 0x38800000u) { + const auto scaled = std::fabs(value) * 16777216.0f; + return sign | static_cast(std::nearbyint(scaled)); + } + const auto rounded = magnitude + 0xfffu + ((magnitude >> 13u) & 1u); + return sign | ((rounded - 0x38000000u) >> 13u); +} + +float RoundToHalf(float value) { + return HalfToFloat(FloatToHalf(value)); +} + +std::uint32_t Pack(std::uint32_t low, std::uint32_t high) { + return low | (high << 16u); +} + +std::vector Triangles() { + std::vector vertices; + constexpr std::uint32_t columns = 6; + constexpr std::uint32_t rows = 4; + for (std::uint32_t cell = 0; cell < columns * rows; ++cell) { + const float width = 2.0f / columns; + const float height = 2.0f / rows; + const float x = -1.0f + width * static_cast(cell % columns); + const float y = -1.0f + height * static_cast(cell / columns); + const float skew = 0.07f * static_cast(cell % 5u); + const std::array, 3> corners{{ + {x + (0.05f + skew) * width, y + 0.05f * height}, + {x + 0.93f * width, y + (0.11f + skew) * height}, + {x + (0.37f - skew * 0.5f) * width, y + 0.91f * height}, + }}; + for (std::uint32_t vertex = 0; vertex < 3u; ++vertex) { + vertices.push_back({{corners[vertex][0], corners[vertex][1], 0.5f, 1.0f}, {Pack(LowX[vertex], HighX[vertex]), Pack(LowY[vertex], HighY[vertex]), 0x3c003c00u, 0x3c003c00u}}); + } + } + return vertices; +} + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +ShaderRecompiler::ShaderPixelStageInfo PixelStage(std::uint32_t waveSize, std::uint32_t input0) { + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.wave32 = waveSize == 32u; + pixel.interpolatorCount = 2; + pixel.interpolatorSettings[0] = input0; + pixel.interpolatorSettings[1] = Fp16DefaultHighInput; + pixel.inputAddr = PixelInputBit(PixelInput::PerspectiveCenter) | PixelInputBit(PixelInput::PositionX) | PixelInputBit(PixelInput::PositionY); + pixel.hasPerspectiveCenterVgpr = true; + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + return pixel; +} + +ShaderRecompiler::RecompileResult RecompilePixel(std::span code, std::uint32_t waveSize, std::uint32_t input0, const ShaderRecompiler::SpirvTarget& target, std::uint32_t pushOffset) { + std::vector pixelUserData(8, 0u); + const auto results = BufferDescriptor(Results.data(), ResultDwords * 4u, Width * Height); + std::copy(results.begin(), results.end(), pixelUserData.begin() + 4); + const std::array pixelMemory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(code.data()), code, 0, {}}, + {waveSize, 0, pixelUserData, std::nullopt, PixelStage(waveSize, input0), std::nullopt, pixelMemory}, + target, + {0, 0, pushOffset, 128 - pushOffset} + }; + fragment.useCache = false; + fragment.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; + return ShaderRecompiler::Recompile(fragment); +} + +void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize) { + static const auto triangles = Triangles(); + const auto target = device.Target(); + Pixels.fill(std::byte{0}); + Results.fill(Sentinel); + + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(triangles.data(), sizeof(Vertex), static_cast(triangles.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + const auto pixelResult = RecompilePixel(PixelCode, waveSize, Fp16Input, target, vertexPush); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, vertexPush} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(triangles.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); +} + +float Delta(const std::array& values, std::uint32_t vertex) { + return RoundToHalf(HalfToFloat(values[vertex]) - HalfToFloat(values[0])); +} + +float FirstStep(float delta, float coordinate, float origin) { + return std::fma(delta, coordinate, origin); +} + +double SecondStep(float delta, float coordinate, float firstStep) { + const double product = static_cast(delta) * static_cast(coordinate); + const double sum = product + firstStep; + const double productPart = sum - firstStep; + const double error = (product - productPart) + (firstStep - (sum - productPart)); + if (error == 0.0 || (std::bit_cast(sum) & 1u) != 0u) return sum; + return std::nextafter(sum, error > 0.0 ? INFINITY : -INFINITY); +} + +double Saturate(double value) { + return value > 0.0 ? std::min(value, 1.0) : 0.0; +} + +std::uint32_t HalfOf(double value) { + if (value == 0.0) return std::signbit(value) ? 0x8000u : 0u; + const int step = std::max(std::ilogb(value) - 10, -24); + const double rounded = std::ldexp(std::nearbyint(std::ldexp(value, -step)), step); + if (std::fabs(rounded) > 65504.0) return std::signbit(value) ? 0xfc00u : 0x7c00u; + if (rounded == 0.0) return std::signbit(value) ? 0x8000u : 0u; + return FloatToHalf(static_cast(rounded)); +} + +void Check(std::uint32_t waveSize) { + const auto what = "16-bit interpolation wave" + std::to_string(waveSize); + std::uint32_t covered = 0; + for (std::uint32_t pixel = 0; pixel < Width * Height; ++pixel) { + const auto* result = &Results[pixel * ResultDwords]; + if (std::to_integer(Pixels[pixel * 4u]) != 255u) { + Require(result[0] == Sentinel, what + ": uncovered pixel " + std::to_string(pixel) + " stored a result"); + continue; + } + ++covered; + const float i = std::bit_cast(result[0]); + const float j = std::bit_cast(result[1]); + const auto lowX = FirstStep(Delta(LowX, 1), i, HalfToFloat(LowX[0])); + const auto highX = FirstStep(Delta(HighX, 1), i, HalfToFloat(HighX[0])); + const auto highY = FirstStep(Delta(HighY, 1), i, HalfToFloat(0x4500u)); + const std::array expected{ + result[0], + result[1], + std::bit_cast(lowX), + 0xabcd0000u | HalfOf(SecondStep(Delta(LowX, 2), j, lowX)), + 0x12340000u | HalfOf(SecondStep(Delta(HighX, 2), j, highX)), + std::bit_cast(highY), + HalfOf(SecondStep(Delta(HighY, 2), j, highY)), + HalfOf(Saturate(SecondStep(Delta(LowX, 2), j, lowX))), + std::bit_cast(FirstStep(0.0f, i, 1.0f)), + std::bit_cast(FirstStep(0.0f, i, 0.0f)), + std::bit_cast(FirstStep(Delta(LowX, 1), -i, HalfToFloat(LowX[0]))), + std::bit_cast(FirstStep(Delta(LowX, 1), i, -1.0f)), + }; + for (std::uint32_t dword = 2; dword < ResultDwords; ++dword) { + char text[160]; + std::snprintf(text, sizeof(text), ": pixel %u dword %u is 0x%08x, expected 0x%08x (I %a, J %a)", pixel, dword, result[dword], expected[dword], static_cast(i), static_cast(j)); + Require(result[dword] == expected[dword], what + text); + } + } + Require(covered > 1000, what + ": the triangles covered only " + std::to_string(covered) + " pixels"); +} + +ShaderRecompiler::SpirvTarget OfflineTarget(bool barycentric) { + ShaderRecompiler::SpirvTarget target{}; + target.vulkanVersion = 0x00401000u; + target.spirvVersion = 0x00010300u; + target.subgroupSize = 64; + target.fragmentShaderBarycentricEnabled = barycentric; + return target; +} + +void ExpectFailure(std::span code, std::uint32_t input0, bool barycentric, const char* expected, const char* message) { + try { + static_cast(RecompilePixel(code, 64u, input0, OfflineTarget(barycentric), 0u)); + } catch (const std::exception& error) { + Require(std::string(error.what()).find(expected) != std::string::npos, std::string(message) + ": " + error.what()); + return; + } + throw std::runtime_error(message); +} + +void ExpectRejected(std::uint32_t word0, std::uint32_t word1, const char* expected, const char* message) { + alignas(256) const std::array code{word0, word1, 0xbf810000u}; + ExpectFailure(code, Fp16Input, true, expected, message); +} + +} + +int main() { + try { + ExpectFailure(PixelCode, Fp16Input, false, "requires fragmentShaderBarycentric", "16-bit interpolation was accepted without barycentrics"); + ExpectFailure(PixelCode, 0x03000000u, true, "without FP16_INTERP_MODE", "16-bit interpolation of a 32-bit input was accepted"); + ExpectFailure(PixelCode, 0x03080420u, true, "passes its vertices through unchanged", "16-bit interpolation of a pass-through input was accepted"); + ExpectRejected(0xd75a001bu, 0x0c6a0200u, "interpolation modifiers", "v_interp_p2_f16 accepted an output modifier"); + ExpectRejected(0xd75a081bu, 0x046a0200u, "interpolation modifiers", "v_interp_p2_f16 accepted op_sel"); + ExpectRejected(0xd742001au, 0x20020000u, "interpolation modifiers", "v_interp_p1ll_f16 accepted a negated attribute"); + ExpectRejected(0xd742001au, 0x00000000u, "not a vector register", "v_interp_p1ll_f16 accepted a scalar I coordinate"); + ExpectRejected(0xd743001du, 0x003e0140u, "not a vector register", "v_interp_p1lv_f16 accepted a scalar P0"); + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (!device->Target().fragmentShaderBarycentricEnabled) { + std::puts("skipped, the device has no fragmentShaderBarycentric"); + return VulkanTestSkipped; + } + for (const auto waveSize : {64u, 32u}) { + Draw(*device, waveSize); + Check(waveSize); + } + std::puts("16-bit interpolation tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationModes.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationModes.cpp new file mode 100644 index 000000000..cb363ba33 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationModes.cpp @@ -0,0 +1,171 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "SpirvBackend/SpirvEmitterInstructions.hpp" +#include "SpirvBackend/SpirvEmitterHelpers.hpp" +#include "IntermediateRepresentation/IrBuilder.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; +using AgcDriver::Graphics::Require; + +struct Row { + std::array input; + std::array, 8> expected; +}; + +constexpr std::array ModeNames{"IEEE=0 f32-flush f16-flush", "IEEE=0 f32-flush f16-keep", "IEEE=0 f32-keep f16-flush", "IEEE=0 f32-keep f16-keep", + "IEEE=1 f32-flush f16-flush", "IEEE=1 f32-flush f16-keep", "IEEE=1 f32-keep f16-flush", "IEEE=1 f32-keep f16-keep"}; +constexpr std::array ModeFlags{ + InterpolationFlush32 | InterpolationFlush16, InterpolationFlush32, InterpolationFlush16, 0u, + InterpolationQuiet | InterpolationFlush32 | InterpolationFlush16, InterpolationQuiet | InterpolationFlush32, InterpolationQuiet | InterpolationFlush16, InterpolationQuiet}; +constexpr std::array OpNames{"v_interp_p1ll_f16", "v_interp_p2_f16", "v_interp_p1lv_f16"}; + +constexpr std::array Rows{{ + {{0x33800000u, 0x7fa00000u, 0x43000000u, 0x33800000u, 0x4f000000u, 0xabcd0001u, 0x33800000u, 0x7fa00000u, 0x33800000u}, {{{0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x00005800u, 0x7fa00000u}, {0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x00005800u, 0x7fa00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x00005800u, 0x7fe00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x00005800u, 0x7fe00000u}}}}, + {{0xbf800000u, 0x00000001u, 0x7fa00000u, 0xbf800000u, 0x477ff000u, 0xabcd8200u, 0xbf800000u, 0x00000001u, 0xb8000000u}, {{{0x7fa00000u, 0x0000fc00u, 0x80000000u}, {0x7fa00000u, 0x0000fc00u, 0xb8000000u}, {0x7fa00000u, 0x0000fc00u, 0x80000001u}, {0x7fa00000u, 0x0000fc00u, 0xb8000000u}, {0x7fe00000u, 0x0000fc00u, 0x80000000u}, {0x7fe00000u, 0x0000fc00u, 0xb8000000u}, {0x7fe00000u, 0x0000fc00u, 0x80000001u}, {0x7fe00000u, 0x0000fc00u, 0xb8000000u}}}}, + {{0x33800000u, 0x7fa00000u, 0xff800000u, 0x33800000u, 0x7f7fffffu, 0xabcdbc00u, 0x33800000u, 0x7fa00000u, 0xbf800000u}, {{{0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x00007c00u, 0x7fa00000u}, {0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x00007c00u, 0x7fa00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x00007c00u, 0x7fe00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x00007c00u, 0x7fe00000u}}}}, + {{0xb8000000u, 0x7fa00000u, 0x80000000u, 0xb8000000u, 0x7f7fffffu, 0xabcdbc00u, 0xb8000000u, 0x7fa00000u, 0xbf800000u}, {{{0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x0000fc00u, 0x7fa00000u}, {0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x0000fc00u, 0x7fa00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x0000fc00u, 0x7fe00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x0000fc00u, 0x7fe00000u}}}}, + {{0x33800000u, 0xff800000u, 0xff800000u, 0x33800000u, 0x7f800000u, 0xabcd0000u, 0x33800000u, 0xff800000u, 0x00000000u}, {{{0xffc00000u, 0x0000fe00u, 0xffc00000u}, {0xff800000u, 0x00007c00u, 0xff800000u}, {0xffc00000u, 0x0000fe00u, 0xffc00000u}, {0xff800000u, 0x00007c00u, 0xff800000u}, {0xffc00000u, 0x0000fe00u, 0xffc00000u}, {0xff800000u, 0x00007c00u, 0xff800000u}, {0xffc00000u, 0x0000fe00u, 0xffc00000u}, {0xff800000u, 0x00007c00u, 0xff800000u}}}}, + {{0x33800000u, 0x7f800000u, 0xffc02000u, 0x33800000u, 0x47800000u, 0xabcd8000u, 0x33800000u, 0x7f800000u, 0x80000000u}, {{{0xffc00000u, 0x00008000u, 0xffc00000u}, {0xffc02000u, 0x00001c00u, 0x7f800000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xffc02000u, 0x00001c00u, 0x7f800000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xffc02000u, 0x00001c00u, 0x7f800000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xffc02000u, 0x00001c00u, 0x7f800000u}}}}, + {{0xb8000000u, 0x477ff000u, 0x00000000u, 0xb8000000u, 0x4f000000u, 0xabcd5800u, 0xb8000000u, 0x477ff000u, 0x43000000u}, {{{0x00000000u, 0x00008000u, 0x43000000u}, {0xbffff000u, 0x0000fc00u, 0x42fc0040u}, {0x00000000u, 0x00008000u, 0x43000000u}, {0xbffff000u, 0x0000fc00u, 0x42fc0040u}, {0x00000000u, 0x00008000u, 0x43000000u}, {0xbffff000u, 0x0000fc00u, 0x42fc0040u}, {0x00000000u, 0x00008000u, 0x43000000u}, {0xbffff000u, 0x0000fc00u, 0x42fc0040u}}}}, + {{0x33800000u, 0x7fa00000u, 0x477fe000u, 0x33800000u, 0x47800000u, 0xabcd7d00u, 0x33800000u, 0x7fa00000u, 0x7fa00000u}, {{{0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x00001c00u, 0x7fa00000u}, {0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x00001c00u, 0x7fa00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x00001c00u, 0x7fe00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x00001c00u, 0x7fe00000u}}}}, + {{0xb8000000u, 0x47800000u, 0x7fa00000u, 0xb8000000u, 0x7f7fffffu, 0xabcd7bffu, 0xb8000000u, 0x47800000u, 0x477fe000u}, {{{0x7fa00000u, 0x00008000u, 0x477fe000u}, {0x7fa00000u, 0x0000fc00u, 0x477fde00u}, {0x7fa00000u, 0x00008000u, 0x477fe000u}, {0x7fa00000u, 0x0000fc00u, 0x477fde00u}, {0x7fe00000u, 0x00008000u, 0x477fe000u}, {0x7fe00000u, 0x0000fc00u, 0x477fde00u}, {0x7fe00000u, 0x00008000u, 0x477fe000u}, {0x7fe00000u, 0x0000fc00u, 0x477fde00u}}}}, + {{0x33800000u, 0xff800000u, 0xbf800000u, 0x33800000u, 0x3f800000u, 0xabcd7bffu, 0x33800000u, 0xff800000u, 0x477fe000u}, {{{0xffc00000u, 0x00008000u, 0xffc00000u}, {0xff800000u, 0x00000001u, 0xff800000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xff800000u, 0x00000001u, 0xff800000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xff800000u, 0x00000001u, 0xff800000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xff800000u, 0x00000001u, 0xff800000u}}}}, + {{0xb8000000u, 0x477ff000u, 0x33800000u, 0xb8000000u, 0x3f800000u, 0xabcd8200u, 0xb8000000u, 0x477ff000u, 0xb8000000u}, {{{0x00000000u, 0x00008000u, 0x80000000u}, {0xbffff000u, 0x00008200u, 0xbffff100u}, {0x00000000u, 0x00008000u, 0x80000000u}, {0xbffff000u, 0x00008200u, 0xbffff100u}, {0x00000000u, 0x00008000u, 0x80000000u}, {0xbffff000u, 0x00008200u, 0xbffff100u}, {0x00000000u, 0x00008000u, 0x80000000u}, {0xbffff000u, 0x00008200u, 0xbffff100u}}}}, + {{0x33800000u, 0x7fa00000u, 0xb8000000u, 0x33800000u, 0xff800000u, 0xabcd7bffu, 0x33800000u, 0x7fa00000u, 0x477fe000u}, {{{0x7fa00000u, 0x0000fe00u, 0x7fa00000u}, {0x7fa00000u, 0x0000fc00u, 0x7fa00000u}, {0x7fa00000u, 0x0000fe00u, 0x7fa00000u}, {0x7fa00000u, 0x0000fc00u, 0x7fa00000u}, {0x7fe00000u, 0x0000fe00u, 0x7fe00000u}, {0x7fe00000u, 0x0000fc00u, 0x7fe00000u}, {0x7fe00000u, 0x0000fe00u, 0x7fe00000u}, {0x7fe00000u, 0x0000fc00u, 0x7fe00000u}}}}, + {{0xff800000u, 0x00000001u, 0x43000000u, 0xff800000u, 0x47800000u, 0xabcd7d00u, 0xff800000u, 0x00000001u, 0x7fa00000u}, {{{0xffc00000u, 0x0000fc00u, 0xffc00000u}, {0xffc00000u, 0x0000fc00u, 0xffc00000u}, {0xff800000u, 0x0000fc00u, 0x7fa00000u}, {0xff800000u, 0x0000fc00u, 0x7fa00000u}, {0xffc00000u, 0x0000fc00u, 0xffc00000u}, {0xffc00000u, 0x0000fc00u, 0xffc00000u}, {0xff800000u, 0x0000fc00u, 0x7fe00000u}, {0xff800000u, 0x0000fc00u, 0x7fe00000u}}}}, + {{0x7f800000u, 0x7fa00000u, 0x33800000u, 0x7f800000u, 0x80400000u, 0xabcd3c00u, 0x7f800000u, 0x7fa00000u, 0x3f800000u}, {{{0x7fa00000u, 0x0000fe00u, 0x7fa00000u}, {0x7fa00000u, 0x0000fe00u, 0x7fa00000u}, {0x7fa00000u, 0x0000fc00u, 0x7fa00000u}, {0x7fa00000u, 0x0000fc00u, 0x7fa00000u}, {0x7fe00000u, 0x0000fe00u, 0x7fe00000u}, {0x7fe00000u, 0x0000fe00u, 0x7fe00000u}, {0x7fe00000u, 0x0000fc00u, 0x7fe00000u}, {0x7fe00000u, 0x0000fc00u, 0x7fe00000u}}}}, + {{0x3f800000u, 0x00000001u, 0x7fa00000u, 0x3f800000u, 0xff800000u, 0xabcd0001u, 0x3f800000u, 0x00000001u, 0x33800000u}, {{{0x7fa00000u, 0x0000fc00u, 0x00000000u}, {0x7fa00000u, 0x0000fc00u, 0x33800000u}, {0x7fa00000u, 0x0000fc00u, 0x00000001u}, {0x7fa00000u, 0x0000fc00u, 0x33800000u}, {0x7fe00000u, 0x0000fc00u, 0x00000000u}, {0x7fe00000u, 0x0000fc00u, 0x33800000u}, {0x7fe00000u, 0x0000fc00u, 0x00000001u}, {0x7fe00000u, 0x0000fc00u, 0x33800000u}}}}, + {{0xb8000000u, 0x7fa00000u, 0x33800000u, 0xb8000000u, 0x4f000000u, 0xabcd7e00u, 0xb8000000u, 0x7fa00000u, 0x7fc00000u}, {{{0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x0000fc00u, 0x7fa00000u}, {0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x0000fc00u, 0x7fa00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x0000fc00u, 0x7fe00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x0000fc00u, 0x7fe00000u}}}}, + {{0xff800000u, 0x80400000u, 0x43000000u, 0xff800000u, 0x477ff000u, 0xabcd7d00u, 0xff800000u, 0x80400000u, 0x7fa00000u}, {{{0xffc00000u, 0x0000fc00u, 0xffc00000u}, {0xffc00000u, 0x0000fc00u, 0xffc00000u}, {0x7f800000u, 0x0000fc00u, 0x7fa00000u}, {0x7f800000u, 0x0000fc00u, 0x7fa00000u}, {0xffc00000u, 0x0000fc00u, 0xffc00000u}, {0xffc00000u, 0x0000fc00u, 0xffc00000u}, {0x7f800000u, 0x0000fc00u, 0x7fe00000u}, {0x7f800000u, 0x0000fc00u, 0x7fe00000u}}}}, + {{0x33800000u, 0x4f000000u, 0x3f800000u, 0x33800000u, 0x47800000u, 0xabcdbc00u, 0x33800000u, 0x4f000000u, 0xbf800000u}, {{{0x3f800000u, 0x00008000u, 0xbf800000u}, {0x43010000u, 0x00001c00u, 0x42fe0000u}, {0x3f800000u, 0x00008000u, 0xbf800000u}, {0x43010000u, 0x00001c00u, 0x42fe0000u}, {0x3f800000u, 0x00008000u, 0xbf800000u}, {0x43010000u, 0x00001c00u, 0x42fe0000u}, {0x3f800000u, 0x00008000u, 0xbf800000u}, {0x43010000u, 0x00001c00u, 0x42fe0000u}}}}, + {{0x7f800000u, 0x00000001u, 0x3f800000u, 0x7f800000u, 0x80400000u, 0xabcd5800u, 0x7f800000u, 0x00000001u, 0x43000000u}, {{{0xffc00000u, 0x0000fe00u, 0xffc00000u}, {0xffc00000u, 0x0000fe00u, 0xffc00000u}, {0x7f800000u, 0x0000fc00u, 0x7f800000u}, {0x7f800000u, 0x0000fc00u, 0x7f800000u}, {0xffc00000u, 0x0000fe00u, 0xffc00000u}, {0xffc00000u, 0x0000fe00u, 0xffc00000u}, {0x7f800000u, 0x0000fc00u, 0x7f800000u}, {0x7f800000u, 0x0000fc00u, 0x7f800000u}}}}, + {{0x33800000u, 0x7fa00000u, 0x7fc00000u, 0x33800000u, 0x3f800000u, 0xabcd7d00u, 0x33800000u, 0x7fa00000u, 0x7fa00000u}, {{{0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x00000001u, 0x7fa00000u}, {0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x00000001u, 0x7fa00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x00000001u, 0x7fe00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x00000001u, 0x7fe00000u}}}}, + {{0xb8000000u, 0x7fa00000u, 0x477fe000u, 0xb8000000u, 0x3f800000u, 0xabcd0000u, 0xb8000000u, 0x7fa00000u, 0x00000000u}, {{{0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x00008200u, 0x7fa00000u}, {0x7fa00000u, 0x00008000u, 0x7fa00000u}, {0x7fa00000u, 0x00008200u, 0x7fa00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x00008200u, 0x7fe00000u}, {0x7fe00000u, 0x00008000u, 0x7fe00000u}, {0x7fe00000u, 0x00008200u, 0x7fe00000u}}}}, + {{0x33800000u, 0x4f000000u, 0x7fa00000u, 0x33800000u, 0x477ff000u, 0xabcd8000u, 0x33800000u, 0x4f000000u, 0x80000000u}, {{{0x7fa00000u, 0x00008000u, 0x00000000u}, {0x7fa00000u, 0x00001bffu, 0x43000000u}, {0x7fa00000u, 0x00008000u, 0x00000000u}, {0x7fa00000u, 0x00001bffu, 0x43000000u}, {0x7fe00000u, 0x00008000u, 0x00000000u}, {0x7fe00000u, 0x00001bffu, 0x43000000u}, {0x7fe00000u, 0x00008000u, 0x00000000u}, {0x7fe00000u, 0x00001bffu, 0x43000000u}}}}, + {{0xb8000000u, 0x7f800000u, 0x00000000u, 0xb8000000u, 0x3f800000u, 0xabcd7bffu, 0xb8000000u, 0x7f800000u, 0x477fe000u}, {{{0xffc00000u, 0x00008000u, 0xffc00000u}, {0xff800000u, 0x00008200u, 0xff800000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xff800000u, 0x00008200u, 0xff800000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xff800000u, 0x00008200u, 0xff800000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xff800000u, 0x00008200u, 0xff800000u}}}}, + {{0xb8000000u, 0x477ff000u, 0x33800000u, 0xb8000000u, 0xff800000u, 0xabcd0001u, 0xb8000000u, 0x477ff000u, 0x33800000u}, {{{0x00000000u, 0x0000fe00u, 0x00000000u}, {0xbffff000u, 0x00007c00u, 0xbffff000u}, {0x00000000u, 0x0000fe00u, 0x00000000u}, {0xbffff000u, 0x00007c00u, 0xbffff000u}, {0x00000000u, 0x0000fe00u, 0x00000000u}, {0xbffff000u, 0x00007c00u, 0xbffff000u}, {0x00000000u, 0x0000fe00u, 0x00000000u}, {0xbffff000u, 0x00007c00u, 0xbffff000u}}}}, + {{0xb8000000u, 0x00000000u, 0x7fc00000u, 0xb8000000u, 0x7f7fffffu, 0xabcd8200u, 0xb8000000u, 0x00000000u, 0xb8000000u}, {{{0x7fc00000u, 0x00008000u, 0x80000000u}, {0x7fc00000u, 0x0000fc00u, 0xb8000000u}, {0x7fc00000u, 0x00008000u, 0x80000000u}, {0x7fc00000u, 0x0000fc00u, 0xb8000000u}, {0x7fc00000u, 0x00008000u, 0x80000000u}, {0x7fc00000u, 0x0000fc00u, 0xb8000000u}, {0x7fc00000u, 0x00008000u, 0x80000000u}, {0x7fc00000u, 0x0000fc00u, 0xb8000000u}}}}, + {{0xffc02000u, 0x00000001u, 0x7f800000u, 0xffc02000u, 0x7fa00000u, 0xabcd0000u, 0xffc02000u, 0x00000001u, 0x00000000u}, {{{0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}}}}, + {{0xffc02000u, 0x80400000u, 0x3f800000u, 0xffc02000u, 0x47800000u, 0xabcdfe01u, 0xffc02000u, 0x80400000u, 0xffc02000u}, {{{0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}}}}, + {{0x7fa00000u, 0x4f000000u, 0x3f800000u, 0x7fa00000u, 0x7fc00000u, 0xabcdfc00u, 0x7fa00000u, 0x4f000000u, 0xff800000u}, {{{0x7fa00000u, 0x00007f00u, 0x7fa00000u}, {0x7fa00000u, 0x00007f00u, 0x7fa00000u}, {0x7fa00000u, 0x00007f00u, 0x7fa00000u}, {0x7fa00000u, 0x00007f00u, 0x7fa00000u}, {0x7fe00000u, 0x00007f00u, 0x7fe00000u}, {0x7fe00000u, 0x00007f00u, 0x7fe00000u}, {0x7fe00000u, 0x00007f00u, 0x7fe00000u}, {0x7fe00000u, 0x00007f00u, 0x7fe00000u}}}}, + {{0x7f800000u, 0x00000001u, 0xff800000u, 0x7f800000u, 0x477ff000u, 0xabcd7e00u, 0x7f800000u, 0x00000001u, 0x7fc00000u}, {{{0xffc00000u, 0x00007c00u, 0xffc00000u}, {0xffc00000u, 0x00007c00u, 0xffc00000u}, {0xffc00000u, 0x00007c00u, 0x7fc00000u}, {0xffc00000u, 0x00007c00u, 0x7fc00000u}, {0xffc00000u, 0x00007c00u, 0xffc00000u}, {0xffc00000u, 0x00007c00u, 0xffc00000u}, {0xffc00000u, 0x00007c00u, 0x7fc00000u}, {0xffc00000u, 0x00007c00u, 0x7fc00000u}}}}, + {{0xffc02000u, 0x80400000u, 0xb8000000u, 0xffc02000u, 0x00000000u, 0xabcd7c00u, 0xffc02000u, 0x80400000u, 0x7f800000u}, {{{0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}, {0xffc02000u, 0x0000fe01u, 0xffc02000u}}}}, + {{0x7f800000u, 0xffc00001u, 0x7fa00000u, 0x7f800000u, 0xff800000u, 0xabcd0000u, 0x7f800000u, 0xffc00001u, 0x00000000u}, {{{0xffc00001u, 0x0000fc00u, 0xffc00001u}, {0xffc00001u, 0x0000fc00u, 0xffc00001u}, {0xffc00001u, 0x0000fc00u, 0xffc00001u}, {0xffc00001u, 0x0000fc00u, 0xffc00001u}, {0xffc00001u, 0x0000fc00u, 0xffc00001u}, {0xffc00001u, 0x0000fc00u, 0xffc00001u}, {0xffc00001u, 0x0000fc00u, 0xffc00001u}, {0xffc00001u, 0x0000fc00u, 0xffc00001u}}}}, + {{0x80000000u, 0x7f800000u, 0x43000000u, 0x80000000u, 0x3f800000u, 0xabcd0000u, 0x80000000u, 0x7f800000u, 0x00000000u}, {{{0xffc00000u, 0x00008000u, 0xffc00000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}, {0xffc00000u, 0x00008000u, 0xffc00000u}}}}, +}}; + +constexpr std::uint32_t Words = 12; +constexpr std::uint32_t Sentinel = 0xdeadbeefu; +alignas(256) std::array Data{}; + +RecompileResult Compile(std::uint32_t mode) { + IrProgram program; + const ShaderStageInputInfo inputs{}; + SpirvEmitterState state(program, inputs); + auto& module = state.module; + module.EmitCapability(spv::CapabilityShader); + module.EmitCapability(spv::CapabilityFloat64); + module.EmitCapability(spv::CapabilitySignedZeroInfNanPreserve); + module.EmitExtension("SPV_KHR_float_controls"); + module.AddMemoryModel(spv::AddressingModelLogical, spv::MemoryModelGLSL450); + const auto u32 = TypeU32(state); + const auto array = module.DecoratedType(spv::OpTypeRuntimeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, 4u}}}, u32); + const auto structure = module.Type(spv::OpTypeStruct, array); + module.AddAnnotation(spv::OpDecorate, structure, spv::DecorationBlock); + module.AddAnnotation(spv::OpMemberDecorate, structure, 0u, spv::DecorationOffset, 0u); + const auto buffer = module.DefineGlobalVariable(module.Type(spv::OpTypePointer, spv::StorageClassStorageBuffer, structure), spv::StorageClassStorageBuffer); + module.AddAnnotation(spv::OpDecorate, buffer, spv::DecorationDescriptorSet, 0u); + module.AddAnnotation(spv::OpDecorate, buffer, spv::DecorationBinding, 0u); + const auto invocation = module.DefineGlobalVariable(TypePointer(state, spv::StorageClassInput, TypeU32Vector(state, 3u)), spv::StorageClassInput); + module.AddAnnotation(spv::OpDecorate, invocation, spv::DecorationBuiltIn, spv::BuiltInGlobalInvocationId); + const auto element = module.Type(spv::OpTypePointer, spv::StorageClassStorageBuffer, u32); + const auto main = module.AllocateId(); + const auto voidType = module.Type(spv::OpTypeVoid); + module.EmitEntryPoint(spv::ExecutionModelGLCompute, main, "main", {invocation}); + module.AddExecutionMode(main, spv::ExecutionModeLocalSize, static_cast(Rows.size()), 1u, 1u); + for (const auto width : {32u, 64u}) module.AddExecutionMode(main, spv::ExecutionModeSignedZeroInfNanPreserve, width); + module.AddFunction(spv::OpFunction, voidType, main, spv::FunctionControlMaskNone, module.Type(spv::OpTypeFunction, voidType)); + module.AddFunction(spv::OpLabel, module.AllocateId()); + const auto id = module.AllocateId(); + module.AddFunction(spv::OpLoad, TypeU32Vector(state, 3u), id, invocation); + const auto lane = module.AllocateId(); + module.AddFunction(spv::OpCompositeExtract, u32, lane, id, 0u); + const auto rowBase = Binary(state, spv::OpIMul, u32, lane, ConstantU32(state, Words)); + const auto address = [&](std::uint32_t offset) { + const auto pointer = module.AllocateId(); + module.AddFunction(spv::OpAccessChain, element, pointer, buffer, ConstantU32(state, 0u), Binary(state, spv::OpIAdd, u32, rowBase, ConstantU32(state, offset))); + return pointer; + }; + const auto load = [&](std::uint32_t offset) { + const auto word = module.AllocateId(); + module.AddFunction(spv::OpLoad, u32, word, address(offset)); + return Unary(state, spv::OpBitcast, TypeF32(state), word); + }; + IrProgram constants; + IrBuilder builder(constants); + builder.SetInsertionPoint(constants.CreateBlock()); + const auto* flags = &builder.Constant(mode); + for (std::uint32_t op = 0; op < 3u; ++op) { + const auto delta = load(op * 3u); + const auto coordinate = load(op * 3u + 1u); + const auto base = load(op * 3u + 2u); + const auto value = op == 1u ? EmitFPInterpolateF16(state, delta, coordinate, base, flags) : EmitFPInterpolateF32(state, delta, coordinate, base, flags); + const auto result = op == 1u ? EmitConvertF16F32(state, value) : Unary(state, spv::OpBitcast, u32, value); + module.AddFunction(spv::OpStore, address(9u + op), result); + } + module.AddFunction(spv::OpReturn); + module.AddFunction(spv::OpFunctionEnd); + RecompileResult result{}; + result.spirv = module.Finalize(); + const auto base = reinterpret_cast(Data.data()); + result.bindings.push_back({DescriptorKind::StorageBuffer, DescriptorRole::GuestBuffers, 0u, 0u, 1u, + {static_cast(base), static_cast((base >> 32u) & 0xffffu), static_cast(sizeof(Data)), 0x31016facu}, false}); + return result; +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t mode) { + Data.fill(Sentinel); + for (std::uint32_t row = 0; row < Rows.size(); ++row) std::copy(Rows[row].input.begin(), Rows[row].input.end(), Data.begin() + row * Words); + device.Dispatch(Compile(ModeFlags[mode]), 1u, 1u, 1u); + device.WaitIdle(); + for (std::uint32_t row = 0; row < Rows.size(); ++row) { + for (std::uint32_t input = 0; input < 9u; ++input) Require(Data[row * Words + input] == Rows[row].input[input], "interpolation modes changed an input"); + for (std::uint32_t op = 0; op < 3u; ++op) { + const auto actual = Data[row * Words + 9u + op]; + const auto expected = Rows[row].expected[mode][op]; + char message[200]; + std::snprintf(message, sizeof(message), "%s, %s, row %u: got 0x%08x, expected 0x%08x", OpNames[op], ModeNames[mode], row, actual, expected); + Require(actual == expected, message); + } + } + for (std::size_t index = Rows.size() * Words; index < Data.size(); ++index) Require(Data[index] == Sentinel, "interpolation modes wrote beyond the rows"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const auto target = device->Target(); + if (std::find(target.supportedCapabilities.begin(), target.supportedCapabilities.end(), spv::CapabilityFloat64) == target.supportedCapabilities.end()) { + std::puts("skipped, interpolation modes require Float64"); + return VulkanTestSkipped; + } + for (std::uint32_t mode = 0; mode < ModeFlags.size(); ++mode) Run(*device, mode); + std::puts("interpolation modes passed: 32 hardware rows, 3 ops, 8 float modes"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsAddtid.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsAddtid.cpp index 8d00ecc2d..b7e506135 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LdsAddtid.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsAddtid.cpp @@ -290,7 +290,7 @@ constexpr const char* Names64[Checked] = { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics.cpp index fe681f6ba..d9b8c358b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics.cpp @@ -69,7 +69,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics64.cpp index 5d9e2bc7e..3ca9e67f2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics64.cpp @@ -124,7 +124,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint64_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsCondxchg.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsCondxchg.cpp new file mode 100644 index 000000000..df89a989d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsCondxchg.cpp @@ -0,0 +1,176 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t MaxThreads = 64; +constexpr std::uint32_t Words = 8; +constexpr std::uint32_t LdsDwords = 4096; +constexpr std::uint32_t Sign = 0x80000000u; +constexpr std::uint32_t Sentinel = 0xdeadbeefu; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +auto Code(std::uint32_t waveSize, std::uint32_t offset, std::uint32_t destination, bool inactive, bool contended) { + const std::uint32_t saveExec = waveSize == 32u ? 0xbe943c6au : 0xbe94246au; + const std::uint32_t restoreExec = waveSize == 32u ? 0xbefe03c1u : 0xbefe04c1u; + return std::to_array({ + 0x34020085u, 0xe0381000u, 0x80000201u, 0xe0381010u, 0x80000601u, 0xbf8c3f70u, + 0x7e160304u, 0x7e180305u, 0x7e1a02ffu, Sentinel, 0x7e140280u, + contended ? 0x7d840080u : 0x7d841480u, saveExec, + 0xd9340000u, 0x00000206u, 0xd9340008u, 0x00000806u, 0xd9340010u, 0x00000806u, + 0xbf8cc07fu, restoreExec, 0xbf8a0000u, + inactive ? 0x36140081u : 0x36140080u, 0x7d841480u, saveExec, + 0xd9f80000u | offset, (destination << 24u) | 0x00000b07u, + 0xbf8cc07fu, restoreExec, 0xbf8a0000u, + 0x7e280300u | destination, 0x7e2a0300u | (destination + 1u), + 0xd9d80000u, 0x16000006u, 0xd9d80008u, 0x18000006u, 0xd9d80010u, 0x1a000006u, + 0xbf8cc07fu, 0xe0781000u, 0x80011401u, 0xe0781010u, 0x80011801u, 0xbf810000u, + }); +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +template +void Translate(std::span code, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target, TUse&& use) { + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{waveSize, 1, 1}, LdsDwords, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory}, + target, + {0, 0, 0, 128} + }; + request.useCache = false; + use(ShaderRecompiler::Recompile(request)); +} + +void Expect(std::uint32_t tid, std::uint32_t index, std::uint32_t expected) { + const auto actual = Output[tid * Words + index]; + Require(actual == expected, "lds condxchg: lane " + std::to_string(tid) + " word " + std::to_string(index) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected)); +} + +template +void ExpectError(TCheck&& check, const std::string& reason) { + std::string error; + try { + check(); + } catch (const std::exception& exception) { + error = exception.what(); + } + Require(error.find(reason) != std::string::npos, "lds condxchg: expected \"" + reason + "\", got \"" + error + "\""); +} + +void CheckRejections(AgcDriver::VulkanDevice& device) { + const auto translate = [&](std::span code) { Translate(code, 32u, device.Target(), [](const auto&) {}); }; + constexpr std::array boundary{0xd9f80000u, 0xfe00fe00u, 0xbf810000u}; + translate(boundary); + ExpectError([&] { translate(std::span(boundary).first(1u)); }, "truncated DS instruction"); + constexpr std::array sourceOverflow{0xd9f80000u, 0x0200ff00u, 0xbf810000u}; + constexpr std::array destinationOverflow{0xd9f80000u, 0xff000400u, 0xbf810000u}; + constexpr std::array gds{0xd9fa0000u, 0x02000400u, 0xbf810000u}; + ExpectError([&] { translate(sourceOverflow); }, "DS conditional exchange source register range overflow"); + ExpectError([&] { translate(destinationOverflow); }, "DS conditional exchange destination register range overflow"); + ExpectError([&] { translate(gds); }, "DS conditional exchange GDS mode is not supported"); +} + +void Fill(std::uint32_t threads, std::uint32_t offset, bool contended) { + for (std::uint32_t tid = 0u; tid < threads; ++tid) { + auto* in = &Input[tid * Words]; + const auto seed = (tid + 1u) * 0x9e3779b97f4a7c15ull; + in[0] = contended ? 0xffffffffu : static_cast(seed); + in[1] = contended ? 0xfffffffeu : static_cast(seed >> 32u); + in[2] = contended ? Sign | (tid + 1u) : ((tid & 1u) != 0u ? Sign : 0u) | ((tid / 4u) % 2u != 0u ? 0x7fffffffu : 0u); + in[3] = contended ? Sign | (tid + 1u) : ((tid & 2u) != 0u ? Sign : 0u) | ((tid / 8u) % 2u != 0u ? 0x13579bdfu : 0u); + const bool outOfRange = !contended && ((tid & 8u) == 0u || offset >= LdsDwords * 4u); + in[4] = contended ? 0u : outOfRange ? tid * 32u : LdsDwords * 4u - 24u - tid * 32u; + in[5] = in[4] + (outOfRange ? 0x10000u : 0u) - offset; + in[6] = 0xa5a5a5a5u; + in[7] = 0x5a5a5a5au; + } +} + +void CheckContention(std::uint32_t threads) { + for (std::uint32_t word = 0u; word < 2u; ++word) { + const auto finalValue = Output[2u + word]; + Require(finalValue >= 1u && finalValue <= threads, "lds condxchg: invalid final contended dword"); + std::array seen{}; + for (std::uint32_t tid = 0u; tid < threads; ++tid) { + Expect(tid, 2u + word, finalValue); + const auto old = Output[tid * Words + word]; + const auto index = old == Input[word] ? 0u : old; + Require(index <= threads && index != tid + 1u && !seen[index], "lds condxchg: invalid or duplicate returned contended dword"); + seen[index] = true; + } + Require(seen[0] && !seen[finalValue], "lds condxchg: contended dword returns lost an exchange"); + } +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target, std::uint32_t offset, std::uint32_t destination, bool inactive, bool contended) { + Fill(waveSize, offset, contended); + Output.fill(Sentinel); + alignas(256) const auto code = Code(waveSize, offset, destination, inactive, contended); + Translate(code, waveSize, target, [&](const auto& result) { + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + }); + for (std::uint32_t tid = 0u; tid < waveSize; ++tid) { + const auto* in = &Input[tid * Words]; + if (!contended) { + const bool active = !inactive || tid % 2u == 0u; + const bool inBounds = in[5] + offset < LdsDwords * 4u; + Expect(tid, 0u, active ? inBounds ? in[0] : 0u : destination == 12u ? in[3] : in[5]); + Expect(tid, 1u, active ? inBounds ? in[1] : 0u : destination == 12u ? Sentinel : in[6]); + Expect(tid, 2u, active && inBounds && (in[2] & Sign) != 0u ? in[2] & ~Sign : in[0]); + Expect(tid, 3u, active && inBounds && (in[3] & Sign) != 0u ? in[3] & ~Sign : in[1]); + } + for (std::uint32_t index = 4u; index < Words; ++index) Expect(tid, index, in[6u + index % 2u]); + } + if (contended) CheckContention(waveSize); +} + +void CheckExecution(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target) { + for (const auto offset : {0u, 0x120u, 0xfff8u}) Run(device, waveSize, target, offset, 12u, false, false); + Run(device, waveSize, target, 0x120u, 12u, true, false); + Run(device, waveSize, target, 0xfff8u, 7u, true, false); + Run(device, waveSize, target, 0x120u, 12u, false, true); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + CheckRejections(*device); + CheckExecution(*device, 32u, device->Target()); + CheckExecution(*device, 64u, device->Target()); + CheckExecution(*device, 64u, device->ComputeTarget(32u)); + std::puts("lds condxchg tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsIntegerAtomics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsIntegerAtomics.cpp index 28be61149..e79c87bde 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LdsIntegerAtomics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsIntegerAtomics.cpp @@ -286,7 +286,7 @@ constexpr const char* Names64[Checked] = { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsReadWrite.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsReadWrite.cpp index 98002a448..098ccb3ef 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LdsReadWrite.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsReadWrite.cpp @@ -322,7 +322,7 @@ constexpr const char* Names64[Checked] = { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsSrc2.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsSrc2.cpp new file mode 100644 index 000000000..56eb1d256 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsSrc2.cpp @@ -0,0 +1,160 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084, 0xe0381000, 0x80000201, 0x340c0082, 0xbf8c3f70, 0x4a0e0cff, 0x00001000, 0x4a100c80, + 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, 0xda007c00, 0x00000007, 0xd8d80000, 0x0a000007, + 0x4a0e0cff, 0x00001080, 0x4a100cff, 0x00002080, 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, + 0xda040400, 0x00000007, 0xd8d80000, 0x0b000007, 0x4a0e0cff, 0x00001100, 0x4a100cff, 0x00000100, + 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, 0xda087c00, 0x00000007, 0xd8d80000, 0x0c000007, + 0x4a0e0cff, 0x00001180, 0x4a100cff, 0x00002180, 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, + 0xda0c0400, 0x00000007, 0xd8d80000, 0x0d000007, 0x4a0e0cff, 0x00001200, 0x4a100cff, 0x00000200, + 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, 0xda107c00, 0x00000007, 0xd8d80000, 0x0e000007, + 0x4a0e0cff, 0x00001280, 0x4a100cff, 0x00002280, 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, + 0xda140400, 0x00000007, 0xd8d80000, 0x0f000007, 0x4a0e0cff, 0x00001300, 0x4a100cff, 0x00000300, + 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, 0xda187c00, 0x00000007, 0xd8d80000, 0x10000007, + 0x4a0e0cff, 0x00001380, 0x4a100cff, 0x00002380, 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, + 0xda1c0400, 0x00000007, 0xd8d80000, 0x11000007, 0x4a0e0cff, 0x00001400, 0x4a100cff, 0x00000400, + 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, 0xda207c00, 0x00000007, 0xd8d80000, 0x12000007, + 0x4a0e0cff, 0x00001480, 0x4a100cff, 0x00002480, 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, + 0xda240400, 0x00000007, 0xd8d80000, 0x13000007, 0x4a0e0cff, 0x00001500, 0x4a100cff, 0x00000500, + 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, 0xda287c00, 0x00000007, 0xd8d80000, 0x14000007, + 0x4a0e0cff, 0x00001580, 0x4a100cff, 0x00002580, 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, + 0xda2c0400, 0x00000007, 0xd8d80000, 0x15000007, 0x4a0e0cff, 0x00001600, 0x4a100cff, 0x00000600, + 0xd8340000, 0x00000207, 0xd8340000, 0x00000308, 0xda347c00, 0x00000007, 0xd8d80000, 0x16000007, + 0x34020086, 0xbf8cc07f, 0xe0781000, 0x80010a01, 0xe0781010, 0x80010e01, 0xe0781020, 0x80011201, + 0xe0781030, 0x80011601, 0xbf810000, +}; + +alignas(256) constexpr std::array AddressOffsetCode{0xda008000, 0x00000000, 0xbf810000}; + +void Fill(std::uint32_t tid, std::uint32_t* words) { + const std::uint64_t seed = (tid + 1u) * 0x9e3779b97f4a7c15ull; + const auto mixed = seed * 0xbf58476d1ce4e5b9ull; + words[0] = tid % 4u == 0u ? tid % 7u : static_cast(seed); + words[1] = tid % 4u == 0u ? (tid * 3u) % 7u : tid % 4u == 1u ? words[0] : static_cast(mixed >> 32u); + words[2] = std::bit_cast(static_cast(static_cast(tid % 9u) - 4) * 1.75f + 0.5f); + words[3] = std::bit_cast(static_cast(static_cast((tid * 5u) % 11u) - 5) * 0.625f + 0.25f); +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span code) { + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 10240u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + return ShaderRecompiler::Recompile(request); +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + const auto result = Compile(device, Code); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(Code.data())); + device.WaitIdle(); +} + +void CheckLanes() { + constexpr std::array names{ + "ds_add_src2_u32", "ds_sub_src2_u32", "ds_rsub_src2_u32", "ds_inc_src2_u32", "ds_dec_src2_u32", "ds_min_src2_i32", "ds_max_src2_i32", "ds_min_src2_u32", + "ds_max_src2_u32", "ds_and_src2_b32", "ds_or_src2_b32", "ds_xor_src2_b32", "ds_write_src2_b32", + }; + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* in = &Input[tid * Inputs]; + const std::uint32_t a = in[0]; + const std::uint32_t b = in[1]; + const auto sa = static_cast(a); + const auto sb = static_cast(b); + const std::array expected{ + a + b, a - b, b - a, a >= b ? 0u : a + 1u, (a == 0u || a > b) ? b : a - 1u, + static_cast(std::min(sa, sb)), static_cast(std::max(sa, sb)), std::min(a, b), std::max(a, b), + a & b, a | b, a ^ b, b, + }; + for (std::uint32_t op = 0; op < expected.size(); ++op) { + const std::uint32_t actual = Output[tid * Results + op]; + Require(actual == expected[op], "lds src2: lane " + std::to_string(tid) + " " + names[op] + " is " + Hex(actual) + ", expected " + Hex(expected[op])); + } + } +} + +void CheckFloatRejected(AgcDriver::VulkanDevice& device) { + for (const std::uint32_t opcode : {0x92u, 0x93u, 0x95u}) { + alignas(256) static std::array code{}; + code = {0xd8000000u | (opcode << 18u) | 1u, 0x00000000u, 0xbf810000u}; + bool refused = false; + try { + (void)Compile(device, code); + } catch (const std::exception& error) { + refused = std::string(error.what()).find("f32 denormal mode") != std::string::npos; + } + Require(refused, "lds src2: f32 opcode " + Hex(opcode) + " was not refused"); + } +} + +void CheckAddressOffsetRejected(AgcDriver::VulkanDevice& device) { + try { + (void)Compile(device, AddressOffsetCode); + } catch (const std::exception&) { + return; + } + Require(false, "lds src2: an offset taken from the address was accepted"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + CheckLanes(); + CheckAddressOffsetRejected(*device); + CheckFloatRejected(*device); + std::puts("lds src2 tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LookBackScan.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LookBackScan.cpp index 48e3d4346..210d8e681 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LookBackScan.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LookBackScan.cpp @@ -2,6 +2,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "Recompiler.hpp" #include "VulkanTestDevice.hpp" +#include #include #include #include @@ -104,6 +105,10 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityInt64Atomics)) { + std::puts("skipped, the device has no shaderBufferInt64Atomics"); + return VulkanTestSkipped; + } if (device->Target().subgroupSize < 32) { std::printf("skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize); return VulkanTestSkipped; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ManyBuffers.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ManyBuffers.cpp index 3a26a3ca1..ec85957f9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ManyBuffers.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ManyBuffers.cpp @@ -106,7 +106,7 @@ void Run(AgcDriver::VulkanDevice& device, std::span code, s Table[buffer * 4u + 0u] = static_cast(address); Table[buffer * 4u + 1u] = static_cast((address >> 32u) & 0xffffu); Table[buffer * 4u + 2u] = RangeBytes; - Table[buffer * 4u + 3u] = 0x01016facu; + Table[buffer * 4u + 3u] = 0x31016facu; } const auto table = reinterpret_cast(Table.data()); const std::vector userData{static_cast(table), static_cast(table >> 32u)}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/MixPrecision.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/MixPrecision.cpp index 339dff87d..1d7b3d44a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/MixPrecision.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/MixPrecision.cpp @@ -81,7 +81,7 @@ std::string Hex(std::uint32_t value) { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/NanQuieting.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/NanQuieting.cpp new file mode 100644 index 000000000..07de164aa --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/NanQuieting.cpp @@ -0,0 +1,242 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e160280u, 0x7e180280u, + 0x7e1a0280u, 0x7e1c0280u, 0x7e1e0280u, 0x7e200280u, 0x7e220280u, 0x7e240280u, 0x7e260280u, 0x7e280280u, + 0x7e2a0280u, 0x7e2c0280u, 0x7e2e0280u, 0x7e300280u, 0x7e320280u, 0x2c3c0e90u, 0x7e146704u, 0x7e165504u, + 0x7e184104u, 0x7e1a4904u, 0x7e1c8104u, 0xd762000fu, 0x02010304u, 0xd55f0010u, 0x041a0b04u, 0xd5500011u, + 0x041a0b04u, 0x7e24ab07u, 0x7e26a907u, 0x7e28bf07u, 0xd53b0015u, 0x02010307u, 0x7e2cb307u, 0xd75f0017u, + 0x041e3d07u, 0x7e304305u, 0x7e32bd1eu, 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, 0xe0781020u, + 0x80011203u, 0xe0781030u, 0x80011603u, 0xbf810000u, +}; + +constexpr std::uint32_t Rows[32][4] = { + {0x7f812345u, 0x40000000u, 0x40400000u, 0x40007d23u}, + {0xff800001u, 0x40000000u, 0x40400000u, 0x4000fc01u}, + {0x7fa00000u, 0x40000000u, 0x40400000u, 0x40007d00u}, + {0x7fc00000u, 0x40000000u, 0x40400000u, 0x40007e00u}, + {0xffc00001u, 0x40000000u, 0x40400000u, 0x4000fe01u}, + {0x7fc12345u, 0x40000000u, 0x40400000u, 0x40007e55u}, + {0x3f800000u, 0x40000000u, 0x40400000u, 0x40003c00u}, + {0x40800000u, 0x40000000u, 0x40400000u, 0x40004400u}, + {0x3f800000u, 0x7f812345u, 0x40400000u, 0x7e553c00u}, + {0x3f800000u, 0xff800001u, 0x40400000u, 0x3c003c00u}, + {0x3f800000u, 0x7fa00000u, 0x40400000u, 0x44003c00u}, + {0x3f800000u, 0x7fc00000u, 0x40400000u, 0x7d233c00u}, + {0x3f800000u, 0xffc00001u, 0x40400000u, 0xfc013c00u}, + {0x3f800000u, 0x7fc12345u, 0x40400000u, 0x7d003c00u}, + {0x3f800000u, 0x3f800000u, 0x40400000u, 0x7e003c00u}, + {0x3f800000u, 0x40800000u, 0x40400000u, 0xfe013c00u}, + {0x3f800000u, 0x40000000u, 0x7fc00000u, 0x7e553c00u}, + {0x3f800000u, 0x40000000u, 0xffc00001u, 0x3c003c00u}, + {0x3f800000u, 0x40000000u, 0x7fc12345u, 0x44003c00u}, + {0x3f800000u, 0x40000000u, 0x3f800000u, 0x7d233c00u}, + {0x3f800000u, 0x40000000u, 0x40800000u, 0xfc013c00u}, + {0x3f800000u, 0x40000000u, 0x7f812345u, 0x7d003c00u}, + {0x3f800000u, 0x40000000u, 0xff800001u, 0x7e003c00u}, + {0x3f800000u, 0x40000000u, 0x7fa00000u, 0xfe013c00u}, + {0x7f812345u, 0x7f812345u, 0x7fc00000u, 0x7e557d23u}, + {0xff800001u, 0xff800001u, 0xffc00001u, 0x3c00fc01u}, + {0x7fa00000u, 0x7fa00000u, 0x7fc12345u, 0x44007d00u}, + {0x7fc00000u, 0x7fc00000u, 0x3f800000u, 0x7d237e00u}, + {0xffc00001u, 0xffc00001u, 0x40800000u, 0xfc01fe01u}, + {0x7fc12345u, 0x7fc12345u, 0x7f812345u, 0x7d007e55u}, + {0x3f800000u, 0x3f800000u, 0xff800001u, 0x7e003c00u}, + {0x40800000u, 0x40800000u, 0x7fa00000u, 0xfe014400u} +}; +constexpr std::uint32_t ExpectedIeee[32][16] = { + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7f800000u, 0x7fc12345u, 0x00007f23u, 0x00007f23u, 0x00007f23u, 0x00007f23u, 0x00007f23u, 0x00007f23u, 0x40000000u, 0x00004000u}, + {0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0x7f800000u, 0xffc00001u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x40000000u, 0x00004000u}, + {0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7f800000u, 0x7fe00000u, 0x00007f00u, 0x00007f00u, 0x00007f00u, 0x00007f00u, 0x00007f00u, 0x00007f00u, 0x40000000u, 0x00004000u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7f800000u, 0x7fc00000u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x40000000u, 0x00004000u}, + {0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0x7f800000u, 0xffc00001u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x40000000u, 0x00004000u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7f800000u, 0x7fc12345u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x40000000u, 0x00004000u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x3f800000u, 0x40000000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00003c00u, 0x40000000u, 0x00004000u}, + {0x40000000u, 0x3e800000u, 0x00000000u, 0x40800000u, 0x3f000000u, 0x41000000u, 0x40800000u, 0x41000000u, 0x00004000u, 0x00003400u, 0x00000000u, 0x00004800u, 0x00003800u, 0x00004400u, 0x40000000u, 0x00004000u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fc12345u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007e55u, 0x7fc12345u, 0x00007e55u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0xffc00001u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00003c00u, 0xffc00001u, 0x00003c00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fe00000u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00003c00u, 0x7fe00000u, 0x00004400u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fc00000u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007f23u, 0x7fc00000u, 0x00007f23u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0xffc00001u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x0000fe01u, 0xffc00001u, 0x0000fe01u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fc12345u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007f00u, 0x7fc12345u, 0x00007f00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x3f800000u, 0x3f800000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007e00u, 0x3f800000u, 0x00007e00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x3f800000u, 0x40800000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x0000fe01u, 0x40800000u, 0x0000fe01u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fc00000u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007e55u, 0x40000000u, 0x00007e55u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0xffc00001u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00003c00u, 0x40000000u, 0x00003c00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fc12345u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00003c00u, 0x40000000u, 0x00004400u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x3f800000u, 0x40000000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007f23u, 0x40000000u, 0x00007f23u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x3f800000u, 0x40000000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x0000fe01u, 0x40000000u, 0x0000fe01u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fc12345u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007f00u, 0x40000000u, 0x00007f00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0xffc00001u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007e00u, 0x40000000u, 0x00007e00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fe00000u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x0000fe01u, 0x40000000u, 0x0000fe01u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc00000u, 0xff7fffffu, 0x00007f23u, 0x00007f23u, 0x00007f23u, 0x00007f23u, 0x00007f23u, 0x00007f23u, 0x7fc12345u, 0x00007e55u}, + {0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xff7fffffu, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0xffc00001u, 0x00003c00u}, + {0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fe00000u, 0x7fc12345u, 0xff7fffffu, 0x00007f00u, 0x00007f00u, 0x00007f00u, 0x00007f00u, 0x00007f00u, 0x00007f00u, 0x7fe00000u, 0x00004400u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0xff7fffffu, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x7fc00000u, 0x00007f23u}, + {0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xff7fffffu, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0xffc00001u, 0x0000fe01u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0xff7fffffu, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x7fc12345u, 0x00007f00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0xffc00001u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007e00u, 0x3f800000u, 0x00007e00u}, + {0x40000000u, 0x3e800000u, 0x00000000u, 0x40800000u, 0x3f000000u, 0x41000000u, 0x7fe00000u, 0xff7fffffu, 0x00004000u, 0x00003400u, 0x00000000u, 0x00004800u, 0x00003800u, 0x0000fe01u, 0x40800000u, 0x0000fe01u} +}; +constexpr std::uint32_t ExpectedNoIeee[32][16] = { + {0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f800000u, 0x7f812345u, 0x00007d23u, 0x00007d23u, 0x00007d23u, 0x00007d23u, 0x00007d23u, 0x00007d23u, 0x40000000u, 0x00004000u}, + {0xff800001u, 0xff800001u, 0xff800001u, 0xff800001u, 0xff800001u, 0xff800001u, 0x7f800000u, 0xff800001u, 0x0000fc01u, 0x0000fc01u, 0x0000fc01u, 0x0000fc01u, 0x0000fc01u, 0x0000fc01u, 0x40000000u, 0x00004000u}, + {0x7fa00000u, 0x7fa00000u, 0x7fa00000u, 0x7fa00000u, 0x7fa00000u, 0x7fa00000u, 0x7f800000u, 0x7fa00000u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x40000000u, 0x00004000u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7f800000u, 0x7fc00000u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x40000000u, 0x00004000u}, + {0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0x7f800000u, 0xffc00001u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x40000000u, 0x00004000u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7f800000u, 0x7fc12345u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x40000000u, 0x00004000u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x3f800000u, 0x40000000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00003c00u, 0x40000000u, 0x00004000u}, + {0x40000000u, 0x3e800000u, 0x00000000u, 0x40800000u, 0x3f000000u, 0x41000000u, 0x40800000u, 0x41000000u, 0x00004000u, 0x00003400u, 0x00000000u, 0x00004800u, 0x00003800u, 0x00004400u, 0x40000000u, 0x00004000u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7f812345u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007e55u, 0x7f812345u, 0x00007e55u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0xff800001u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00003c00u, 0xff800001u, 0x00003c00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fa00000u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00003c00u, 0x7fa00000u, 0x00004400u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fc00000u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007d23u, 0x7fc00000u, 0x00007d23u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0xffc00001u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x0000fc01u, 0xffc00001u, 0x0000fc01u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fc12345u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007d00u, 0x7fc12345u, 0x00007d00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x3f800000u, 0x3f800000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007e00u, 0x3f800000u, 0x00007e00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x3f800000u, 0x40800000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x0000fe01u, 0x40800000u, 0x0000fe01u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fc00000u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007e55u, 0x40000000u, 0x00007e55u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0xffc00001u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00003c00u, 0x40000000u, 0x00003c00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fc12345u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00003c00u, 0x40000000u, 0x00004400u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x3f800000u, 0x40000000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007d23u, 0x40000000u, 0x00007d23u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x3f800000u, 0x40000000u, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x0000fc01u, 0x40000000u, 0x0000fc01u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7f812345u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007d00u, 0x40000000u, 0x00007d00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0xff800001u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007e00u, 0x40000000u, 0x00007e00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0x7fa00000u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x0000fe01u, 0x40000000u, 0x0000fe01u}, + {0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7fc00000u, 0xff7fffffu, 0x00007d23u, 0x00007d23u, 0x00007d23u, 0x00007d23u, 0x00007d23u, 0x00007d23u, 0x7f812345u, 0x00007e55u}, + {0xff800001u, 0xff800001u, 0xff800001u, 0xff800001u, 0xff800001u, 0xff800001u, 0xffc00001u, 0xff7fffffu, 0x0000fc01u, 0x0000fc01u, 0x0000fc01u, 0x0000fc01u, 0x0000fc01u, 0x0000fc01u, 0xff800001u, 0x00003c00u}, + {0x7fa00000u, 0x7fa00000u, 0x7fa00000u, 0x7fa00000u, 0x7fa00000u, 0x7fa00000u, 0x7fc12345u, 0xff7fffffu, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x7fa00000u, 0x00004400u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0xff7fffffu, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x7fc00000u, 0x00007d23u}, + {0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xffc00001u, 0xff7fffffu, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0x0000fe01u, 0xffc00001u, 0x0000fc01u}, + {0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7fc12345u, 0x7f812345u, 0xff7fffffu, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x00007e55u, 0x7fc12345u, 0x00007d00u}, + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x40000000u, 0xff800001u, 0xff7fffffu, 0x00003c00u, 0x00003c00u, 0x00000000u, 0x00004000u, 0x00003800u, 0x00007e00u, 0x3f800000u, 0x00007e00u}, + {0x40000000u, 0x3e800000u, 0x00000000u, 0x40800000u, 0x3f000000u, 0x41000000u, 0x7fa00000u, 0xff7fffffu, 0x00004000u, 0x00003400u, 0x00000000u, 0x00004800u, 0x00003800u, 0x0000fe01u, 0x40800000u, 0x0000fe01u} +}; +constexpr const char* Names[16] = { + "v_sqrt_f32", + "v_rcp_f32", + "v_fract_f32", + "v_floor_f32", + "v_frexp_mant_f32", + "v_ldexp_f32", + "v_div_fixup_f32", + "v_mullit_f32", + "v_sqrt_f16", + "v_rcp_f16", + "v_fract_f16", + "v_ldexp_f16", + "v_frexp_mant_f16", + "v_div_fixup_f16", + "v_trunc_f32", + "v_rndne_f16" +}; + +void Fill(std::uint32_t tid, std::uint32_t* words) { + std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words); +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) { + Require(actual == expected, std::string("nan quieting: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); +} + +bool IsNan32(std::uint32_t value) { + return (value & 0x7fffffffu) > 0x7f800000u; +} + +bool IsNan16(std::uint32_t value) { + return (value & 0x7fffu) > 0x7c00u; +} + +void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = floatMode; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(const std::uint32_t (&expected)[32][16], const char* mode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t i = 0; i < 16; ++i) { + const std::string name = std::string(mode) + " " + Names[i]; + const bool half = i >= 8u && i != 14u; + const std::uint32_t got = half ? out[i] & 0xffffu : out[i]; + const std::uint32_t want = half ? expected[tid][i] & 0xffffu : expected[tid][i]; + const bool approximate = i == 0u || i == 1u || i == 8u || i == 9u; + if (approximate && !(half ? IsNan16(want) : IsNan32(want))) continue; + Expect(tid, got, want, name.c_str()); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, std::nullopt); + Check(ExpectedNoIeee, "no float mode"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + Check(ExpectedNoIeee, "IEEE=0"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false}); + Check(ExpectedIeee, "IEEE=1"); + std::puts("nan quieting tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/NggVertexWaveInfo.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/NggVertexWaveInfo.cpp new file mode 100644 index 000000000..ba1c0124f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/NggVertexWaveInfo.cpp @@ -0,0 +1,130 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 32; +constexpr std::byte Kept{0x40}; +alignas(256) std::array Pixels{}; + +alignas(256) constexpr std::array VertexCode{ + 0x8f008803, 0x94fe00c1, 0x99030380, 0x94fe03c1, + 0xe0382000, 0x80020005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +alignas(256) constexpr std::array PixelCode{ + 0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +constexpr std::array, 3> Triangle{{ + {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f} +}}; + +void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize) { + Pixels.fill(Kept); + const auto target = device.Target(); + + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast(Triangle.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 8, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.wave32 = waveSize == 32u; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData(8, 0u); + const std::array pixelMemory{{{reinterpret_cast(PixelCode.data()), std::as_bytes(std::span(PixelCode))}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, + {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + {0, 0, vertexPush, 128 - vertexPush} + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, vertexPush} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Triangle.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); +} + +void Check(std::uint32_t waveSize) { + const auto what = "NGG vertex prologue wave" + std::to_string(waveSize); + for (std::uint32_t pixel = 0; pixel < Width * Height; ++pixel) { + for (std::uint32_t channel = 0; channel < 4; ++channel) { + const auto actual = Pixels[pixel * 4u + channel]; + Require(actual == std::byte{255}, what + ": pixel " + std::to_string(pixel) + " channel " + std::to_string(channel) + " is " + std::to_string(static_cast(actual)) + ", expected 255 (EXEC built from the merged wave info must cover every vertex)"); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (device->Target().subgroupSize < 32u) { + std::printf("skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize); + return VulkanTestSkipped; + } + for (const auto waveSize : {64u, 32u}) { + Draw(*device, waveSize); + Check(waveSize); + } + std::puts("NGG vertex wave info tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/OversizedDescriptorSets.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/OversizedDescriptorSets.cpp new file mode 100644 index 000000000..be56587bf --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/OversizedDescriptorSets.cpp @@ -0,0 +1,258 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Check; +using AgcDriver::Graphics::Context; +using AgcDriver::Graphics::DescriptorCache; +using AgcDriver::Graphics::Require; + +constexpr std::uint32_t Buffers = 4097; +constexpr std::uint32_t Spacing = 256; + +class Device { +public: + Device() { +#ifdef _WIN32 + library = SDL_LoadObject("vulkan-1.dll"); +#else + library = SDL_LoadObject("libvulkan.so.1"); +#endif + Require(library != nullptr, "cannot load Vulkan"); + try { + instanceProc = reinterpret_cast(SDL_LoadFunction(library, "vkGetInstanceProcAddr")); + Require(instanceProc != nullptr, "missing Vulkan instance resolver"); + VkApplicationInfo application{VK_STRUCTURE_TYPE_APPLICATION_INFO}; + application.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo info{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; + info.pApplicationInfo = &application; + Check(function("vkCreateInstance")(&info, nullptr, &instance), "vkCreateInstance"); + std::uint32_t count = 0; + const auto enumerate = function("vkEnumeratePhysicalDevices"); + Check(enumerate(instance, &count, nullptr), "vkEnumeratePhysicalDevices"); + Require(count != 0, "no Vulkan device"); + std::vector devices(count); + Check(enumerate(instance, &count, devices.data()), "vkEnumeratePhysicalDevices"); + const auto properties = function("vkGetPhysicalDeviceProperties"); + context.physical = devices.front(); + for (const auto candidate : devices) { + VkPhysicalDeviceProperties described{}; + properties(candidate, &described); + if (described.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) { + context.physical = candidate; + break; + } + } + VkPhysicalDeviceProperties selected{}; + properties(context.physical, &selected); + context.limits = selected.limits; + std::printf("device %s\n", selected.deviceName); + const auto queues = function("vkGetPhysicalDeviceQueueFamilyProperties"); + queues(context.physical, &count, nullptr); + std::vector families(count); + queues(context.physical, &count, families.data()); + std::uint32_t family = 0; + while (family < count && (families[family].queueFlags & VK_QUEUE_COMPUTE_BIT) == 0) ++family; + Require(family < count, "no Vulkan compute queue"); + const float priority = 1; + VkDeviceQueueCreateInfo queue{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + queue.queueFamilyIndex = family; + queue.queueCount = 1; + queue.pQueuePriorities = &priority; + VkDeviceCreateInfo device{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + device.queueCreateInfoCount = 1; + device.pQueueCreateInfos = &queue; + Check(function("vkCreateDevice")(context.physical, &device, nullptr, &context.device), "vkCreateDevice"); + context.deviceProc = function("vkGetDeviceProcAddr"); + function("vkGetPhysicalDeviceMemoryProperties")(context.physical, &context.memory); + context.formatProperties = function("vkGetPhysicalDeviceFormatProperties"); + context.Function("vkGetDeviceQueue")(context.device, family, 0, &context.queue); + VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.queueFamilyIndex = family; + Check(context.Function("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); + } catch (...) { + release(); + throw; + } + } + + ~Device() { release(); } + Device(const Device&) = delete; + Device& operator=(const Device&) = delete; + const Context& GetContext() const { return context; } + +private: + template + TFunction function(const char* name) const { + const auto result = reinterpret_cast(instanceProc(instance, name)); + Require(result != nullptr, name); + return result; + } + + void release() noexcept { + if (context.pool != VK_NULL_HANDLE) context.Function("vkDestroyCommandPool")(context.device, context.pool, nullptr); + context.bufferPool.reset(); + if (context.device != VK_NULL_HANDLE) function("vkDestroyDevice")(context.device, nullptr); + if (instance != VK_NULL_HANDLE) function("vkDestroyInstance")(instance, nullptr); + if (library != nullptr) SDL_UnloadObject(library); + } + + void* library = nullptr; + PFN_vkGetInstanceProcAddr instanceProc = nullptr; + VkInstance instance = VK_NULL_HANDLE; + Context context{}; +}; + +void Emit(std::vector& words, std::uint32_t opcode, std::initializer_list operands) { + words.push_back(static_cast((operands.size() + 1u) << 16u) | opcode); + words.insert(words.end(), operands); +} + +std::vector SumLastTwoModule() { + enum : std::uint32_t { Void = 1, FunctionType, Uint, Block, Length, Array, ArrayPointer, Variable, UintPointer, Zero, Last, BeforeLast, Main, Label, LastPointer, LastValue, BeforeLastPointer, BeforeLastValue, Sum, Destination, Bound }; + constexpr std::uint32_t storageBuffer = 12; + std::vector words{0x07230203u, 0x00010300u, 0u, Bound, 0u}; + Emit(words, 17, {1}); + Emit(words, 14, {0, 1}); + Emit(words, 15, {5, Main, 0x6e69616du, 0u}); + Emit(words, 16, {Main, 17, 1, 1, 1}); + Emit(words, 71, {Block, 2}); + Emit(words, 72, {Block, 0, 35, 0}); + Emit(words, 71, {Variable, 34, 0}); + Emit(words, 71, {Variable, 33, 0}); + Emit(words, 19, {Void}); + Emit(words, 33, {FunctionType, Void}); + Emit(words, 21, {Uint, 32, 0}); + Emit(words, 30, {Block, Uint}); + Emit(words, 43, {Uint, Length, Buffers}); + Emit(words, 28, {Array, Block, Length}); + Emit(words, 32, {ArrayPointer, storageBuffer, Array}); + Emit(words, 59, {ArrayPointer, Variable, storageBuffer}); + Emit(words, 32, {UintPointer, storageBuffer, Uint}); + Emit(words, 43, {Uint, Zero, 0}); + Emit(words, 43, {Uint, Last, Buffers - 1u}); + Emit(words, 43, {Uint, BeforeLast, Buffers - 2u}); + Emit(words, 54, {Void, Main, 0, FunctionType}); + Emit(words, 248, {Label}); + Emit(words, 65, {UintPointer, LastPointer, Variable, Last, Zero}); + Emit(words, 61, {Uint, LastValue, LastPointer}); + Emit(words, 65, {UintPointer, BeforeLastPointer, Variable, BeforeLast, Zero}); + Emit(words, 61, {Uint, BeforeLastValue, BeforeLastPointer}); + Emit(words, 128, {Uint, Sum, LastValue, BeforeLastValue}); + Emit(words, 65, {UintPointer, Destination, Variable, Zero, Zero}); + Emit(words, 62, {Destination, Sum}); + Emit(words, 253, {}); + Emit(words, 56, {}); + return words; +} + +std::uint32_t Input(std::uint32_t buffer) { + return buffer * 3u + 1u; +} + +void Run(const Context& context) { + DescriptorCache cache(context); + const VkDescriptorSetLayoutBinding binding{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, Buffers, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}; + const std::array key{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, Buffers, VK_SHADER_STAGE_COMPUTE_BIT}; + auto setLayout = cache.Layout(key, std::span(&binding, 1)); + const std::array sizes{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, Buffers}}}; + const auto first = cache.Allocate(setLayout, sizes); + const auto second = cache.Allocate(setLayout, sizes); + Require(first.set != VK_NULL_HANDLE && second.set != VK_NULL_HANDLE, "a set of " + std::to_string(Buffers) + " storage buffers, above the chain pool's capacity, got no descriptor set"); + Require(first.pool != second.pool && cache.Counters().pools == 0, "the oversized sets do not have a pool each"); + + AgcDriver::Graphics::Buffer storage(context, static_cast(Buffers) * Spacing, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + auto* words = reinterpret_cast(storage.Bytes().data()); + for (std::uint32_t buffer = 0; buffer < Buffers; ++buffer) words[buffer * (Spacing / 4u)] = Input(buffer); + std::vector infos; + for (std::uint32_t buffer = 0; buffer < Buffers; ++buffer) infos.push_back({storage.Handle(), static_cast(buffer) * Spacing, 4}); + VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; + write.dstSet = second.set; + write.descriptorCount = Buffers; + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write.pBufferInfo = infos.data(); + context.Function("vkUpdateDescriptorSets")(context.device, 1, &write, 0, nullptr); + + const auto module = SumLastTwoModule(); + VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + moduleInfo.codeSize = module.size() * sizeof(std::uint32_t); + moduleInfo.pCode = module.data(); + VkShaderModule shader = VK_NULL_HANDLE; + Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &shader), "vkCreateShaderModule"); + VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + layoutInfo.setLayoutCount = 1; + layoutInfo.pSetLayouts = &setLayout; + VkPipelineLayout pipelineLayout = VK_NULL_HANDLE; + Check(context.Function("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &pipelineLayout), "vkCreatePipelineLayout"); + VkComputePipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + pipelineInfo.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; + pipelineInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + pipelineInfo.stage.module = shader; + pipelineInfo.stage.pName = "main"; + pipelineInfo.layout = pipelineLayout; + VkPipeline pipeline = VK_NULL_HANDLE; + Check(context.Function("vkCreateComputePipelines")(context.device, VK_NULL_HANDLE, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateComputePipelines"); + { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + AgcDriver::Graphics::CommandBatch batch(context); + context.Function("vkCmdBindPipeline")(batch.Handle(), VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); + context.Function("vkCmdBindDescriptorSets")(batch.Handle(), VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &second.set, 0, nullptr); + context.Function("vkCmdDispatch")(batch.Handle(), 1, 1, 1); + AgcDriver::Graphics::RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + batch.SubmitAndWait(); + } + context.Function("vkDestroyPipeline")(context.device, pipeline, nullptr); + context.Function("vkDestroyPipelineLayout")(context.device, pipelineLayout, nullptr); + context.Function("vkDestroyShaderModule")(context.device, shader, nullptr); + storage.Invalidate(); + const auto expected = Input(Buffers - 1u) + Input(Buffers - 2u); + Require(words[0] == expected, "the dispatch through the oversized set wrote " + std::to_string(words[0]) + ", expected " + std::to_string(expected) + " from its last two storage buffers"); + + cache.Free(first); + cache.Free(second); + const auto again = cache.Allocate(setLayout, sizes); + Require(again.set != VK_NULL_HANDLE && again.pool != VK_NULL_HANDLE && cache.Counters().pools == 0, "an oversized set after the others were freed got no descriptor set"); + cache.Free(again); +} + +} + +int main() { + try { + std::unique_ptr device; + try { + device = std::make_unique(); + } catch (const std::exception& error) { + if (std::getenv("ANYPS5_REQUIRE_VULKAN") != nullptr) throw; + std::printf("skipped, no usable Vulkan device: %s\n", error.what()); + return VulkanTestSkipped; + } + const auto& limits = device->GetContext().limits; + if (limits.maxPerStageDescriptorStorageBuffers < Buffers || limits.maxPerStageResources < Buffers || limits.maxDescriptorSetStorageBuffers < Buffers) { + std::printf("skipped, the device binds fewer than %u storage buffers per stage or set\n", Buffers); + return VulkanTestSkipped; + } + Run(device->GetContext()); + std::puts("oversized descriptor set tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PackedAlu.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PackedAlu.cpp new file mode 100644 index 000000000..a54e9e63f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PackedAlu.cpp @@ -0,0 +1,276 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 64; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084, 0x34060088, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xcc00400a, 0x1c1a0b04, 0xcc00280b, + 0x141a0b04, 0xcc09400c, 0x1c1a0b04, 0xcc09300d, 0x0c1a0b04, 0xcc01400e, 0x1a020b04, 0xcc01580f, + 0x02020b04, 0xcc024010, 0x1a020b04, 0xcc024811, 0x1a020b04, 0xcc0a4012, 0x1a020b04, 0xcc0a4013, + 0x02020b04, 0xcc034014, 0x1a020b04, 0xcc035815, 0x1a020b04, 0xcc0b4016, 0x1a020b04, 0xcc0b4817, + 0x12020b04, 0xcc044018, 0x1a020b04, 0xcc045019, 0x0a020b04, 0xcc05401a, 0x1a020b04, 0xcc05581b, + 0x02020b04, 0xcc06401c, 0x1a020b04, 0xcc06481d, 0x1a020b04, 0xcc07401e, 0x1a020b04, 0xcc07401f, + 0x02020b04, 0xcc084020, 0x1a020b04, 0xcc085821, 0x1a020b04, 0xcc0c4022, 0x1a020b04, 0xcc0c4823, + 0x12020b04, 0xcc0d4024, 0x1a020b04, 0xcc0d5025, 0x0a020b04, 0x7e4c0306, 0x784c0b04, 0xcc0e4027, + 0x1c1a0b04, 0xcc0e5828, 0x041a0b04, 0xcc0e4229, 0x3c1a0b04, 0xcc0f402a, 0x1a020b04, 0xcc0f482b, + 0x1a020b04, 0xcc0f412c, 0x5a020b04, 0xcc10402d, 0x1a020b04, 0xcc10402e, 0x02020b04, 0xcc10402f, + 0x7a020b04, 0xcc114030, 0x1a020b04, 0xcc115831, 0x1a020b04, 0xcc114232, 0x3a020b04, 0xcc124033, + 0x1a020b04, 0xcc124834, 0x12020b04, 0xcc124135, 0x5a020b04, 0xe0781000, 0x80010a03, 0xe0781010, + 0x80010e03, 0xe0781020, 0x80011203, 0xe0781030, 0x80011603, 0xe0781040, 0x80011a03, 0xe0781050, + 0x80011e03, 0xe0781060, 0x80012203, 0xe0781070, 0x80012603, 0xe0781080, 0x80012a03, 0xe0781090, + 0x80012e03, 0xe07810a0, 0x80013203, 0xbf810000, +}; + +constexpr std::uint32_t Rows[32][4] = { + {0xfc007c00u, 0x80000000u, 0x3c003c00u, 0x00000000u}, + {0xfc007c00u, 0x7c00fc00u, 0xbc007c00u, 0x00000000u}, + {0x3c01fc00u, 0x7fffffffu, 0x3800bc00u, 0x00000000u}, + {0x14007fffu, 0x00008000u, 0xc0001400u, 0x00000000u}, + {0xfffffe55u, 0x80007e00u, 0xfe00fffeu, 0x00000000u}, + {0x7fff3c01u, 0x7fff3c01u, 0x7ffffe55u, 0x00000000u}, + {0xedccedccu, 0x00008000u, 0x7bff6000u, 0x00000000u}, + {0x8400bc00u, 0xfc001234u, 0x0000edccu, 0x00000000u}, + {0x7d237bffu, 0x35557bffu, 0xfe005640u, 0x00000000u}, + {0x35557c01u, 0x80008000u, 0x80000000u, 0x00000000u}, + {0x7fffc248u, 0x7c010000u, 0x00000000u, 0x00000000u}, + {0x1234fffeu, 0x3c01c000u, 0x7fffffffu, 0x00000000u}, + {0x42481234u, 0x8400edccu, 0x7c00fffeu, 0x00000000u}, + {0xfffefe00u, 0xffff7fffu, 0x04003800u, 0x00000000u}, + {0x12344248u, 0x7e007fffu, 0xfe007e00u, 0x00000000u}, + {0x3555fc00u, 0x00005640u, 0x80000800u, 0x00000000u}, + {0xedcc7d23u, 0x12347d23u, 0x38007fffu, 0x00000000u}, + {0x7c00ffffu, 0x7fff8000u, 0x3c005640u, 0x00000000u}, + {0x3c01fffeu, 0x00003555u, 0xfffe3800u, 0x00000000u}, + {0xc2484248u, 0x7bff4248u, 0x14003800u, 0x00000000u}, + {0xedccfe55u, 0xedcc0000u, 0x40008000u, 0x00000000u}, + {0xfffeedccu, 0x00000400u, 0x12341400u, 0x00000000u}, + {0x7d236000u, 0x0000fe55u, 0x7fffbc00u, 0x00000000u}, + {0x7c018000u, 0xc0000000u, 0x7c008400u, 0x00000000u}, + {0x3555fe55u, 0x7c00c248u, 0x14007e00u, 0x00000000u}, + {0xedcc3c01u, 0x4248ffffu, 0xfbff3c00u, 0x00000000u}, + {0x7d237d23u, 0xffff0000u, 0xfe55edccu, 0x00000000u}, + {0xc2480000u, 0x00000400u, 0x00008000u, 0x00000000u}, + {0x3555edccu, 0xfffffc00u, 0x00008400u, 0x00000000u}, + {0x00001234u, 0x7d230000u, 0x8400c000u, 0x00000000u}, + {0x00007c00u, 0xfc000000u, 0x7d557e55u, 0x00000000u}, + {0x7c000000u, 0x00007c00u, 0xfe557d55u, 0x00000000u}, +}; +constexpr std::uint32_t Expected[32][44] = { + {0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0x80000000u, 0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80000000u, 0xfc007c00u, 0x7c007c00u, 0x80000000u, 0x80008000u, 0xfc007c00u, 0x8000fc00u, 0x80000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc00fc00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0xfc00fc00u, 0x80007c00u, 0x7c000000u, 0x7c007c00u}, + {0xbc007c00u, 0x7c00bc00u, 0xbc007c00u, 0x7c00bc00u, 0x00000000u, 0x00000000u, 0x78007800u, 0x7800f800u, 0x78007800u, 0x78007800u, 0x80008000u, 0x80008000u, 0x80008000u, 0x00000000u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c00fc00u, 0x7c007c00u, 0xfc007c00u, 0xfc00fe00u, 0xfc00fe00u, 0xfc00fe00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fc00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u}, + {0x7bffc000u, 0xc000fbffu, 0x7bffc000u, 0x7fff3c00u, 0x43ff0400u, 0x040043ffu, 0xbc00fbffu, 0xbc003c00u, 0xbc00fbffu, 0xfbfffbffu, 0xbc02fc01u, 0xbc02bc02u, 0xbc02fc01u, 0x7c013c02u, 0xfffeffffu, 0xfffe7fffu, 0x3fffffffu, 0xffff3fffu, 0x3fffffffu, 0x3fffffffu, 0x7fffffffu, 0xffffffffu, 0x3c01fc00u, 0x3c013c01u, 0x7fffffffu, 0xfc00ffffu, 0x3c01fc00u, 0x3c017fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x7fff7fffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x3c01fc00u, 0x3c013c01u, 0x3c017c00u, 0x3c01fc00u, 0xfc003c01u, 0xbc01fc00u}, + {0xc0009400u, 0x1400c000u, 0xc0009400u, 0x1400c000u, 0x00008000u, 0x80000000u, 0x1400ffffu, 0x14009400u, 0x1400ffffu, 0xffffffffu, 0x1400ffffu, 0x14001400u, 0x1400ffffu, 0x7fff9400u, 0x00000000u, 0x80000000u, 0x00000001u, 0x00010000u, 0x0000ffffu, 0x00008000u, 0x14007fffu, 0x7fff7fffu, 0x00008000u, 0x00000000u, 0x14008000u, 0x7fff8000u, 0x00007fffu, 0x14000000u, 0xc0007fffu, 0xc0007fffu, 0x7fff1400u, 0xc000ffffu, 0x14007fffu, 0x14001400u, 0x94007fffu, 0x00007fffu, 0x7fff7fffu, 0x0000ffffu, 0x00008000u, 0x00000000u, 0x80008000u, 0x14008000u, 0x00001400u, 0x00000000u}, + {0x7e00d5feu, 0x7ffe8000u, 0x7e00d5feu, 0x81fe7e00u, 0x8000d600u, 0xd6008000u, 0x7fff7c55u, 0x7fff7dffu, 0x7fff7c55u, 0x7c557c55u, 0x7fff8055u, 0x7fff7fffu, 0x7fff8055u, 0x7e5581ffu, 0x0000c000u, 0x00000000u, 0x000103f0u, 0x03f00001u, 0xffff03f0u, 0xffff0000u, 0xffff7e00u, 0x7e007e00u, 0x8000fe55u, 0x80008000u, 0xfffffe55u, 0xfe55ffffu, 0x80007e00u, 0x7e008000u, 0xfffffe55u, 0xfffffe55u, 0xfe55ffffu, 0xffff7e55u, 0xfffffe55u, 0xffffffffu, 0x7ffffe55u, 0xfffffe55u, 0xfe55fe55u, 0xffff7e55u, 0x8000fe55u, 0x80008000u, 0x00007e55u, 0x8000fe55u, 0x8000ffffu, 0x8000fe55u}, + {0x80007656u, 0x4254c3feu, 0x80007656u, 0x4254c3feu, 0x00017801u, 0x78010001u, 0xfffe7802u, 0xfffebc00u, 0xfffe7802u, 0x78027802u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc0243feu, 0x80007802u, 0x8000fffeu, 0x00001e00u, 0x1e000000u, 0x00001e00u, 0x00000000u, 0x7fff3c01u, 0x3c013c01u, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x3c013c01u, 0x7ffffe55u, 0x7ffffe55u, 0x7fff7fffu, 0x7ffffe55u, 0x7fff4001u, 0x7fff7fffu, 0xffff0000u, 0x7fff3c02u, 0x3c023c02u, 0x7fff3c02u, 0x7fff3c01u, 0x7fff7fffu, 0x7fffbc01u, 0x7fff3c01u, 0x3c013c01u, 0xffff3c01u}, + {0x7bff6000u, 0x60007bffu, 0x7bff6000u, 0x60007bffu, 0x00000000u, 0x00000000u, 0xedcc6dccu, 0xedcc6dccu, 0xedcc6dccu, 0x6dcc6dccu, 0xedcc6dccu, 0xedccedccu, 0xedcc6dccu, 0xedcc6dccu, 0x00000000u, 0x00000000u, 0x00000008u, 0x00080000u, 0x0000fff8u, 0x0000fff8u, 0x0000edccu, 0xedccedccu, 0xedcc8000u, 0xedccedccu, 0xedccedccu, 0xedccedccu, 0x00008000u, 0x80000000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0xedccedccu, 0xedccedccu, 0x6dccedccu, 0x80000000u, 0x00000000u, 0x80000000u, 0xedccedccu, 0xedccedccu, 0xedcc8000u, 0x00008000u, 0x00008000u, 0x6dcc0000u}, + {0x00001dccu, 0xedccd000u, 0x00001dccu, 0xbdcc0000u, 0x00003000u, 0x30000000u, 0x8000ce34u, 0x80009634u, 0x8000ce34u, 0xce34ce34u, 0x8800a9ccu, 0x88008800u, 0x8800a9ccu, 0xc00071ccu, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0xfc001234u, 0xfc001234u, 0x12341234u, 0x8400bc00u, 0x84008400u, 0xfc00bc00u, 0xfc008400u, 0x84001234u, 0x1234bc00u, 0x7c00edccu, 0x7c00edccu, 0x92347c00u, 0xfc00edccu, 0xfc00bbfeu, 0xfc0011b4u, 0xfc00bc01u, 0x7c009234u, 0x92349234u, 0x7c009234u, 0xfc00bc00u, 0xfc00fc00u, 0x84001234u, 0x84001234u, 0xbc001234u, 0x04009234u}, + {0xc99f5e41u, 0x4ceb74ddu, 0xc99f5e41u, 0xcd1df4abu, 0xcb9f0801u, 0x0801cb9fu, 0xb278f7feu, 0xb278f922u, 0xb278f7feu, 0xf7fef7feu, 0x47ce0000u, 0x47ce47ceu, 0x47ce0000u, 0x46aa0124u, 0xaaa88000u, 0xdff88000u, 0x06aa0000u, 0x000006aau, 0x06aa0000u, 0x06aa0f7fu, 0x7d237bffu, 0x7bff7bffu, 0x35557bffu, 0x35553555u, 0x7d237bffu, 0x7bff7d23u, 0x35557bffu, 0x7bff3555u, 0x7f237c00u, 0x7f237c00u, 0xfe007f23u, 0x7f23fc00u, 0x7f237c00u, 0x7f237f23u, 0xff230000u, 0x7f237c00u, 0x7c007c00u, 0x7f237c00u, 0x7f237bffu, 0x7f237f23u, 0x7f23fbffu, 0x7f237bffu, 0x7bff7f23u, 0xff237bffu}, + {0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80008000u, 0x80008000u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01fc01u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01b555u, 0x00000000u, 0x00000000u, 0x04004000u, 0x40000400u, 0xfc00c000u, 0xfc00fc00u, 0x35557c01u, 0x7c017c01u, 0x80008000u, 0x80008000u, 0x80008000u, 0x80008000u, 0x35557c01u, 0x35557c01u, 0x80007e01u, 0x80007e01u, 0x7e010000u, 0x0000fe01u, 0x35557e01u, 0x35553555u, 0xb5557e01u, 0x80007e01u, 0x7e017e01u, 0x8000fe01u, 0x80007e01u, 0x80008000u, 0x0000fe01u, 0x35557e01u, 0x7e013555u, 0x80007e01u}, + {0x03ff0000u, 0xa2480000u, 0x03ff0000u, 0x0000a248u, 0x03ff0000u, 0x000003ffu, 0xfc00c248u, 0xfc007fffu, 0xfc00c248u, 0xc248c248u, 0x03fec248u, 0x03fe03feu, 0x03fec248u, 0x46477fffu, 0x80000000u, 0x00000100u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fff0000u, 0x00000000u, 0x7c01c248u, 0x7c017c01u, 0x7fffc248u, 0xc2487fffu, 0x7c010000u, 0x00007c01u, 0x7fff0000u, 0x7fff0000u, 0x00007fffu, 0x7fff0000u, 0x7fffc248u, 0x7fff7fffu, 0xffffc248u, 0x7fff8000u, 0x80008000u, 0x7fff8000u, 0x7e01c248u, 0x7e017e01u, 0xfe010000u, 0x7e010000u, 0x7e010000u, 0x7e018000u}, + {0xc2337fffu, 0x87fd7fffu, 0xc2337fffu, 0xffff07fdu, 0x42348000u, 0x80004234u, 0x4e35bffeu, 0x4e35d234u, 0x4e35bffeu, 0xbffebffeu, 0xd6333ffeu, 0xd633d633u, 0xd6333ffeu, 0xc3fd5234u, 0xc0100000u, 0x00004000u, 0x03c00003u, 0x000303c0u, 0x03c0ffffu, 0x03c0fc00u, 0x3c01fffeu, 0xfffefffeu, 0x1234c000u, 0x12341234u, 0x3c01fffeu, 0xfffec000u, 0x1234c000u, 0x12343c01u, 0x7ffffffeu, 0x7ffffffeu, 0xfffeffffu, 0x7fff7ffeu, 0x3c02fffeu, 0x3c02bfffu, 0x3c00fffeu, 0x1236fffeu, 0xfffefffeu, 0x12367ffeu, 0x1234c000u, 0x12341234u, 0xbc01c000u, 0x3c01c000u, 0x3c011234u, 0x3c014000u}, + {0x9c00a56eu, 0xcffef560u, 0x9c00a56eu, 0x795e4c00u, 0x2000a570u, 0xa5702000u, 0xc6480000u, 0xc6483014u, 0xc6480000u, 0x00000000u, 0xbe482468u, 0xbe48be48u, 0xbe482468u, 0x8e34547cu, 0x0000dcc0u, 0xcc004000u, 0x00840edcu, 0x0edc0084u, 0xff84fedcu, 0xff84ffedu, 0x42481234u, 0x12341234u, 0x8400edccu, 0x84008400u, 0x8400edccu, 0x8400edccu, 0x42481234u, 0x42481234u, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x4248edccu, 0x4248edcbu, 0xc2486dccu, 0x8a48c47fu, 0xc47fc47fu, 0x8a48c47fu, 0x8400edccu, 0x84008400u, 0x0400edccu, 0x42481234u, 0x12344248u, 0x84006dccu}, + {0x04023a00u, 0x3a000402u, 0x04023a00u, 0x38020600u, 0x00020200u, 0x02000002u, 0xfffd7dffu, 0xfffd7ffdu, 0xfffd7dffu, 0x7dff7dffu, 0xffff7e01u, 0xffffffffu, 0xffff7e01u, 0xfe017fffu, 0xc0007fffu, 0xc000ffffu, 0x00037fffu, 0x7fff0003u, 0xffff7fffu, 0xffff0001u, 0xffff7fffu, 0x7fff7fffu, 0xfffefe00u, 0xfffefffeu, 0xfffffe00u, 0xfffffffeu, 0xfffe7fffu, 0x7ffffe00u, 0xfffefe00u, 0xfffefe00u, 0xfe00fffeu, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0x7ffefe00u, 0xfffefe00u, 0xfe00fe00u, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0xfffe7e00u, 0xfffefe00u, 0xfe00fffeu, 0x7ffefe00u}, + {0x96003bb8u, 0xee00ebccu, 0x96003bb8u, 0x6bcc6e00u, 0x9800bdb8u, 0xbdb89800u, 0x9034c247u, 0x90349233u, 0x9034c247u, 0xc247c247u, 0x9434c249u, 0x94349434u, 0x9434c249u, 0xc4489235u, 0xe000ff00u, 0xfff00000u, 0x07e0007fu, 0x007f07e0u, 0x07e0007fu, 0x07e007ffu, 0x7e007fffu, 0x7fff7fffu, 0x12344248u, 0x12341234u, 0x7e007fffu, 0x7e007fffu, 0x12344248u, 0x12344248u, 0x7e007fffu, 0x7e007fffu, 0x7fff7e00u, 0xfe007fffu, 0x7e007fffu, 0x7e007fffu, 0x7e00ffffu, 0x7e007fffu, 0x7fff7fffu, 0x7e00ffffu, 0x12344248u, 0x12341234u, 0x1234c248u, 0x12344248u, 0x42481234u, 0x92344248u}, + {0x80000800u, 0x08006340u, 0x80000800u, 0xeb408000u, 0x00000000u, 0x00000000u, 0x35555240u, 0x35558b95u, 0x35555240u, 0x52405240u, 0x3555a5c0u, 0x35553555u, 0x3555a5c0u, 0xfc00df15u, 0x00005640u, 0xc8000000u, 0x00005640u, 0x56400000u, 0x00005640u, 0x000002b2u, 0x35555640u, 0x56405640u, 0x0000fc00u, 0x00000000u, 0x3555fc00u, 0xfc005640u, 0x00005640u, 0x35550000u, 0x0000fc00u, 0x0000fc00u, 0xfc000800u, 0x80007c00u, 0x3555fc00u, 0x35555645u, 0xb555fc00u, 0x0000fc00u, 0xfc00fc00u, 0x0000fc00u, 0x0000fc00u, 0x00000000u, 0x80005640u, 0x35555640u, 0x00005640u, 0x0000d640u}, + {0xdd70b2c8u, 0x611b56e4u, 0xdd70b2c8u, 0x9ee3191cu, 0xa57032c9u, 0x32c9a570u, 0x0000fa46u, 0x00006aefu, 0x0000fa46u, 0xfa46fa46u, 0xdb980000u, 0xdb98db98u, 0xdb980000u, 0x6aef70a9u, 0x4000e918u, 0x300091a0u, 0x00010fa4u, 0x0fa40001u, 0x00010fa4u, 0x00010007u, 0x12347d23u, 0x7d237d23u, 0xedcc7d23u, 0xedccedccu, 0xedcc7d23u, 0x7d23edccu, 0x12347d23u, 0x7d231234u, 0xc3fd7f23u, 0xc3fd7f23u, 0x7f237fffu, 0x44ffff23u, 0xedcc7f23u, 0xedcc7f23u, 0x6dcc7f23u, 0xc47f7f23u, 0x7f237f23u, 0xc47fff23u, 0xedcc7f23u, 0xedccedccu, 0xedccff23u, 0x12347f23u, 0x7f237f23u, 0x6dcc7f23u}, + {0xc000d640u, 0xd6413c00u, 0xc000d640u, 0x5640bc01u, 0x84008000u, 0x80008400u, 0xfbff7fffu, 0xfbfffc00u, 0xfbff7fffu, 0x7fff7fffu, 0xfc017fffu, 0xfc01fc01u, 0xfc017fffu, 0x8000fc00u, 0x7fff0000u, 0x80008000u, 0x7fff0001u, 0x00017fffu, 0x7fffffffu, 0x7fff8000u, 0x7fffffffu, 0xffffffffu, 0x7c008000u, 0x7c007c00u, 0x7fffffffu, 0xffff8000u, 0x7c008000u, 0x7c007fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffff7fffu, 0x7fffffffu, 0x7fff7c00u, 0x7fffffffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x7c008000u, 0x7c007c00u, 0x7c008000u, 0x7c008000u, 0xffff7c00u, 0xfc000000u}, + {0xfffecd56u, 0x38002153u, 0xfffecd56u, 0x5955fffeu, 0x00009556u, 0x95560000u, 0x3c013553u, 0x3c017156u, 0x3c013553u, 0x35533553u, 0x3c01caa9u, 0x3c013c01u, 0x3c01caa9u, 0xfffe06acu, 0x00004000u, 0x6aaa0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00001aaau, 0x3c013555u, 0x35553555u, 0x0000fffeu, 0x00000000u, 0x3c01fffeu, 0xfffe3c01u, 0x00003555u, 0x35550000u, 0xfffefffeu, 0xfffefffeu, 0xfffe3800u, 0xfffe7ffeu, 0x3c01fffeu, 0x3c013d56u, 0xbc01fffeu, 0x0000fffeu, 0xfffefffeu, 0x00007ffeu, 0x00003555u, 0x00000000u, 0x80003555u, 0x3c013555u, 0x00003c01u, 0x0000b555u}, + {0x31b86c40u, 0xd5b84840u, 0x31b86c40u, 0x6c40b1b8u, 0x1db83440u, 0x34401db8u, 0x3e478490u, 0x3e470490u, 0x3e478490u, 0x84908490u, 0x46490000u, 0x46494649u, 0x46490000u, 0xc6498000u, 0xff004800u, 0x4800ff00u, 0x007b0042u, 0x0042007bu, 0x007b0042u, 0x007b0042u, 0x7bff4248u, 0x42484248u, 0xc2484248u, 0xc248c248u, 0xc2484248u, 0x7bffc248u, 0x7bff4248u, 0x42484248u, 0xfc00492fu, 0xfc00492fu, 0x48effc00u, 0x7c00c8afu, 0x7bff4648u, 0x7bff0000u, 0x7bff0000u, 0xfc0048efu, 0x48ef48efu, 0xfc0048efu, 0xc2484248u, 0xc248c248u, 0xfbffc248u, 0x7bff4248u, 0x7bff4248u, 0x7bff4248u}, + {0x9a908000u, 0xdcbc4000u, 0x9a908000u, 0x80009cbcu, 0x5a900000u, 0x00005a90u, 0xdb98fe55u, 0xdb98edccu, 0xdb98fe55u, 0xfe55fe55u, 0x0000fe55u, 0x00000000u, 0x0000fe55u, 0x1089edccu, 0xc0000000u, 0x0000b980u, 0x000e0000u, 0x0000000eu, 0xfffe0000u, 0xfffe0000u, 0xedcc0000u, 0x00000000u, 0xedccfe55u, 0xedccedccu, 0xedccfe55u, 0xfe55edccu, 0xedcc0000u, 0x0000edccu, 0x7c00fe55u, 0x7c00fe55u, 0xfe557c00u, 0xfc007e55u, 0xf1ccfe55u, 0xf1ccedccu, 0x0000fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0xedcc0000u, 0xedccedccu, 0xedcc0000u, 0xedcc0000u, 0xedcc0000u, 0x6dcc8000u}, + {0x12344400u, 0x14000a34u, 0x12344400u, 0x0c001234u, 0x00003000u, 0x30000000u, 0xfffef1ccu, 0xfffe03feu, 0xfffef1ccu, 0xf1ccf1ccu, 0xfffee9ccu, 0xfffefffeu, 0xfffee9ccu, 0xedccfbfeu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000400u, 0x04000400u, 0xfffeedccu, 0xfffefffeu, 0xfffeedccu, 0xedccfffeu, 0x00000400u, 0x04000000u, 0xfffeb5c8u, 0xfffeb5c8u, 0xb5c9fffeu, 0xfffe35d0u, 0xfffeedccu, 0xfffefffeu, 0x7ffeedccu, 0xfffeb5ccu, 0xb5ccb5ccu, 0xfffeb5ccu, 0x0000edccu, 0x00000000u, 0x80000400u, 0x00000400u, 0x00000400u, 0x00008400u}, + {0x7fff9c00u, 0xbc00c69eu, 0x7fff9c00u, 0x029f7fffu, 0x0000e000u, 0xe0000000u, 0x7d235e55u, 0x7d237b78u, 0x7d235e55u, 0x5e555e55u, 0x7d2361abu, 0x7d237d23u, 0x7d2361abu, 0x60007eceu, 0x0000fe55u, 0xf2a80000u, 0x0000fe55u, 0xfe550000u, 0x0000fe55u, 0x0000ffcau, 0x7d236000u, 0x60006000u, 0x0000fe55u, 0x00000000u, 0x7d23fe55u, 0x6000fe55u, 0x00006000u, 0x7d230000u, 0x7f23fe55u, 0x7f23fe55u, 0xfe557f23u, 0x7f23fe55u, 0x7f23fe55u, 0x7f237f23u, 0xff237e55u, 0x7f23fe55u, 0xfe55fe55u, 0x7f237e55u, 0x7f236000u, 0x7f237f23u, 0x7f23e000u, 0x7f236000u, 0x60007f23u, 0xff236000u}, + {0x3c008400u, 0x84007c00u, 0x3c008400u, 0x84007c00u, 0xc0000000u, 0x0000c000u, 0x3c018000u, 0x3c017c01u, 0x3c018000u, 0x80008000u, 0xbc018000u, 0xbc01bc01u, 0xbc018000u, 0xc0007c01u, 0x80000000u, 0x0000c000u, 0x60000000u, 0x00006000u, 0xe0000000u, 0xe0000000u, 0x7c010000u, 0x00000000u, 0xc0008000u, 0xc000c000u, 0xc0008000u, 0xc0007c01u, 0x7c010000u, 0x00008000u, 0x7e018400u, 0x7e018400u, 0x7c007e01u, 0x7e018400u, 0x7e010000u, 0x7e017e01u, 0xfe018000u, 0x7e018000u, 0x80008000u, 0x7e018000u, 0x7e018000u, 0x7e017e01u, 0x7e010000u, 0x7e010000u, 0x80007e01u, 0xfe018000u}, + {0x40006fe8u, 0xaa007de8u, 0x40006fe8u, 0xe7e84000u, 0x2c00f1e8u, 0xf1e82c00u, 0xb155c09du, 0xb155f79du, 0xb155c09du, 0xc09dc09du, 0xb9553c0du, 0xb955b955u, 0xb9553c0du, 0x8255730du, 0x80004900u, 0x49008000u, 0x03e00612u, 0x061203e0u, 0x03e0fe12u, 0x03e0fe12u, 0x7c00fe55u, 0xfe55fe55u, 0x3555c248u, 0x35553555u, 0x7c00fe55u, 0xfe55c248u, 0x3555c248u, 0x35557c00u, 0x7c00fe55u, 0x7c00fe55u, 0xfe557e00u, 0xfc007e55u, 0x7c00fe55u, 0x7c00c19du, 0x7c00fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0x3555c248u, 0x35553555u, 0xfc00c248u, 0x7c00c248u, 0x7c003555u, 0x7c004248u}, + {0x755fffffu, 0x5e480e33u, 0x755fffffu, 0x4e341e47u, 0x7960c3ffu, 0xc3ff7960u, 0x30143c00u, 0x3014edcbu, 0x30143c00u, 0x3c003c00u, 0xab843c02u, 0xab84ab84u, 0xab843c02u, 0xf9b9edcdu, 0x8000fffeu, 0xf0008490u, 0x00047fffu, 0x7fff0004u, 0x0004ffffu, 0x0004ffffu, 0x42483c01u, 0x3c013c01u, 0xedccffffu, 0xedccedccu, 0xedccffffu, 0x4248ffffu, 0x42483c01u, 0xedcc3c01u, 0xfc00ffffu, 0xfc00ffffu, 0xfffff48du, 0xf9b8ffffu, 0xedcbffffu, 0xedcbffffu, 0x6dcd7fffu, 0xf48dffffu, 0xffffffffu, 0xf48d7fffu, 0xedcc3c01u, 0xedccedccu, 0xedccbc01u, 0x42483c01u, 0x4248edccu, 0x6dcc3c01u}, + {0x8132edccu, 0x70a9fe55u, 0x8132edccu, 0xedcc8132u, 0x82dd0000u, 0x000082ddu, 0x7d227d23u, 0x7d227d23u, 0x7d227d23u, 0x7d237d23u, 0x7d247d23u, 0x7d247d24u, 0x7d247d23u, 0x7d247d23u, 0xfff80000u, 0x0000fff8u, 0x1fff0000u, 0x00001fffu, 0xffff0000u, 0xffff0000u, 0x7d237d23u, 0x7d237d23u, 0xffff0000u, 0xffffffffu, 0xffff7d23u, 0xffff7d23u, 0x7d230000u, 0x00007d23u, 0x7f237f23u, 0x7f237f23u, 0x7f237f23u, 0x7f23ff23u, 0x7f237f23u, 0x7f237f23u, 0xff237f23u, 0x7f237f23u, 0x7f237f23u, 0x7f23ff23u, 0x7f237f23u, 0x7f237f23u, 0x7f23ff23u, 0x7f237f23u, 0x7f237f23u, 0xff237f23u}, + {0x00008000u, 0x80002000u, 0x00008000u, 0xa0000000u, 0x00000000u, 0x00000000u, 0xc2480400u, 0xc248c648u, 0xc2480400u, 0x04000400u, 0xc248fc00u, 0xc248c248u, 0xc248fc00u, 0x0000be48u, 0x00000400u, 0x00000000u, 0x00000400u, 0x04000000u, 0x00000400u, 0x00000004u, 0x00000400u, 0x04000400u, 0xc2480000u, 0xc248c248u, 0xc2480400u, 0x0000c248u, 0x00000000u, 0x04000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u, 0xc2480400u, 0xc248c248u, 0x42488400u, 0x80000000u, 0x00000000u, 0x80000000u, 0xc2480000u, 0xc248c248u, 0xc2488000u, 0x00000400u, 0x00000400u, 0x42480000u}, + {0xcaab5400u, 0x9634ac00u, 0xcaab5400u, 0x30001234u, 0xcaabd000u, 0xd000caabu, 0x3554e9ccu, 0x35543155u, 0x3554e9ccu, 0xe9cce9ccu, 0x3556f1ccu, 0x35563556u, 0x3556f1ccu, 0xedcd3955u, 0xffe00000u, 0x8000f000u, 0x07ff000fu, 0x000f07ffu, 0xffffffffu, 0xffffffe0u, 0x3555fc00u, 0xfc00fc00u, 0xffffedccu, 0xffffffffu, 0xfffffc00u, 0xfffffc00u, 0x3555edccu, 0x3555edccu, 0xffff7c00u, 0xffff7c00u, 0x7c00ffffu, 0x7ffffc00u, 0xfffffc00u, 0xfffffc00u, 0xffff7c00u, 0xffff7c00u, 0x7c007c00u, 0xffff7c00u, 0x3555fc00u, 0x35553555u, 0x3555fc00u, 0x3555edccu, 0xedcc3555u, 0xb5557c00u}, + {0x8400c000u, 0xa11c8400u, 0x8400c000u, 0xc000651cu, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x82dd1234u, 0x82dd82ddu, 0x82dd1234u, 0x95110000u, 0x7d230000u, 0x0000d230u, 0x7d230000u, 0x00007d23u, 0x7d230000u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x00000000u, 0x00001234u, 0x7f23c000u, 0x7f23c000u, 0x84007f23u, 0xff23c000u, 0x7f231234u, 0x7f230000u, 0x7f231234u, 0x7f230000u, 0x00000000u, 0x7f230000u, 0x7f230000u, 0x7f237f23u, 0xff239234u, 0x7f231234u, 0x7f230000u, 0x7f231234u}, + {0x7d557e55u, 0x7e557d55u, 0x7d557e55u, 0x7e557d55u, 0x00000000u, 0x00000000u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0x7c007c00u, 0x04007c00u, 0x04000400u, 0x04007c00u, 0x80000000u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0xfc000000u, 0x00007c00u, 0x7c007c00u, 0xfc000000u, 0xfc00fc00u, 0xfc007c00u, 0xfc000000u, 0x00000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0x0000fc00u, 0x00007c00u, 0x7c000000u, 0x80007c00u}, + {0xfe557d55u, 0x7d55fe55u, 0xfe557d55u, 0x7d55fe55u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00f800u, 0x7c007c00u, 0x7c007c00u, 0x7c008400u, 0x7c007c00u, 0x7c008400u, 0x00000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x7c007c00u, 0x7c007c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x00007c00u, 0x00000000u, 0x7c000000u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x00000000u, 0x00000000u, 0x80008000u, 0x7c007c00u, 0x00007c00u, 0x00000000u}, +}; +constexpr std::uint32_t ExpectedIeeeOff[32][44] = { + {0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0x80000000u, 0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80000000u, 0xfc007c00u, 0x7c007c00u, 0x80000000u, 0x80008000u, 0xfc007c00u, 0x8000fc00u, 0x80000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc00fc00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0xfc00fc00u, 0x80007c00u, 0x7c000000u, 0x7c007c00u}, + {0xbc007c00u, 0x7c00bc00u, 0xbc007c00u, 0x7c00bc00u, 0x00000000u, 0x00000000u, 0x78007800u, 0x7800f800u, 0x78007800u, 0x78007800u, 0x80008000u, 0x80008000u, 0x80008000u, 0x00000000u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c00fc00u, 0x7c007c00u, 0xfc007c00u, 0xfc00fe00u, 0xfc00fe00u, 0xfc00fe00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fc00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u}, + {0x7bffc000u, 0xc000fbffu, 0x7bffc000u, 0x7fff3c00u, 0x43ff0400u, 0x040043ffu, 0xbc00fbffu, 0xbc003c00u, 0xbc00fbffu, 0xfbfffbffu, 0xbc02fc01u, 0xbc02bc02u, 0xbc02fc01u, 0x7c013c02u, 0xfffeffffu, 0xfffe7fffu, 0x3fffffffu, 0xffff3fffu, 0x3fffffffu, 0x3fffffffu, 0x7fffffffu, 0xffffffffu, 0x3c01fc00u, 0x3c013c01u, 0x7fffffffu, 0xfc00ffffu, 0x3c01fc00u, 0x3c017fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x7fff7fffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x3c01fc00u, 0x3c013c01u, 0x3c017c00u, 0x3c01fc00u, 0xfc003c01u, 0xbc01fc00u}, + {0xc0009400u, 0x1400c000u, 0xc0009400u, 0x1400c000u, 0x00008000u, 0x80000000u, 0x1400ffffu, 0x14009400u, 0x1400ffffu, 0xffffffffu, 0x1400ffffu, 0x14001400u, 0x1400ffffu, 0x7fff9400u, 0x00000000u, 0x80000000u, 0x00000001u, 0x00010000u, 0x0000ffffu, 0x00008000u, 0x14007fffu, 0x7fff7fffu, 0x00008000u, 0x00000000u, 0x14008000u, 0x7fff8000u, 0x00007fffu, 0x14000000u, 0xc0007fffu, 0xc0007fffu, 0x7fff1400u, 0xc000ffffu, 0x14007fffu, 0x14001400u, 0x94007fffu, 0x00007fffu, 0x7fff7fffu, 0x0000ffffu, 0x00008000u, 0x00000000u, 0x80008000u, 0x14008000u, 0x00001400u, 0x00000000u}, + {0x7e00d5feu, 0x7ffe8000u, 0x7e00d5feu, 0x81fe7e00u, 0x8000d600u, 0xd6008000u, 0x7fff7c55u, 0x7fff7dffu, 0x7fff7c55u, 0x7c557c55u, 0x7fff8055u, 0x7fff7fffu, 0x7fff8055u, 0x7e5581ffu, 0x0000c000u, 0x00000000u, 0x000103f0u, 0x03f00001u, 0xffff03f0u, 0xffff0000u, 0xffff7e00u, 0x7e007e00u, 0x8000fe55u, 0x80008000u, 0xfffffe55u, 0xfe55ffffu, 0x80007e00u, 0x7e008000u, 0xfffffe55u, 0xfffffe55u, 0xfe55ffffu, 0xffff7e55u, 0xfffffe55u, 0xffffffffu, 0x7ffffe55u, 0xfffffe55u, 0xfe55fe55u, 0xffff7e55u, 0x8000fe55u, 0x80008000u, 0x00007e55u, 0x8000fe55u, 0x8000ffffu, 0x8000fe55u}, + {0x80007656u, 0x4254c3feu, 0x80007656u, 0x4254c3feu, 0x00017801u, 0x78010001u, 0xfffe7802u, 0xfffebc00u, 0xfffe7802u, 0x78027802u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc0243feu, 0x80007802u, 0x8000fffeu, 0x00001e00u, 0x1e000000u, 0x00001e00u, 0x00000000u, 0x7fff3c01u, 0x3c013c01u, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x3c013c01u, 0x7ffffe55u, 0x7ffffe55u, 0x7fff7fffu, 0x7ffffe55u, 0x7fff4001u, 0x7fff7fffu, 0xffff0000u, 0x7fff3c02u, 0x3c023c02u, 0x7fff3c02u, 0x7fff3c01u, 0x7fff7fffu, 0x7fffbc01u, 0x7fff3c01u, 0x3c013c01u, 0xffff3c01u}, + {0x7bff6000u, 0x60007bffu, 0x7bff6000u, 0x60007bffu, 0x00000000u, 0x00000000u, 0xedcc6dccu, 0xedcc6dccu, 0xedcc6dccu, 0x6dcc6dccu, 0xedcc6dccu, 0xedccedccu, 0xedcc6dccu, 0xedcc6dccu, 0x00000000u, 0x00000000u, 0x00000008u, 0x00080000u, 0x0000fff8u, 0x0000fff8u, 0x0000edccu, 0xedccedccu, 0xedcc8000u, 0xedccedccu, 0xedccedccu, 0xedccedccu, 0x00008000u, 0x80000000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0xedccedccu, 0xedccedccu, 0x6dccedccu, 0x80000000u, 0x00000000u, 0x80000000u, 0xedccedccu, 0xedccedccu, 0xedcc8000u, 0x00008000u, 0x00008000u, 0x6dcc0000u}, + {0x00001dccu, 0xedccd000u, 0x00001dccu, 0xbdcc0000u, 0x00003000u, 0x30000000u, 0x8000ce34u, 0x80009634u, 0x8000ce34u, 0xce34ce34u, 0x8800a9ccu, 0x88008800u, 0x8800a9ccu, 0xc00071ccu, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0xfc001234u, 0xfc001234u, 0x12341234u, 0x8400bc00u, 0x84008400u, 0xfc00bc00u, 0xfc008400u, 0x84001234u, 0x1234bc00u, 0x7c00edccu, 0x7c00edccu, 0x92347c00u, 0xfc00edccu, 0xfc00bbfeu, 0xfc0011b4u, 0xfc00bc01u, 0x7c009234u, 0x92349234u, 0x7c009234u, 0xfc00bc00u, 0xfc00fc00u, 0x84001234u, 0x84001234u, 0xbc001234u, 0x04009234u}, + {0xc99f5e41u, 0x4ceb74ddu, 0xc99f5e41u, 0xcd1df4abu, 0xcb9f0801u, 0x0801cb9fu, 0xb278f7feu, 0xb278f922u, 0xb278f7feu, 0xf7fef7feu, 0x47ce0000u, 0x47ce47ceu, 0x47ce0000u, 0x46aa0124u, 0xaaa88000u, 0xdff88000u, 0x06aa0000u, 0x000006aau, 0x06aa0000u, 0x06aa0f7fu, 0x7d237bffu, 0x7bff7bffu, 0x35557bffu, 0x35553555u, 0x7d237bffu, 0x7bff7d23u, 0x35557bffu, 0x7bff3555u, 0x7d237c00u, 0x7d237c00u, 0xfe007d23u, 0x7d23fc00u, 0x7d237c00u, 0x7d237d23u, 0xfd230000u, 0x7d237c00u, 0x7c007c00u, 0x7d237c00u, 0x35557bffu, 0x35553555u, 0xb555fbffu, 0x35557bffu, 0x7bff7bffu, 0x35557bffu}, + {0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80008000u, 0x80008000u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01fc01u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01b555u, 0x00000000u, 0x00000000u, 0x04004000u, 0x40000400u, 0xfc00c000u, 0xfc00fc00u, 0x35557c01u, 0x7c017c01u, 0x80008000u, 0x80008000u, 0x80008000u, 0x80008000u, 0x35557c01u, 0x35557c01u, 0x80007c01u, 0x80007c01u, 0x7c010000u, 0x0000fc01u, 0x35557c01u, 0x35553555u, 0xb5557c01u, 0x80007c01u, 0x7c017c01u, 0x8000fc01u, 0x80008000u, 0x80008000u, 0x00008000u, 0x35558000u, 0x80003555u, 0x80000000u}, + {0x03ff0000u, 0xa2480000u, 0x03ff0000u, 0x0000a248u, 0x03ff0000u, 0x000003ffu, 0xfc00c248u, 0xfc007fffu, 0xfc00c248u, 0xc248c248u, 0x03fec248u, 0x03fe03feu, 0x03fec248u, 0x46477fffu, 0x80000000u, 0x00000100u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fff0000u, 0x00000000u, 0x7c01c248u, 0x7c017c01u, 0x7fffc248u, 0xc2487fffu, 0x7c010000u, 0x00007c01u, 0x7fff0000u, 0x7fff0000u, 0x00007fffu, 0x7fff0000u, 0x7fffc248u, 0x7fff7fffu, 0xffffc248u, 0x7fff8000u, 0x80008000u, 0x7fff8000u, 0x7fffc248u, 0x7fff7fffu, 0x7fff0000u, 0x7fff0000u, 0xc2480000u, 0xffff8000u}, + {0xc2337fffu, 0x87fd7fffu, 0xc2337fffu, 0xffff07fdu, 0x42348000u, 0x80004234u, 0x4e35bffeu, 0x4e35d234u, 0x4e35bffeu, 0xbffebffeu, 0xd6333ffeu, 0xd633d633u, 0xd6333ffeu, 0xc3fd5234u, 0xc0100000u, 0x00004000u, 0x03c00003u, 0x000303c0u, 0x03c0ffffu, 0x03c0fc00u, 0x3c01fffeu, 0xfffefffeu, 0x1234c000u, 0x12341234u, 0x3c01fffeu, 0xfffec000u, 0x1234c000u, 0x12343c01u, 0x7ffffffeu, 0x7ffffffeu, 0xfffeffffu, 0x7fff7ffeu, 0x3c02fffeu, 0x3c02bfffu, 0x3c00fffeu, 0x1236fffeu, 0xfffefffeu, 0x12367ffeu, 0x1234c000u, 0x12341234u, 0xbc01c000u, 0x3c01c000u, 0x3c011234u, 0x3c014000u}, + {0x9c00a56eu, 0xcffef560u, 0x9c00a56eu, 0x795e4c00u, 0x2000a570u, 0xa5702000u, 0xc6480000u, 0xc6483014u, 0xc6480000u, 0x00000000u, 0xbe482468u, 0xbe48be48u, 0xbe482468u, 0x8e34547cu, 0x0000dcc0u, 0xcc004000u, 0x00840edcu, 0x0edc0084u, 0xff84fedcu, 0xff84ffedu, 0x42481234u, 0x12341234u, 0x8400edccu, 0x84008400u, 0x8400edccu, 0x8400edccu, 0x42481234u, 0x42481234u, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x4248edccu, 0x4248edcbu, 0xc2486dccu, 0x8a48c47fu, 0xc47fc47fu, 0x8a48c47fu, 0x8400edccu, 0x84008400u, 0x0400edccu, 0x42481234u, 0x12344248u, 0x84006dccu}, + {0x04023a00u, 0x3a000402u, 0x04023a00u, 0x38020600u, 0x00020200u, 0x02000002u, 0xfffd7dffu, 0xfffd7ffdu, 0xfffd7dffu, 0x7dff7dffu, 0xffff7e01u, 0xffffffffu, 0xffff7e01u, 0xfe017fffu, 0xc0007fffu, 0xc000ffffu, 0x00037fffu, 0x7fff0003u, 0xffff7fffu, 0xffff0001u, 0xffff7fffu, 0x7fff7fffu, 0xfffefe00u, 0xfffefffeu, 0xfffffe00u, 0xfffffffeu, 0xfffe7fffu, 0x7ffffe00u, 0xfffefe00u, 0xfffefe00u, 0xfe00fffeu, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0x7ffefe00u, 0xfffefe00u, 0xfe00fe00u, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0xfffe7e00u, 0xfffefe00u, 0xfe00fffeu, 0x7ffefe00u}, + {0x96003bb8u, 0xee00ebccu, 0x96003bb8u, 0x6bcc6e00u, 0x9800bdb8u, 0xbdb89800u, 0x9034c247u, 0x90349233u, 0x9034c247u, 0xc247c247u, 0x9434c249u, 0x94349434u, 0x9434c249u, 0xc4489235u, 0xe000ff00u, 0xfff00000u, 0x07e0007fu, 0x007f07e0u, 0x07e0007fu, 0x07e007ffu, 0x7e007fffu, 0x7fff7fffu, 0x12344248u, 0x12341234u, 0x7e007fffu, 0x7e007fffu, 0x12344248u, 0x12344248u, 0x7e007fffu, 0x7e007fffu, 0x7fff7e00u, 0xfe007fffu, 0x7e007fffu, 0x7e007fffu, 0x7e00ffffu, 0x7e007fffu, 0x7fff7fffu, 0x7e00ffffu, 0x12344248u, 0x12341234u, 0x1234c248u, 0x12344248u, 0x42481234u, 0x92344248u}, + {0x80000800u, 0x08006340u, 0x80000800u, 0xeb408000u, 0x00000000u, 0x00000000u, 0x35555240u, 0x35558b95u, 0x35555240u, 0x52405240u, 0x3555a5c0u, 0x35553555u, 0x3555a5c0u, 0xfc00df15u, 0x00005640u, 0xc8000000u, 0x00005640u, 0x56400000u, 0x00005640u, 0x000002b2u, 0x35555640u, 0x56405640u, 0x0000fc00u, 0x00000000u, 0x3555fc00u, 0xfc005640u, 0x00005640u, 0x35550000u, 0x0000fc00u, 0x0000fc00u, 0xfc000800u, 0x80007c00u, 0x3555fc00u, 0x35555645u, 0xb555fc00u, 0x0000fc00u, 0xfc00fc00u, 0x0000fc00u, 0x0000fc00u, 0x00000000u, 0x80005640u, 0x35555640u, 0x00005640u, 0x0000d640u}, + {0xdd70b2c8u, 0x611b56e4u, 0xdd70b2c8u, 0x9ee3191cu, 0xa57032c9u, 0x32c9a570u, 0x0000fa46u, 0x00006aefu, 0x0000fa46u, 0xfa46fa46u, 0xdb980000u, 0xdb98db98u, 0xdb980000u, 0x6aef70a9u, 0x4000e918u, 0x300091a0u, 0x00010fa4u, 0x0fa40001u, 0x00010fa4u, 0x00010007u, 0x12347d23u, 0x7d237d23u, 0xedcc7d23u, 0xedccedccu, 0xedcc7d23u, 0x7d23edccu, 0x12347d23u, 0x7d231234u, 0xc3fd7d23u, 0xc3fd7d23u, 0x7d237fffu, 0x44fffd23u, 0xedcc7d23u, 0xedcc7d23u, 0x6dcc7d23u, 0xc47f7d23u, 0x7d237d23u, 0xc47ffd23u, 0xedcc7d23u, 0xedccedccu, 0xedccfd23u, 0x12347d23u, 0x1234edccu, 0x6dcc7d23u}, + {0xc000d640u, 0xd6413c00u, 0xc000d640u, 0x5640bc01u, 0x84008000u, 0x80008400u, 0xfbff7fffu, 0xfbfffc00u, 0xfbff7fffu, 0x7fff7fffu, 0xfc017fffu, 0xfc01fc01u, 0xfc017fffu, 0x8000fc00u, 0x7fff0000u, 0x80008000u, 0x7fff0001u, 0x00017fffu, 0x7fffffffu, 0x7fff8000u, 0x7fffffffu, 0xffffffffu, 0x7c008000u, 0x7c007c00u, 0x7fffffffu, 0xffff8000u, 0x7c008000u, 0x7c007fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffff7fffu, 0x7fffffffu, 0x7fff7c00u, 0x7fffffffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x7c008000u, 0x7c007c00u, 0x7c008000u, 0x7c008000u, 0xffff7c00u, 0xfc000000u}, + {0xfffecd56u, 0x38002153u, 0xfffecd56u, 0x5955fffeu, 0x00009556u, 0x95560000u, 0x3c013553u, 0x3c017156u, 0x3c013553u, 0x35533553u, 0x3c01caa9u, 0x3c013c01u, 0x3c01caa9u, 0xfffe06acu, 0x00004000u, 0x6aaa0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00001aaau, 0x3c013555u, 0x35553555u, 0x0000fffeu, 0x00000000u, 0x3c01fffeu, 0xfffe3c01u, 0x00003555u, 0x35550000u, 0xfffefffeu, 0xfffefffeu, 0xfffe3800u, 0xfffe7ffeu, 0x3c01fffeu, 0x3c013d56u, 0xbc01fffeu, 0x0000fffeu, 0xfffefffeu, 0x00007ffeu, 0x00003555u, 0x00000000u, 0x80003555u, 0x3c013555u, 0x00003c01u, 0x0000b555u}, + {0x31b86c40u, 0xd5b84840u, 0x31b86c40u, 0x6c40b1b8u, 0x1db83440u, 0x34401db8u, 0x3e478490u, 0x3e470490u, 0x3e478490u, 0x84908490u, 0x46490000u, 0x46494649u, 0x46490000u, 0xc6498000u, 0xff004800u, 0x4800ff00u, 0x007b0042u, 0x0042007bu, 0x007b0042u, 0x007b0042u, 0x7bff4248u, 0x42484248u, 0xc2484248u, 0xc248c248u, 0xc2484248u, 0x7bffc248u, 0x7bff4248u, 0x42484248u, 0xfc00492fu, 0xfc00492fu, 0x48effc00u, 0x7c00c8afu, 0x7bff4648u, 0x7bff0000u, 0x7bff0000u, 0xfc0048efu, 0x48ef48efu, 0xfc0048efu, 0xc2484248u, 0xc248c248u, 0xfbffc248u, 0x7bff4248u, 0x7bff4248u, 0x7bff4248u}, + {0x9a908000u, 0xdcbc4000u, 0x9a908000u, 0x80009cbcu, 0x5a900000u, 0x00005a90u, 0xdb98fe55u, 0xdb98edccu, 0xdb98fe55u, 0xfe55fe55u, 0x0000fe55u, 0x00000000u, 0x0000fe55u, 0x1089edccu, 0xc0000000u, 0x0000b980u, 0x000e0000u, 0x0000000eu, 0xfffe0000u, 0xfffe0000u, 0xedcc0000u, 0x00000000u, 0xedccfe55u, 0xedccedccu, 0xedccfe55u, 0xfe55edccu, 0xedcc0000u, 0x0000edccu, 0x7c00fe55u, 0x7c00fe55u, 0xfe557c00u, 0xfc007e55u, 0xf1ccfe55u, 0xf1ccedccu, 0x0000fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0xedcc0000u, 0xedccedccu, 0xedcc0000u, 0xedcc0000u, 0xedcc0000u, 0x6dcc8000u}, + {0x12344400u, 0x14000a34u, 0x12344400u, 0x0c001234u, 0x00003000u, 0x30000000u, 0xfffef1ccu, 0xfffe03feu, 0xfffef1ccu, 0xf1ccf1ccu, 0xfffee9ccu, 0xfffefffeu, 0xfffee9ccu, 0xedccfbfeu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000400u, 0x04000400u, 0xfffeedccu, 0xfffefffeu, 0xfffeedccu, 0xedccfffeu, 0x00000400u, 0x04000000u, 0xfffeb5c8u, 0xfffeb5c8u, 0xb5c9fffeu, 0xfffe35d0u, 0xfffeedccu, 0xfffefffeu, 0x7ffeedccu, 0xfffeb5ccu, 0xb5ccb5ccu, 0xfffeb5ccu, 0x0000edccu, 0x00000000u, 0x80000400u, 0x00000400u, 0x00000400u, 0x00008400u}, + {0x7fff9c00u, 0xbc00c69eu, 0x7fff9c00u, 0x029f7fffu, 0x0000e000u, 0xe0000000u, 0x7d235e55u, 0x7d237b78u, 0x7d235e55u, 0x5e555e55u, 0x7d2361abu, 0x7d237d23u, 0x7d2361abu, 0x60007eceu, 0x0000fe55u, 0xf2a80000u, 0x0000fe55u, 0xfe550000u, 0x0000fe55u, 0x0000ffcau, 0x7d236000u, 0x60006000u, 0x0000fe55u, 0x00000000u, 0x7d23fe55u, 0x6000fe55u, 0x00006000u, 0x7d230000u, 0x7d23fe55u, 0x7d23fe55u, 0xfe557d23u, 0x7d23fe55u, 0x7d23fe55u, 0x7d237d23u, 0xfd237e55u, 0x7d23fe55u, 0xfe55fe55u, 0x7d237e55u, 0x00006000u, 0x00000000u, 0x8000e000u, 0x00006000u, 0x60007d23u, 0x00006000u}, + {0x3c008400u, 0x84007c00u, 0x3c008400u, 0x84007c00u, 0xc0000000u, 0x0000c000u, 0x3c018000u, 0x3c017c01u, 0x3c018000u, 0x80008000u, 0xbc018000u, 0xbc01bc01u, 0xbc018000u, 0xc0007c01u, 0x80000000u, 0x0000c000u, 0x60000000u, 0x00006000u, 0xe0000000u, 0xe0000000u, 0x7c010000u, 0x00000000u, 0xc0008000u, 0xc000c000u, 0xc0008000u, 0xc0007c01u, 0x7c010000u, 0x00008000u, 0x7c018400u, 0x7c018400u, 0x7c007c01u, 0x7c018400u, 0x7c010000u, 0x7c017c01u, 0xfc018000u, 0x7c018000u, 0x80008000u, 0x7c018000u, 0xc0008000u, 0xc000c000u, 0x40000000u, 0xc0000000u, 0x80000000u, 0xc0008000u}, + {0x40006fe8u, 0xaa007de8u, 0x40006fe8u, 0xe7e84000u, 0x2c00f1e8u, 0xf1e82c00u, 0xb155c09du, 0xb155f79du, 0xb155c09du, 0xc09dc09du, 0xb9553c0du, 0xb955b955u, 0xb9553c0du, 0x8255730du, 0x80004900u, 0x49008000u, 0x03e00612u, 0x061203e0u, 0x03e0fe12u, 0x03e0fe12u, 0x7c00fe55u, 0xfe55fe55u, 0x3555c248u, 0x35553555u, 0x7c00fe55u, 0xfe55c248u, 0x3555c248u, 0x35557c00u, 0x7c00fe55u, 0x7c00fe55u, 0xfe557e00u, 0xfc007e55u, 0x7c00fe55u, 0x7c00c19du, 0x7c00fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0x3555c248u, 0x35553555u, 0xfc00c248u, 0x7c00c248u, 0x7c003555u, 0x7c004248u}, + {0x755fffffu, 0x5e480e33u, 0x755fffffu, 0x4e341e47u, 0x7960c3ffu, 0xc3ff7960u, 0x30143c00u, 0x3014edcbu, 0x30143c00u, 0x3c003c00u, 0xab843c02u, 0xab84ab84u, 0xab843c02u, 0xf9b9edcdu, 0x8000fffeu, 0xf0008490u, 0x00047fffu, 0x7fff0004u, 0x0004ffffu, 0x0004ffffu, 0x42483c01u, 0x3c013c01u, 0xedccffffu, 0xedccedccu, 0xedccffffu, 0x4248ffffu, 0x42483c01u, 0xedcc3c01u, 0xfc00ffffu, 0xfc00ffffu, 0xfffff48du, 0xf9b8ffffu, 0xedcbffffu, 0xedcbffffu, 0x6dcd7fffu, 0xf48dffffu, 0xffffffffu, 0xf48d7fffu, 0xedcc3c01u, 0xedccedccu, 0xedccbc01u, 0x42483c01u, 0x4248edccu, 0x6dcc3c01u}, + {0x8132edccu, 0x70a9fe55u, 0x8132edccu, 0xedcc8132u, 0x82dd0000u, 0x000082ddu, 0x7d227d23u, 0x7d227d23u, 0x7d227d23u, 0x7d237d23u, 0x7d247d23u, 0x7d247d24u, 0x7d247d23u, 0x7d247d23u, 0xfff80000u, 0x0000fff8u, 0x1fff0000u, 0x00001fffu, 0xffff0000u, 0xffff0000u, 0x7d237d23u, 0x7d237d23u, 0xffff0000u, 0xffffffffu, 0xffff7d23u, 0xffff7d23u, 0x7d230000u, 0x00007d23u, 0x7d237d23u, 0x7d237d23u, 0x7d237d23u, 0x7d23fd23u, 0x7d237d23u, 0x7d237d23u, 0xfd237d23u, 0x7d237d23u, 0x7d237d23u, 0x7d23fd23u, 0x7d230000u, 0x7d237d23u, 0x7d230000u, 0x7d230000u, 0x7d230000u, 0xfd238000u}, + {0x00008000u, 0x80002000u, 0x00008000u, 0xa0000000u, 0x00000000u, 0x00000000u, 0xc2480400u, 0xc248c648u, 0xc2480400u, 0x04000400u, 0xc248fc00u, 0xc248c248u, 0xc248fc00u, 0x0000be48u, 0x00000400u, 0x00000000u, 0x00000400u, 0x04000000u, 0x00000400u, 0x00000004u, 0x00000400u, 0x04000400u, 0xc2480000u, 0xc248c248u, 0xc2480400u, 0x0000c248u, 0x00000000u, 0x04000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u, 0xc2480400u, 0xc248c248u, 0x42488400u, 0x80000000u, 0x00000000u, 0x80000000u, 0xc2480000u, 0xc248c248u, 0xc2488000u, 0x00000400u, 0x00000400u, 0x42480000u}, + {0xcaab5400u, 0x9634ac00u, 0xcaab5400u, 0x30001234u, 0xcaabd000u, 0xd000caabu, 0x3554e9ccu, 0x35543155u, 0x3554e9ccu, 0xe9cce9ccu, 0x3556f1ccu, 0x35563556u, 0x3556f1ccu, 0xedcd3955u, 0xffe00000u, 0x8000f000u, 0x07ff000fu, 0x000f07ffu, 0xffffffffu, 0xffffffe0u, 0x3555fc00u, 0xfc00fc00u, 0xffffedccu, 0xffffffffu, 0xfffffc00u, 0xfffffc00u, 0x3555edccu, 0x3555edccu, 0xffff7c00u, 0xffff7c00u, 0x7c00ffffu, 0x7ffffc00u, 0xfffffc00u, 0xfffffc00u, 0xffff7c00u, 0xffff7c00u, 0x7c007c00u, 0xffff7c00u, 0x3555fc00u, 0x35553555u, 0x3555fc00u, 0x3555edccu, 0xedcc3555u, 0xb5557c00u}, + {0x8400c000u, 0xa11c8400u, 0x8400c000u, 0xc000651cu, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x82dd1234u, 0x82dd82ddu, 0x82dd1234u, 0x95110000u, 0x7d230000u, 0x0000d230u, 0x7d230000u, 0x00007d23u, 0x7d230000u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x00000000u, 0x00001234u, 0x7d23c000u, 0x7d23c000u, 0x84007d23u, 0xfd23c000u, 0x7d231234u, 0x7d230000u, 0x7d231234u, 0x7d230000u, 0x00000000u, 0x7d230000u, 0x00000000u, 0x00000000u, 0x00009234u, 0x00001234u, 0x12340000u, 0x80001234u}, + {0x7d557e55u, 0x7e557d55u, 0x7d557e55u, 0x7e557d55u, 0x00000000u, 0x00000000u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0x7c007c00u, 0x04007c00u, 0x04000400u, 0x04007c00u, 0x80000000u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0xfc000000u, 0x00007c00u, 0x7c007c00u, 0xfc000000u, 0xfc00fc00u, 0xfc007c00u, 0xfc000000u, 0x00000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0x0000fc00u, 0x00007c00u, 0x7c000000u, 0x80007c00u}, + {0xfe557d55u, 0x7d55fe55u, 0xfe557d55u, 0x7d55fe55u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00f800u, 0x7c007c00u, 0x7c007c00u, 0x7c008400u, 0x7c007c00u, 0x7c008400u, 0x00000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x7c007c00u, 0x7c007c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x00007c00u, 0x00000000u, 0x7c000000u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x00000000u, 0x00000000u, 0x80008000u, 0x7c007c00u, 0x00007c00u, 0x00000000u}, +}; +constexpr const char* Names[44] = { + "v_pk_mad_i16 v4, v5, v6", + "v_pk_mad_i16 v4, v5, v6 op_sel:[1,0,1] op_sel_hi:[0,1,0]", + "v_pk_mad_u16 v4, v5, v6", + "v_pk_mad_u16 v4, v5, v6 op_sel:[0,1,1] op_sel_hi:[1,0,0]", + "v_pk_mul_lo_u16 v4, v5", + "v_pk_mul_lo_u16 v4, v5 op_sel:[1,1] op_sel_hi:[0,0]", + "v_pk_add_i16 v4, v5", + "v_pk_add_i16 v4, v5 op_sel:[1,0]", + "v_pk_add_u16 v4, v5", + "v_pk_add_u16 v4, v5 op_sel_hi:[0,0]", + "v_pk_sub_i16 v4, v5", + "v_pk_sub_i16 v4, v5 op_sel:[1,1]", + "v_pk_sub_u16 v4, v5", + "v_pk_sub_u16 v4, v5 op_sel:[1,0] op_sel_hi:[0,1]", + "v_pk_lshlrev_b16 v4, v5", + "v_pk_lshlrev_b16 v4, v5 op_sel:[0,1] op_sel_hi:[1,0]", + "v_pk_lshrrev_b16 v4, v5", + "v_pk_lshrrev_b16 v4, v5 op_sel:[1,1] op_sel_hi:[0,0]", + "v_pk_ashrrev_i16 v4, v5", + "v_pk_ashrrev_i16 v4, v5 op_sel:[1,0]", + "v_pk_max_i16 v4, v5", + "v_pk_max_i16 v4, v5 op_sel_hi:[0,0]", + "v_pk_min_i16 v4, v5", + "v_pk_min_i16 v4, v5 op_sel:[1,1]", + "v_pk_max_u16 v4, v5", + "v_pk_max_u16 v4, v5 op_sel:[1,0] op_sel_hi:[0,1]", + "v_pk_min_u16 v4, v5", + "v_pk_min_u16 v4, v5 op_sel:[0,1] op_sel_hi:[1,0]", + "v_pk_fmac_f16 v4, v5", + "v_pk_fma_f16 v4, v5, v6", + "v_pk_fma_f16 v4, v5, v6 op_sel:[1,1,0] op_sel_hi:[0,0,1]", + "v_pk_fma_f16 v4, v5, v6 neg_lo:[1,0,0] neg_hi:[0,1,0]", + "v_pk_add_f16 v4, v5", + "v_pk_add_f16 v4, v5 op_sel:[1,0]", + "v_pk_add_f16 v4, v5 neg_lo:[0,1] neg_hi:[1,0]", + "v_pk_mul_f16 v4, v5", + "v_pk_mul_f16 v4, v5 op_sel_hi:[0,0]", + "v_pk_mul_f16 v4, v5 neg_lo:[1,1]", + "v_pk_min_f16 v4, v5", + "v_pk_min_f16 v4, v5 op_sel:[1,1]", + "v_pk_min_f16 v4, v5 neg_lo:[1,0] neg_hi:[0,1]", + "v_pk_max_f16 v4, v5", + "v_pk_max_f16 v4, v5 op_sel:[1,0] op_sel_hi:[0,1]", + "v_pk_max_f16 v4, v5 neg_lo:[0,1] neg_hi:[1,0]", +}; + +bool Ieee = false; +bool FlushHalf = false; + +void Fill(std::uint32_t tid, std::uint32_t* words) { + std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words); +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) { + Require(actual == expected, std::string(Ieee ? "packed alu, IEEE mode: lane " : "packed alu: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + if (Ieee) request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; + if (FlushHalf) request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0x30u, true, false, false}; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* in = &Input[tid * Inputs]; + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t i = 0; i < 44; ++i) { + Expect(tid, out[i], (Ieee ? Expected : ExpectedIeeeOff)[tid][i], Names[i]); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + for (const bool ieee : {false, true}) { + Ieee = ieee; + Run(*device); + Check(); + } + Ieee = false; + FlushHalf = true; + bool refused = false; + try { + Run(*device); + } catch (const std::exception& error) { + refused = std::string(error.what()).find("f16 denormals not kept") != std::string::npos; + } + Require(refused, "packed alu: flushed f16 denormals were not refused"); + std::puts("packed alu tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PackedIntegerClamp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PackedIntegerClamp.cpp index 5ce6b0a7f..cf1a52000 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PackedIntegerClamp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PackedIntegerClamp.cpp @@ -125,7 +125,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Permlane16.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Permlane16.cpp index d66899aff..b73c7be79 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Permlane16.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Permlane16.cpp @@ -227,7 +227,7 @@ constexpr const char* Names[Checked] = {"permlane16_identity", "permlane16_rever std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelInterlock.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelInterlock.cpp new file mode 100644 index 000000000..ea1bd10b2 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelInterlock.cpp @@ -0,0 +1,237 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 32; +constexpr std::uint32_t Layers = 64; +constexpr std::size_t GuestBytes = 65536; +constexpr std::uint32_t FaultColumns = 8; +alignas(256) std::array Pixels{}; +alignas(256) std::array Counters{}; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +alignas(256) constexpr std::array PixelCode{ + 0x7e040f00, 0x7e060f01, 0x34060686, 0x4a040702, 0x34040482, + 0xe030d000, 0x80000402, 0xbf8c3f70, 0x4a080881, 0xe0705000, 0x80000402, + 0xbbfd0000, 0xbf900007, 0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000, +}; + +alignas(256) constexpr std::array FaultPixelCode{ + 0x7e040f00, 0x7e060f01, 0x36100487, 0x34060686, 0x4a040702, 0x34040482, 0x7e0a02ff, 0x00008000, + 0x7e0c02ff, 0x00010000, 0x7d841080, 0x02160d05, 0x4a161702, 0xdc319000, 0x0a04000b, 0xe030d000, + 0x80000402, 0xbf8c3f70, 0x4a080881, 0x4a08090a, 0xe0705000, 0x80000402, 0xdce08000, 0x00040402, + 0xbbfd0000, 0xbf900007, 0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000, +}; + +class GuestCounters { +public: + GuestCounters() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, 2 * GuestBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(GuestBytes, 2 * GuestBytes)); +#endif + Require(block != nullptr, "pixel interlock: cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, GuestBytes, true, true); + } + + ~GuestCounters() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + + GuestCounters(const GuestCounters&) = delete; + GuestCounters& operator=(const GuestCounters&) = delete; + + void Clear() { std::memset(block, 0, 2 * GuestBytes); } + std::uint64_t Address() const { return reinterpret_cast(block); } + std::uint32_t Word(std::size_t index) const { + std::uint32_t value = 0; + std::memcpy(&value, block + index * 4u, sizeof(value)); + return value; + } + +private: + std::uint8_t* block = nullptr; +}; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, stride == 0u ? 0x31016facu : 0x01016facu}; +} + +const std::vector>& Triangles() { + static const auto triangles = [] { + std::vector> vertices; + for (std::uint32_t layer = 0; layer < Layers; ++layer) { + vertices.push_back({-1.0f, -1.0f, 0.5f, 1.0f}); + vertices.push_back({3.0f, -1.0f, 0.5f, 1.0f}); + vertices.push_back({-1.0f, 3.0f, 0.5f, 1.0f}); + } + return vertices; + }(); + return triangles; +} + +void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, std::span pixelCode, std::span pixelUserData) { + Pixels.fill(std::byte{0}); + Counters.fill(0); + const auto target = device.Target(); + const auto& triangles = Triangles(); + + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(triangles.data(), 16u, static_cast(triangles.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.wave32 = waveSize == 32u; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + pixel.executeOnNoop = true; + pixel.orderedPixelShader = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + const std::array pixelMemory{{{reinterpret_cast(pixelCode.data()), std::as_bytes(pixelCode)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, + {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + {0, 0, vertexPush, 128 - vertexPush} + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, vertexPush} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(triangles.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); +} + +std::vector CounterUserData() { + std::vector userData(4, 0u); + const auto counters = BufferDescriptor(Counters.data(), 0u, static_cast(sizeof(Counters))); + std::copy(counters.begin(), counters.end(), userData.begin()); + return userData; +} + +void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize) { + Draw(device, waveSize, PixelCode, CounterUserData()); +} + +void DrawFault(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, GuestCounters& guest) { + const auto what = "pixel interlock wave" + std::to_string(waveSize) + " BDA fault"; + guest.Clear(); + auto pixelUserData = CounterUserData(); + pixelUserData.push_back(static_cast(guest.Address())); + pixelUserData.push_back(static_cast(guest.Address() >> 32u)); + std::string failure; + try { + Draw(device, waveSize, FaultPixelCode, pixelUserData); + } catch (const std::exception& error) { + failure = error.what(); + } + device.WaitIdle(); + Require(failure.find("BDA access failed") != std::string::npos && failure.find("reason=1") != std::string::npos, what + ": the draw reported \"" + failure + "\", not an unmapped read"); + for (std::uint32_t pixel = 0; pixel < Width * Height; ++pixel) { + const auto expected = pixel % FaultColumns == 0 ? 0u : Layers; + Require(guest.Word(pixel) == expected, what + ": pixel " + std::to_string(pixel) + " counted " + std::to_string(guest.Word(pixel)) + " of " + std::to_string(expected) + " overlapping read-modify-writes"); + Require(guest.Word(GuestBytes / 8u + pixel) == 0u && guest.Word(GuestBytes / 4u + pixel) == 0u, what + ": pixel " + std::to_string(pixel) + " stored into its read range"); + } +} + +void Check(std::uint32_t waveSize, std::uint32_t pass) { + const auto what = "pixel interlock wave" + std::to_string(waveSize) + " pass " + std::to_string(pass); + for (std::uint32_t pixel = 0; pixel < Width * Height; ++pixel) { + for (std::uint32_t channel = 0; channel < 4; ++channel) { + Require(Pixels[pixel * 4u + channel] == std::byte{255}, what + ": missing color at pixel " + std::to_string(pixel)); + } + Require(Counters[pixel] == Layers, what + ": pixel " + std::to_string(pixel) + " counted " + std::to_string(Counters[pixel]) + " of " + std::to_string(Layers) + " overlapping read-modify-writes"); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityFragmentShaderPixelInterlockEXT)) { + std::puts("skipped, the device lacks fragmentShaderPixelInterlock"); + return VulkanTestSkipped; + } + GuestCounters guest; + for (const auto waveSize : {64u, 32u}) { + for (std::uint32_t pass = 0; pass < 3; ++pass) { + Draw(*device, waveSize); + Check(waveSize, pass); + } + DrawFault(*device, waveSize, guest); + Draw(*device, waveSize); + Check(waveSize, 3); + } + std::printf("pixel interlock tests passed: 10 draws of %u overlapping triangles over %u pixels, 2 of them with a killed BDA fault in every %uth column\n", Layers, Width * Height, FaultColumns); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelWaveReduction.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelWaveReduction.cpp new file mode 100644 index 000000000..efdfff7d9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelWaveReduction.cpp @@ -0,0 +1,212 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Sentinel = 0xdeadbeefu; +constexpr std::uint32_t Width = 192; +constexpr std::uint32_t Height = 128; +alignas(256) std::array Pixels{}; +alignas(256) std::array Results{}; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +struct Reduction { + const char* name; + std::uint32_t identity; + std::uint32_t vector; + std::uint32_t scalar; + std::function combine; +}; + +std::uint32_t Key(std::uint32_t pixel) { + return (pixel * 0x9e3779b1u) >> 22u; +} + +std::vector PixelCode(const Reduction& reduction) { + const std::uint32_t dpp = reduction.vector | 0x001a1afau; + return { + 0x7e040f00, 0x7e060f01, 0xd5430006, 0x0409ff03, 0x000000c0, 0x7e120506, 0xd569000b, 0x0001ff06, + 0x9e3779b1, 0x2c161696, 0xbe98047e, 0x879a187e, 0xbeea287e, 0xd501000d, reduction.identity == 0u ? 0x006a1680u : 0x006a16c1u, + dpp, 0xff01110d, dpp, 0xff01120d, dpp, 0xff01140d, dpp, 0xff01180d, 0xd778100c, 0x0305830d, + reduction.vector | 0x001a190du, 0xbefe046a, 0xd760000a, 0x00013f0d, 0xd760000b, 0x00017f0d, reduction.scalar | 0x000c0b0au, + 0x8f099009, 0x880c090c, 0x7e0a020c, 0xe0702000, 0x80010506, 0x7e0e02f2, 0xf800180f, 0x07070707, + 0xbf810000, + }; +} + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +std::vector> Triangles() { + std::vector> vertices; + constexpr std::uint32_t columns = 6; + constexpr std::uint32_t rows = 4; + for (std::uint32_t cell = 0; cell < columns * rows; ++cell) { + const float width = 2.0f / columns; + const float height = 2.0f / rows; + const float x = -1.0f + width * static_cast(cell % columns); + const float y = -1.0f + height * static_cast(cell / columns); + const float skew = 0.07f * static_cast(cell % 5u); + vertices.push_back({x + (0.05f + skew) * width, y + 0.05f * height, 0.5f, 1.0f}); + vertices.push_back({x + 0.93f * width, y + (0.11f + skew) * height, 0.5f, 1.0f}); + vertices.push_back({x + (0.37f - skew * 0.5f) * width, y + 0.91f * height, 0.5f, 1.0f}); + } + return vertices; +} + +void Draw(AgcDriver::VulkanDevice& device, std::span pixelCode, const ShaderRecompiler::SpirvTarget& target) { + static const auto triangles = Triangles(); + Pixels.fill(std::byte{0}); + Results.fill(Sentinel); + + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(triangles.data(), 16u, static_cast(triangles.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {64u, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + std::vector pixelUserData(8, 0u); + const auto results = BufferDescriptor(Results.data(), 4u, static_cast(Results.size())); + std::copy(results.begin(), results.end(), pixelUserData.begin() + 4); + const std::array pixelMemory{{{reinterpret_cast(pixelCode.data()), std::as_bytes(pixelCode)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, + {64u, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + {0, 0, vertexPush, 128 - vertexPush} + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, vertexPush} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 64u, 64u, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(triangles.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); +} + +std::uint32_t Check(const Reduction& reduction, std::uint32_t subgroupSize) { + const auto what = std::string("pixel wave reduction ") + reduction.name; + std::map> waves; + for (std::uint32_t pixel = 0; pixel < Results.size(); ++pixel) { + if (std::to_integer(Pixels[pixel * 4u]) != 255u) { + Require(Results[pixel] == Sentinel, what + ": uncovered pixel " + std::to_string(pixel) + " stored " + std::to_string(Results[pixel])); + continue; + } + Require(Results[pixel] != Sentinel, what + ": covered pixel " + std::to_string(pixel) + " stored nothing"); + waves[Results[pixel] >> 16u].push_back(pixel); + } + std::uint32_t partial = 0; + std::size_t covered = 0; + for (const auto& [first, pixels] : waves) { + Require(std::find(pixels.begin(), pixels.end(), first) != pixels.end(), what + ": the first lane of the wave of pixel " + std::to_string(pixels.front()) + ", pixel " + std::to_string(first) + ", is not a covered pixel of that wave"); + std::uint32_t expected = reduction.identity; + for (const auto pixel : pixels) expected = reduction.combine(expected, Key(pixel)); + for (const auto pixel : pixels) { + Require((Results[pixel] & 0xffffu) == expected, what + ": pixel " + std::to_string(pixel) + " of a wave of " + std::to_string(pixels.size()) + " covered pixels read " + std::to_string(Results[pixel] & 0xffffu) + ", the wave holds " + std::to_string(expected)); + } + if (pixels.size() < subgroupSize) ++partial; + covered += pixels.size(); + } + Require(covered > 1000, what + ": the triangles covered only " + std::to_string(covered) + " pixels"); + Require(partial != 0u, what + ": no wave had fewer covered pixels than the host subgroup"); + return partial; +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const auto target = device->Target(); + if (!TargetHasCapability(target, spv::CapabilityGroupNonUniformArithmetic)) { + std::puts("skipped, the device has no subgroup arithmetic"); + return VulkanTestSkipped; + } + if (target.subgroupSize > 64u) { + std::printf("skipped, the %u-lane subgroup is wider than the wave, so the half-wave scan throws\n", target.subgroupSize); + return VulkanTestSkipped; + } + std::vector layouts; + if (target.subgroupSize >= 32u) layouts.push_back(target); + else std::printf("the device's own layout is skipped, v_permlanex16_b32 reads lanes 16-31, outside the %u-lane subgroup\n", target.subgroupSize); + if (target.subgroupSize < 64u) { + auto wide = target; + wide.subgroupSize = 64u; + layouts.push_back(wide); + } + const std::array reductions{{ + {"UMin", 0xffffffffu, 0x26000000u, 0x83800000u, [](std::uint32_t left, std::uint32_t right) { return std::min(left, right); }}, + {"UMax", 0u, 0x28000000u, 0x84800000u, [](std::uint32_t left, std::uint32_t right) { return std::max(left, right); }}, + {"IAdd", 0u, 0x4a000000u, 0x80000000u, [](std::uint32_t left, std::uint32_t right) { return left + right; }}, + {"AND", 0xffffffffu, 0x36000000u, 0x87000000u, [](std::uint32_t left, std::uint32_t right) { return left & right; }}, + {"OR", 0u, 0x38000000u, 0x88000000u, [](std::uint32_t left, std::uint32_t right) { return left | right; }}, + }}; + for (const auto& layout : layouts) { + for (const auto& reduction : reductions) { + const auto code = PixelCode(reduction); + std::uint32_t partial = 0; + for (std::uint32_t pass = 0; pass < 4; ++pass) { + Draw(*device, code, layout); + partial += Check(reduction, target.subgroupSize); + } + std::printf("%s, laid out for %u lanes on %u-lane subgroups: %u partial waves over 4 draws\n", reduction.name, layout.subgroupSize, target.subgroupSize, partial); + } + } + std::puts("pixel wave reduction tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedExports.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedExports.cpp new file mode 100644 index 000000000..b0b85a7b7 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedExports.cpp @@ -0,0 +1,123 @@ +#include +#include "VulkanTestDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "Optimization/ResourceProgram.hpp" +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; +using AgcDriver::Graphics::Require; + +constexpr std::uint32_t Extent = 16; +alignas(256) std::array pixels{}; +alignas(256) std::array otherPixels{}; +constexpr std::array, 3> positions{{{-1, -1, 0, 1}, {3, -1, 0, 1}, {-1, 3, 0, 1}}}; +constexpr std::array vertexCode{0xe0382000u, 0x80000005u, 0xbf8c3f70u, 0xf80008cfu, 0x03020100u, 0xbf810000u}; + +void Run(AgcDriver::VulkanDevice& device, bool integer) { + const auto address = reinterpret_cast(positions.data()); + const std::array users{static_cast(address), static_cast(address >> 32u) | (16u << 16u), 3, 0x01016facu}; + RecompileRequest vertexRequest{{ShaderStage::Vertex, 0x10000, vertexCode, 0, {}}, {32, 0, users, {}, {}, ShaderVertexStageInfo{}, {}}, device.Target(), {0, 0, 0, 128}}; + const auto vertex = Recompile(vertexRequest); + const std::array values = integer ? std::array{11, 22, 33, 44} : std::array{0, 0x3e800000u, 0x3f000000u, 0x3f800000u}; + const std::array code{0x7e0002ffu, values[0], 0x7e0202ffu, values[1], 0x7e0402ffu, values[2], 0x7e0602ffu, values[3], 0xf800100fu, 0x03020100u, 0xf800183fu, 0x03020100u, 0xbf810000u}; + AgcDriver::Registers context{{0x1b6, 0x8000}, {0x1b3, 0}, {0x1b4, 0}, {0x203, 0}, {0x1c5, integer ? 0x7007u : 0x9009u}}; + RecompileRequest request{{ShaderStage::Fragment, 0x20000, code, 0, {}}, {32, 0, {}, {}, AgcDriver::Graphics::DecodePixelStageInfo(context, {}), {}, {}}, device.Target(), {0, 0, 0, 128}}; + const auto handle = PrepareShader(request); + const auto& artifact = GetPreparedArtifact(*handle); + std::uint32_t scalarType = 0; + unsigned extracts = 0; + const auto& words = artifact.spirv.Words(); + for (std::size_t offset = 5; offset < words.size(); offset += words[offset] >> 16u) { + const auto opcode = words[offset] & 0xffffu; + if (integer && opcode == spv::OpTypeInt && words[offset + 2] == 32 && words[offset + 3] == 0) scalarType = words[offset + 1]; + if (!integer && opcode == spv::OpTypeFloat && words[offset + 2] == 32) scalarType = words[offset + 1]; + if (opcode == spv::OpVectorExtractDynamic) { + Require(scalarType != 0 && words[offset + 1] == scalarType, "runtime export mapping uses the wrong scalar type"); + ++extracts; + } + } + Require(extracts == 8, "fragment did not map both MRT outputs at runtime"); + AgcDriver::DriverDetail::ShaderSnapshot snapshot{0x20000, 0, 1, {code.begin(), code.end()}, {}}; + snapshot.prepared->entries.push_back({0, handle}); + request.shader.code = snapshot.code; + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0, 32, 32, {}, {}}; + state.color = {reinterpret_cast(pixels.data()), {Extent, Extent}, integer ? VK_FORMAT_R32G32B32A32_UINT : VK_FORMAT_R32G32B32A32_SFLOAT, sizeof(pixels), 0}; + state.color.elementBytes = 16; + auto other = state.color; + other.address = reinterpret_cast(otherPixels.data()); + other.slot = 3; + other.exportIndex = 3; + state.colors = {state.color, other}; + state.hasColorTarget = true; + state.renderExtent = {Extent, Extent}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Extent), static_cast(Extent), -static_cast(Extent), 0, 1}; + state.scissor = {{0, 0}, {Extent, Extent}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends.resize(4); + state.blends[0] = state.blends[3] = state.blend; + const AgcDriver::Pm4::DrawParameters draw{0, 3, 0, 1, 0, false}; + std::shared_ptr first; + for (const auto mapping : {0xe4u, 0xe5u, 0xe6u, 0xe7u, 0x1bu, 0u, 0xe4u}) { + state.colors[0].componentMapping = mapping; + state.colors[1].componentMapping = mapping ^ 0xffu; + request.context.pixel = AgcDriver::Graphics::DecodePixelStageInfo(context, AgcDriver::Graphics::ExportMappings(state)); + const auto invocation = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request); + SrtRuntime runtime{}; + const auto capture = invocation.Capture(runtime); + const auto pixel = invocation.Materialize(*capture); + Require(pixel->variantId == artifact.variantId && pixel->specialization.empty(), "export mapping did not materialize a prepared fragment module"); + if (!first) first = pixel; + else if (mapping == 0xe4u) Require(pixel == first, "export mapping did not reuse the materialized result"); + else Require(pixel != first, "export mapping reused stale materialized data"); + for (const bool cached : {true, false}) { + request.useCache = cached; + const auto materialized = MaterializeShader(request, *capture, *handle); + Require(materialized->spirv.data() == pixel->spirv.data() && materialized->PipelineVariantId() == pixel->PipelineVariantId(), "export mapping did not reuse its specialized module"); + for (std::size_t cursor = 5; cursor < materialized->spirv.size();) { + const auto count = materialized->spirv[cursor] >> 16u; + Require(count != 0u && count <= materialized->spirv.size() - cursor, "invalid specialized fragment instruction"); + Require((materialized->spirv[cursor] & 0xffffu) != static_cast(spv::OpVectorExtractDynamic), "fragment export retained a dynamic channel selection"); + cursor += count; + } + } + request.useCache = true; + if (integer) continue; + const std::array shaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::Fragment, pixel.get(), 0}}}; + pixels.fill(0xdeadbeef); + otherPixels.fill(0xdeadbeef); + device.Draw(state, draw, shaders); + device.WaitIdle(); + for (std::size_t i = 0; i < pixels.size(); ++i) { + const auto shift = (i % 4) * 2; + Require(pixels[i] == values[(mapping >> shift) & 3u], "fragment exported the wrong channel"); + Require(otherPixels[i] == values[((mapping ^ 0xffu) >> shift) & 3u], "fragment used the wrong MRT mapping"); + } + } +} + +} + +int main() { + try { + auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, false); + Run(*device, true); + std::cout << "prepared fragment export mapping tests passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp new file mode 100644 index 000000000..770819753 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp @@ -0,0 +1,254 @@ +#include "VulkanTestDevice.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp" +#include "SceShaders.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" +#include "Optimization/ResourceProgram.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void Require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +template +void Reject(TAction action, const char* expected) { + try { action(); } + catch (const std::exception& error) { + Require(std::string(error.what()).find(expected) != std::string::npos, error.what()); + return; + } + throw std::runtime_error(std::string("invalid graphics ABI was accepted; expected: ") + expected); +} + +struct Fixture { + struct Header { + Shader shader{}; + ShaderUserData users{}; + } header; + alignas(256) std::array code{}; + std::shared_ptr snapshot; + + void Initialize(std::uint8_t type) { + header.shader.type = type; + header.shader.user_data = &header.users; + code.fill(0xffffffffu); + code.back() = 0xbf810000u; + const auto address = reinterpret_cast(code.data()); + const auto headerAddress = reinterpret_cast(&header); + snapshot = std::make_shared(); + snapshot->type = type; + snapshot->codeAddress = address; + snapshot->headerAddress = headerAddress; + snapshot->code.assign(code.begin(), code.end()); + snapshot->header.resize(sizeof(header)); + std::memcpy(snapshot->header.data(), &header, sizeof(header)); + } + + void Bind(AgcDriver::QueueState& queue, std::uint32_t program, std::uint32_t resources) const { + const auto address = snapshot->codeAddress + 256u; + queue.shader[program] = static_cast(address >> 8u); + queue.shader[program + 1u] = static_cast(address >> 40u); + queue.shader[resources] = 16u << 1u; + } +}; + +void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path, const std::filesystem::path& dump) { + using namespace AgcDriver::DriverDetail; + using namespace ShaderRecompiler; + Fixture front; + Fixture back; + Fixture domain; + Fixture fragment; + const bool tessellation = path == AgcDriver::Graphics::ShaderPath::Tessellation; + const bool mesh = path == AgcDriver::Graphics::ShaderPath::Geometry; + front.Initialize(tessellation ? 5u : mesh ? 4u : 2u); + back.Initialize(tessellation ? 7u : 6u); + domain.Initialize(2u); + fragment.Initialize(1u); + const std::array pixelCode{0xc8020002u, 0xc8060102u, 0xc80a0202u, 0xc80e0302u, 0xf800180fu, 0x03020100u, 0xbf810000u}; + fragment.snapshot->code.resize(64); + fragment.snapshot->code.insert(fragment.snapshot->code.end(), pixelCode.begin(), pixelCode.end()); + ShaderRegistry registry; + for (const auto* fixture : {&front, &back, &domain, &fragment}) registry.emplace(fixture->snapshot->codeAddress, fixture->snapshot); + AgcDriver::QueueState queue{}; + queue.context[0x8e] = 0xfu; + queue.context[0x8f] = 0xfu; + front.Bind(queue, tessellation ? 0x148u : 0xc8u, tessellation ? 0x10bu : 0x8bu); + if (tessellation || mesh) back.Bind(queue, tessellation ? 0x108u : 0x88u, tessellation ? 0x10bu : 0x8bu); + if (tessellation) domain.Bind(queue, 0xc8u, 0x8bu); + fragment.Bind(queue, 0x8u, 0xbu); + alignas(8) const std::array merged{}; + const auto mergedAddress = reinterpret_cast(merged.data()); + const auto pointerBase = tessellation ? 0x102u : 0x82u; + queue.shader[pointerBase] = static_cast(mergedAddress); + queue.shader[pointerBase + 1u] = static_cast(mergedAddress >> 32u); + DrawDecode prepared{}; + prepared.state.stages.path = path; + prepared.state.stages.vertexWaveSize = tessellation || mesh ? 64u : 32u; + prepared.state.stages.fragmentWaveSize = 32u; + if (mesh) prepared.state.stages.mesh = MeshConfiguration{4, 1, 3, 3, 1, 64, 128, 0, 4}; + if (tessellation) prepared.state.stages.tessellation = TessellationConfiguration{3, 4, 1, 2, 2}; + prepared.pixel.wave32 = true; + prepared.pixel.interpolatorCount = 2; + prepared.pixel.interpolatorSettings[0] = 0x403u; + prepared.pixel.interpolatorSettings[1] = 0x220u; + prepared.pixel.targetOutputMode[0] = 9; + prepared.pixel.targetExportMapping.fill(0xe4u); + DecodeGraphicsPrograms(prepared, queue, registry, true, true); + DrawDecode draw{}; + draw.state = prepared.state; + draw.pixel = prepared.pixel; + DecodeGraphicsPrograms(draw, queue, registry, false, true); + Require(prepared.programs.size() == draw.programs.size() && prepared.roles == draw.roles, "prepared graphics programs differ from draw programs"); + for (std::size_t index = 0; index < prepared.programs.size(); ++index) { + const auto& expected = prepared.programs[index]; + const auto& actual = draw.programs[index]; + Require(expected.codeOffset == 64u && expected.binary.code.size() == (prepared.roles[index] == ProgramRole::Fragment ? pixelCode.size() : 1u) && expected.binary.code[0] == (prepared.roles[index] == ProgramRole::Fragment ? pixelCode[0] : 0xbf810000u), "graphics entry point did not trim the code prefix"); + Require(expected.binary.stage == actual.binary.stage && expected.firstUserSgpr == actual.firstUserSgpr && expected.userData.size() == actual.userData.size(), "graphics preparation changed the user SGPR ABI"); + } + auto target = device.Target(); + std::vector capabilities(target.supportedCapabilities.begin(), target.supportedCapabilities.end()); + std::vector extensions(target.supportedExtensions.begin(), target.supportedExtensions.end()); + if (mesh) { + capabilities.push_back(spv::CapabilityMeshShadingEXT); + extensions.push_back("SPV_EXT_mesh_shader"); + target.supportedCapabilities = capabilities; + target.supportedExtensions = extensions; + target.mesh = MeshTargetLimits{{128, 1, 1}, 128, 32768, 256, 256, 128, 32768, 1, 1}; + } + const auto stages = PrepareGraphicsStages(prepared, target); + Require(stages.size() == (tessellation ? 4u : 2u), "graphics preparation compiled the wrong stages"); + for (const auto& stage : stages) stage.snapshot->prepared->entries.push_back(stage.entry); + const auto& pixelArtifact = GetPreparedArtifact(*stages.back().entry.handle); + Require(!pixelArtifact.fragmentParameters.empty() && pixelArtifact.fragmentParameters.front().sourceLocation == 3u, "prepared fragment lost interpolant mapping"); + if (!mesh && !tessellation) { + const auto frontCount = front.snapshot->prepared->entries.size(); + const auto pixelCount = fragment.snapshot->prepared->entries.size(); + Reject([&] { + ShaderPreparationTransaction transaction; + transaction.Edit(*front.snapshot).entries.push_back(stages.front().entry); + transaction.Edit(*fragment.snapshot).entries.push_back(stages.back().entry); + auto unsupported = target; + unsupported.tessellation.reset(); + ResolvePreparedGraphics(*front.snapshot, fragment.snapshot, 7, unsupported); + transaction.Commit(); + }, "tessellation shaders are unavailable"); + Require(front.snapshot->prepared->entries.size() == frontCount && fragment.snapshot->prepared->entries.size() == pixelCount && front.snapshot->prepared->rectangles.empty() && front.snapshot->prepared->fragments.empty() && !front.snapshot->prepared->rectangleRequested, "failed rectangle compilation published part of the stage group"); + for (const bool primitiveFirst : {false, true}) { + front.snapshot->prepared->rectangles.clear(); + front.snapshot->prepared->rectangleProgress.clear(); + front.snapshot->prepared->fragments.clear(); + front.snapshot->prepared->rectangleRequested = false; + if (primitiveFirst) ResolvePreparedGraphics(*front.snapshot, {}, 7, target); + ResolvePreparedGraphics(*front.snapshot, fragment.snapshot, 0, target); + if (!primitiveFirst) ResolvePreparedGraphics(*front.snapshot, {}, 17, target); + const auto vertexId = GetPreparedArtifact(*stages.front().entry.handle).variantId; + const auto rectangle = PreparedRectangle(*front.snapshot, vertexId, pixelArtifact.variantId); + Require(!rectangle.control.spirv.empty() && !rectangle.evaluation.spirv.empty(), "separate helpers did not prepare rectangle shaders"); + const auto count = front.snapshot->prepared->rectangles.size(); + ResolvePreparedGraphics(*front.snapshot, fragment.snapshot, 7, target); + Require(front.snapshot->prepared->rectangles.size() == count, "repeated helpers duplicated rectangle shaders"); + } + auto changed = prepared; + changed.state.stages.vertexWaveSize = 64; + changed.pixel.interpolatorSettings[0] = 0x404u; + const auto newStages = PrepareGraphicsStages(changed, target); + front.snapshot->prepared->entries.push_back(newStages.front().entry); + ResolvePreparedGraphics(*front.snapshot, {}, 7, target); + const auto newVertexId = GetPreparedArtifact(*newStages.front().entry.handle).variantId; + Require(newVertexId != GetPreparedArtifact(*stages.front().entry.handle).variantId, "rectangle test did not change the vertex variant"); + static_cast(PreparedRectangle(*front.snapshot, newVertexId, pixelArtifact.variantId)); + fragment.snapshot->prepared->entries.push_back(newStages.back().entry); + ResolvePreparedGraphics(*front.snapshot, fragment.snapshot, 0, target); + const auto newPixelId = GetPreparedArtifact(*newStages.back().entry.handle).variantId; + Require(newPixelId != pixelArtifact.variantId, "rectangle test did not change the fragment variant"); + static_cast(PreparedRectangle(*front.snapshot, newVertexId, newPixelId)); + auto temporary = std::make_shared(); + temporary->prepared->entries = fragment.snapshot->prepared->entries; + ResolvePreparedGraphics(*front.snapshot, temporary, 0, target); + std::weak_ptr expired = temporary; + temporary.reset(); + Require(expired.expired(), "helper link retained a shader snapshot"); + ResolvePreparedGraphics(*front.snapshot, {}, 7, target); + Require(front.snapshot->prepared->fragments.size() == 1, "expired helper link was not removed"); + Require(front.snapshot->prepared->rectangleProgress.size() == 1 && !front.snapshot->prepared->rectangleProgress.front().fragment.expired(), "expired rectangle progress was not removed"); + } + if (!dump.empty()) { + for (std::size_t index = 0; index < stages.size(); ++index) { + const auto& words = GetPreparedArtifact(*stages[index].entry.handle).spirv; + const auto name = dump / (std::to_string(static_cast(path)) + "-" + std::to_string(index) + ".spv"); + std::ofstream output(name, std::ios::binary); + output.write(reinterpret_cast(words.data()), static_cast(words.size() * sizeof(std::uint32_t))); + Require(static_cast(output), "cannot write prepared graphics SPIR-V"); + } + } + auto incomplete = prepared; + incomplete.roles.pop_back(); + Reject([&] { static_cast(PrepareGraphicsStages(incomplete, target)); }, "program roles are incomplete"); + auto wrongEntry = prepared; + wrongEntry.programs.front().binary.codeAddress += 4; + Reject([&] { static_cast(PrepareGraphicsStages(wrongEntry, target)); }, "entry point differs"); + std::vector linked; + std::vector memory; + for (std::size_t index = 0; index < draw.programs.size(); ++index) { + const auto& program = draw.programs[index]; + linked.push_back({draw.roles[index], program.binary, program.userDataBase, program.firstUserSgpr, program.userData}); + memory.insert(memory.end(), program.memory.begin(), program.memory.end()); + } + for (std::size_t index = 0; index < draw.programs.size(); ++index) { + if (draw.roles[index] == ProgramRole::GeometryBack) continue; + const auto& program = draw.programs[index]; + const bool pixel = program.binary.stage == ShaderStage::Fragment; + std::optional vertex; + if (!pixel) vertex = AgcDriver::Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, nullptr, true); + RecompileRequest request{program.binary, {pixel ? 32u : prepared.state.stages.vertexWaveSize, program.firstUserSgpr, program.userData, {}, pixel ? std::optional(draw.pixel) : std::nullopt, vertex, memory}, target, {0, 0, 0, mesh ? MeshDrawPushOffsetBytes : 128u}, GraphicsCompileContext{program.firstUserSgpr, linked, prepared.state.stages.mesh, prepared.state.stages.tessellation, {0, 3, 4, 1}}}; + static_cast(InvocationFor(*program.snapshot, program.codeOffset, request)); + request.layout.pushConstantOffsetBytes = 4; + request.layout.pushConstantSizeBytes -= 4; + static_cast(InvocationFor(*program.snapshot, program.codeOffset, request)); + request.layout.pushConstantOffsetBytes = 2; + Reject([&] { static_cast(InvocationFor(*program.snapshot, program.codeOffset, request)); }, "artifact is missing"); + request.layout = {0, 0, 0, mesh ? MeshDrawPushOffsetBytes : 128u}; + if (pixel) request.context.pixel->interpolatorSettings[0] ^= 1u; + else if (tessellation) ++request.graphics->tessellation->outputControlPoints; + else if (mesh) ++request.graphics->mesh->maxVertices; + else ++request.context.userDataBaseRegister; + Reject([&] { static_cast(InvocationFor(*program.snapshot, program.codeOffset, request)); }, "artifact is missing"); + } + ShaderRegistry invalidRegistry; + DrawDecode invalid{}; + invalid.state = prepared.state; + Reject([&] { DecodeGraphicsPrograms(invalid, queue, invalidRegistry, true, true); }, "registered"); + const auto frontBase = tessellation ? 0x148u : 0xc8u; + queue.shader[frontBase + 1u] |= 0x100u; + Reject([&] { DecodeGraphicsPrograms(invalid, queue, registry, true, true); }, "reserved graphics program address"); +} + +} + +int main(int argc, char** argv) { + try { + auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Require(argc <= 2, "invalid test arguments"); + const auto dump = argc == 2 ? std::filesystem::path(argv[1]) : std::filesystem::path{}; + Check(*device, AgcDriver::Graphics::ShaderPath::Vertex, dump); + Check(*device, AgcDriver::Graphics::ShaderPath::Geometry, dump); + Check(*device, AgcDriver::Graphics::ShaderPath::Tessellation, dump); + std::cout << "prepared graphics ABI tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedPublication.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedPublication.cpp new file mode 100644 index 000000000..99f9c45a3 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedPublication.cpp @@ -0,0 +1,203 @@ +#include "VulkanTestDevice.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "Optimization/ResourceProgram.hpp" +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace AgcDriver::DriverDetail; +using namespace ShaderRecompiler; + +void Require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +template +void Reject(TAction action) { + bool failed = false; + try { action(); } catch (const std::runtime_error&) { failed = true; } + Require(failed, "failed shader transaction was accepted"); +} + +void Transactions() { + ShaderSnapshot front{}; + auto pixel = std::make_shared(); + { + ShaderPreparationTransaction transaction; + transaction.Edit(front).entries.push_back({1, {}}); + { + ShaderPreparationTransaction nested; + nested.Edit(*pixel).entries.push_back({2, {}}); + nested.Commit(); + } + Require(front.prepared->entries.empty() && pixel->prepared->entries.empty(), "nested preparation published a partial group"); + } + Require(front.prepared->entries.empty() && pixel->prepared->entries.empty(), "aborted preparation published artifacts"); + { + ShaderPreparationTransaction transaction; + transaction.Edit(front).rectangleRequested = true; + { + ShaderPreparationTransaction nested; + nested.Edit(*pixel).rectangleRequested = true; + } + Reject([&] { transaction.Commit(); }); + } + Require(!front.prepared->rectangleRequested && !pixel->prepared->rectangleRequested, "nested abort did not roll back the group"); + Reject([&] { ResolvePreparedGraphics(front, pixel, 7, {}); }); + Require(!front.prepared->rectangleRequested && front.prepared->fragments.empty(), "failed rectangle preparation published a link"); + const auto rectangle = std::make_shared(); + front.prepared->rectangles.push_back({1, 2, rectangle}); + { + ShaderPreparationTransaction transaction; + auto& prepared = transaction.Edit(front); + Require(prepared.rectangles.front().shaders == rectangle, "transaction copied an immutable rectangle artifact"); + { + ShaderPreparationTransaction nested; + Require(&nested.Edit(front) == &prepared, "nested transaction did not reuse root changes"); + nested.Edit(*pixel).rectangleRequested = true; + nested.Commit(); + } + Require(!pixel->prepared->rectangleRequested, "nested commit published before root commit"); + transaction.Commit(); + } + Require(pixel->prepared->rectangleRequested && front.prepared->rectangles.front().shaders == rectangle, "root commit lost nested changes or shared artifacts"); + std::weak_ptr staged; + { + auto temporary = std::make_shared(); + staged = temporary->prepared; + ShaderPreparationTransaction transaction; + transaction.Edit(*temporary).rectangleRequested = true; + temporary.reset(); + Require(!staged.expired(), "pending publication released its destination"); + transaction.Commit(); + } + Require(staged.expired(), "completed publication retained its destination"); + std::barrier start(3); + std::atomic partial = false; + std::jthread reader([&](std::stop_token stop) { + start.arrive_and_wait(); + while (!stop.stop_requested()) { + std::scoped_lock lock(front.prepared->mutex, pixel->prepared->mutex); + if (front.prepared->entries.size() != pixel->prepared->entries.size()) partial = true; + } + }); + const auto write = [&] { + start.arrive_and_wait(); + for (unsigned i = 0; i < 100; ++i) { + ShaderPreparationTransaction transaction; + transaction.Edit(front).entries.push_back({i, {}}); + std::this_thread::yield(); + transaction.Edit(*pixel).entries.push_back({i, {}}); + transaction.Commit(); + } + }; + auto first = std::async(std::launch::async, write); + auto second = std::async(std::launch::async, write); + first.get(); + second.get(); + reader.request_stop(); + reader.join(); + Require(!partial && front.prepared->entries.size() == 200 && pixel->prepared->entries.size() == 200, "concurrent group publication was partial or lost an update"); +} + +void Registration(AgcDriver::VulkanDevice& device) { + const std::array code{0xbf810000u}; + RecompileRequest request{{ShaderStage::Compute, 0x10000, code, 0, {}}, {32, 0, {}, ShaderComputeStageInfo{{1, 1, 1}, 0, {false, false, false}, false, 0, {}}, {}, {}, {}}, device.ComputeTarget(32), {0, 0, 0, 128}}; + const auto handle = PrepareShader(request); + const auto makeSnapshot = [&] { + auto snapshot = std::make_shared(); + snapshot->codeAddress = 0x10000; + snapshot->headerAddress = 0x20000; + snapshot->code.assign(code.begin(), code.end()); + return snapshot; + }; + auto original = makeSnapshot(); + original->prepared->entries.push_back({0, handle}); + original->prepared->rectangleRequested = true; + std::shared_ptr registry; + PublishRegisteredShader(registry, original); + std::barrier start(4); + std::vector> workers; + for (unsigned i = 0; i < 4; ++i) { + workers.push_back(std::async(std::launch::async, [&] { + start.arrive_and_wait(); + for (unsigned iteration = 0; iteration < 40; ++iteration) { + ShaderPreparationTransaction transaction; + PublishRegisteredShader(registry, makeSnapshot()); + const auto current = registry->at(0x10000); + Require(current == original && current->prepared->rectangleRequested, "identical registration replaced a prepared snapshot"); + auto& entries = transaction.Edit(*current).entries; + entries.push_back({iteration + 1, handle}); + transaction.Commit(); + } + })); + } + for (auto& worker : workers) worker.get(); + Require(original->prepared->entries.size() == 161, "re-registration lost prepared artifacts"); + std::shared_ptr submission = registry; + std::weak_ptr old = original; + auto replacement = makeSnapshot(); + replacement->headerAddress += 256; + std::promise registering; + auto started = registering.get_future(); + std::future replacementWriter; + { + ShaderPreparationTransaction transaction; + transaction.Edit(*original).entries.push_back({999, handle}); + replacementWriter = std::async(std::launch::async, [&] { + registering.set_value(); + PublishRegisteredShader(registry, replacement); + }); + started.get(); + Require(replacementWriter.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout, "replacement interleaved with an unfinished preparation"); + Require(original->prepared->entries.size() == 161, "unfinished preparation changed a live snapshot"); + transaction.Commit(); + } + replacementWriter.get(); + Require(registry->at(0x10000) == replacement && submission->at(0x10000) == original, "replacement modified an in-flight registry"); + Require(original->prepared->entries.size() == 162 && replacement->prepared->entries.empty(), "stale preparation was published into a replacement snapshot"); + original.reset(); + request.shader.code = submission->at(0x10000)->code; + const auto invocation = InvocationFor(*submission->at(0x10000), 0, request); + const auto capture = invocation.Capture({}); + const auto result = invocation.Materialize(*capture); + Require(result->variantId == GetPreparedArtifact(*handle).variantId, "old submission lost its prepared artifact"); + device.Dispatch(*result, 1, 1, 1); + device.WaitIdle(); + Require(!old.expired(), "in-flight submission released its snapshot early"); + submission.reset(); + Require(old.expired(), "completed submission retained an obsolete snapshot"); + Require(invocation.Materialize(*capture)->spirv.data() == result->spirv.data(), "prepared handle did not retain its artifact"); + auto changed = makeSnapshot(); + changed->headerAddress = replacement->headerAddress; + changed->code[0] ^= 1; + PublishRegisteredShader(registry, changed); + Require(registry->at(0x10000) == changed, "changed shader code was treated as identical"); + auto metadata = makeSnapshot(); + metadata->headerAddress = changed->headerAddress; + metadata->code = changed->code; + metadata->header.push_back(std::byte{1}); + PublishRegisteredShader(registry, metadata); + Require(registry->at(0x10000) == metadata, "changed shader metadata was treated as identical"); +} + +} + +int main() { + try { + Transactions(); + auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Registration(*device); + std::cout << "shader publication and lifetime tests passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp new file mode 100644 index 000000000..977f26c1a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp @@ -0,0 +1,287 @@ +#include "VulkanTestDevice.hpp" +#include "Optimization/ResourceProgram.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Submit/include/Acb.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void Require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +template +void ExpectFailure(TAction action, const char* expected) { + try { action(); } + catch (const std::exception& error) { + Require(std::string(error.what()).find(expected) != std::string::npos, error.what()); + return; + } + throw std::runtime_error("expected preparation failure"); +} + +void RunUnregistered(AgcDriver::VulkanDevice& device, ShaderRecompiler::RecompileRequest request) { + const auto code = request.shader.code; + for (const bool sourceFirst : {false, true}) { + AgcDriver::DriverDetail::ShaderSnapshot snapshot{request.shader.codeAddress, 0, 0, {code.begin(), code.end()}, {}}; + request.shader.code = snapshot.code; + if (sourceFirst) { + const auto handle = AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request); + Require(snapshot.prepared->entries.size() == 1 && snapshot.prepared->entries.front().handle == handle, "unregistered compute source was not cached"); + } + static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request)); + Require(snapshot.prepared->entries.size() == 1, "unregistered compute invocation did not cache exactly one artifact"); + const auto handle = snapshot.prepared->entries.front().handle; + const auto& artifact = ShaderRecompiler::GetPreparedArtifact(*handle); + for (const bool useCache : {false, true}) { + request.useCache = useCache; + Require(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request) == handle, "unregistered compute source was prepared again"); + const auto repeated = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request); + ShaderRecompiler::SrtRuntime runtime{}; + runtime.userData = request.context.userData; + const auto capture = repeated.Capture(runtime); + const auto result = repeated.Materialize(*capture); + Require(snapshot.prepared->entries.size() == 1 && result->variantId == artifact.variantId && result->spirv.data() == artifact.spirv.data(), "unregistered compute invocation replaced its cached artifact"); + device.Dispatch(*result, 1, 1, 1); + } + device.WaitIdle(); + } +} + +void Run(AgcDriver::VulkanDevice& device) { + alignas(256) std::array code{0xbf810000u}; + std::array users{}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{1, 1, 1}, 0, {false, false, false}, false, 1, {}}; + ShaderRecompiler::RecompileRequest request{{ShaderRecompiler::ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, users, compute, {}, {}, {}}, device.ComputeTarget(32), {0, 0, 0, 128}}; + RunUnregistered(device, request); + AgcDriver::DriverDetail::ShaderSnapshot snapshot{request.shader.codeAddress, 0, 0, {code.begin(), code.end()}, {}}; + snapshot.header.resize(sizeof(Shader)); + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request)); }, "artifact is missing"); + const auto handle = ShaderRecompiler::PrepareShader(request); + snapshot.prepared->entries.push_back({0, handle}); + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 1, request)); }, "artifact is missing"); + const auto& artifact = ShaderRecompiler::GetPreparedArtifact(*handle); + for (std::uint32_t value = 0; value < 8; ++value) { + users.fill(value); + request.useCache = value % 2 == 0; + const auto ready = AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request); + ShaderRecompiler::SrtRuntime runtime{}; + runtime.userData = users; + const auto capture = ShaderRecompiler::CaptureResources(request, runtime, *ready); + const auto result = ShaderRecompiler::MaterializeShader(request, *capture, *ready); + Require(result->variantId == artifact.variantId && result->spirv.data() == artifact.spirv.data(), "invocation replaced the prepared artifact"); + device.Dispatch(*result, 1, 1, 1); + } + device.WaitIdle(); + auto registeredRequest = request; + registeredRequest.shader.code = snapshot.code; + registeredRequest.useCache = true; + const auto invocation = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, registeredRequest); + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request)); }, "does not refer to registered code"); + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 1, registeredRequest)); }, "outside the snapshot"); + registeredRequest.context.compute->numThreads[0] = 2; + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 0, registeredRequest)); }, "artifact is missing"); + ShaderRecompiler::SrtRuntime preparedRuntime{}; + preparedRuntime.userData = users; + const auto preparedCapture = invocation.Capture(preparedRuntime); + const auto firstResult = invocation.Materialize(*preparedCapture); + const auto repeatedCapture = invocation.Capture(preparedRuntime); + Require(invocation.Materialize(*repeatedCapture) == firstResult, "prepared invocation rebuilt an unchanged materialized result"); + users[0] += 1; + const auto changedCapture = invocation.Capture(preparedRuntime); + const auto changedResult = invocation.Materialize(*changedCapture); + Require(changedResult != firstResult && changedResult->spirv.data() == firstResult->spirv.data(), "prepared invocation did not distinguish changed runtime data"); + users[0] -= 1; + Require(invocation.Materialize(*preparedCapture) == firstResult, "prepared invocation evicted the previous runtime data"); + auto replacement = OpenVulkanTestDevice(); + Require(replacement != nullptr && replacement->Serial() != device.Serial(), "replacement device was not created"); + auto replacementRequest = request; + replacementRequest.target = replacement->ComputeTarget(32); + Require(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, replacementRequest) == handle, "device replacement discarded a compatible prepared artifact"); + replacementRequest.shader.code = snapshot.code; + const auto replacementInvocation = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, replacementRequest); + const auto replacementCapture = replacementInvocation.Capture(preparedRuntime); + const auto replacementResult = replacementInvocation.Materialize(*replacementCapture); + Require(replacementResult->spirv.data() == artifact.spirv.data(), "device replacement recompiled the prepared shader"); + replacement->Dispatch(*replacementResult, 1, 1, 1); + replacement->WaitIdle(); + replacementRequest.target.nonConstantImageOffsets = !replacementRequest.target.nonConstantImageOffsets; + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 0, replacementRequest)); }, "artifact is missing"); + ShaderRecompiler::SrtRuntime runtime{}; + runtime.userData = users; + const auto capture = ShaderRecompiler::CaptureResources(request, runtime, *handle); + request.context.compute->numThreads[0] = 2; + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request)); }, "artifact is missing"); + ExpectFailure([&] { static_cast(ShaderRecompiler::CaptureResources(request, runtime, *handle)); }, "does not match the static ABI"); + ExpectFailure([&] { static_cast(ShaderRecompiler::MaterializeShader(request, *capture, *handle)); }, "does not match the static ABI"); + const auto otherHandle = ShaderRecompiler::PrepareShader(request); + const auto otherCapture = ShaderRecompiler::CaptureResources(request, runtime, *otherHandle); + ExpectFailure([&] { static_cast(invocation.Materialize(*otherCapture)); }, "another prepared shader"); + ExpectFailure([&] { static_cast(ShaderRecompiler::MaterializeShader(request, *capture, *otherHandle)); }, "another prepared shader"); + request.context.compute->numThreads[0] = 1; + request.layout.pushConstantSizeBytes = 124; + Require(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request) == handle, "compatible push constant capacity discarded the prepared artifact"); + request.layout.pushConstantSizeBytes = 126; + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request)); }, "artifact is missing"); + request.layout.pushConstantSizeBytes = 128; + code[0] = 0xffffffffu; + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request)); }, "artifact is missing"); +} + +void PrepareMultisampledStorage(AgcDriver::VulkanDevice& device) { + using namespace ShaderRecompiler; + std::array code{0xd7460000u, 0x0401060cu, 0xd7460001u, 0x0405060du, 0x7e04020eu, 0x7e060280u, 0x7e080208u, 0x7e0a0209u, 0x7e0c020au, 0x7e0e020bu, 0xf0200f38u, 0x00000400u, 0xbf810000u}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x30000u, code, 0, {}}; + const std::array users{}; + request.context.userData = users; + request.context.waveSize = 32; + request.context.compute = ShaderComputeStageInfo{{8, 8, 1}, 0, {true, true, false}, false, 2, {}}; + request.target = device.ComputeTarget(32); + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + for (std::uint32_t variant = 0; variant < 5; ++variant) { + if (variant != 0u) { + code[10] = variant % 2u != 0u ? 0xf0200f38u : 0xf0200f30u; + if (variant >= 3u) code[10] = variant == 3u ? 0xf03c0138u : 0xf03c0130u; + code[5] = 0x7e060283u; + code[4] = 0x7e040282u; + } + const auto handle = PrepareShader(request); + const auto& artifact = GetPreparedArtifact(*handle); + std::vector samples; + for (std::size_t offset = 5; offset < artifact.spirv.size(); offset += artifact.spirv[offset] >> 16u) { + const auto instruction = artifact.spirv[offset]; + if ((instruction & 0xffffu) == spv::OpImageWrite) { + Require(variant < 3u && (instruction >> 16u) == 6u && artifact.spirv[offset + 4] == spv::ImageOperandsSampleMask, "MSAA storage write lost its sample operand"); + samples.push_back(artifact.spirv[offset + 5]); + } + if ((instruction & 0xffffu) == spv::OpImageTexelPointer) { + Require(variant >= 3u && (instruction >> 16u) == 6u, "MSAA atomic lost its sample operand"); + samples.push_back(artifact.spirv[offset + 5]); + } + } + Require(!samples.empty(), "MSAA storage shader has no image writes or atomics"); + const auto expectedSample = variant == 0u ? 0u : variant % 2u != 0u ? 3u : 2u; + for (const auto sample : samples) { + bool found = false; + for (std::size_t offset = 5; offset < artifact.spirv.size(); offset += artifact.spirv[offset] >> 16u) { + if ((artifact.spirv[offset] & 0xffffu) != spv::OpConstant || artifact.spirv[offset + 2] != sample) continue; + Require(artifact.spirv[offset + 3] == expectedSample, "MSAA storage operation addresses the wrong sample"); + found = true; + } + Require(found, "MSAA sample constant is missing"); + } + } + std::vector capabilities(request.target.supportedCapabilities.begin(), request.target.supportedCapabilities.end()); + std::erase(capabilities, spv::CapabilityStorageImageMultisample); + request.target.supportedCapabilities = capabilities; + ExpectFailure([&] { static_cast(PrepareShader(request)); }, "storage image multisampling is unavailable on the target device"); +} + +void Registration(bool indirect) { + alignas(256) std::array code{0xbf810000u}; + struct Header { + Shader shader{}; + std::array registers{}; + std::array context{{{0x1b6, 0}, {0x1b6, 1}}}; + ShaderSpecialRegs specials{}; + } header; + const auto address = reinterpret_cast(code.data()); + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18; + header.shader.header_size = sizeof(header); + header.shader.shader_size = sizeof(code); + header.shader.code = code.data(); + header.shader.sh_registers = header.registers.data(); + header.shader.num_sh_registers = header.registers.size(); + header.shader.cx_registers = header.context.data(); + header.shader.num_cx_registers = header.context.size(); + header.shader.specials = &header.specials; + header.specials.dispatch_modifier = 0x8000; + header.registers = {{{0x20c, static_cast(address >> 8u)}, {0x20d, static_cast(address >> 40u)}, {0x207, 2}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 0}, {0x207, 1}, {0x207, 1}}}; + const auto threadRegisterIndex = header.registers.size() - 1; + AgcDriverRegisterShader_nid_postfix(&header.shader); + std::barrier start(4); + std::vector> registrations; + for (unsigned worker = 0; worker < 4; ++worker) { + registrations.push_back(std::async(std::launch::async, [&] { + start.arrive_and_wait(); + for (unsigned iteration = 0; iteration < 8; ++iteration) { + AgcDriverRegisterShader_nid_postfix(&header.shader); + AgcDriverResolveShaderAbi_nid_postfix(&header.shader, {}, {}); + } + })); + } + for (auto& registration : registrations) registration.get(); + header.registers[1].value |= 0x100u; + ExpectFailure([&] { AgcDriverRegisterShader_nid_postfix(&header.shader); }, "invalid registered program address"); + header.registers[1].value &= 0xffu; + ++header.registers[0].value; + ExpectFailure([&] { AgcDriverRegisterShader_nid_postfix(&header.shader); }, "entry point is outside shader code"); + --header.registers[0].value; + header.shader.num_sh_registers = 255; + ExpectFailure([&] { AgcDriverRegisterShader_nid_postfix(&header.shader); }, "truncated shader metadata"); + header.shader.num_sh_registers = header.registers.size(); + header.registers[threadRegisterIndex].value = 0; + ExpectFailure([&] { AgcDriverRegisterShader_nid_postfix(&header.shader); }, "must be nonzero"); + header.registers[threadRegisterIndex].value = 1; + code[0] = 0xffffffffu; + ExpectFailure([&] { AgcDriverRegisterShader_nid_postfix(&header.shader); }, ""); + std::vector commands; + for (const auto reg : header.registers) commands.insert(commands.end(), {0xc0017600u, reg.offset, reg.value}); + const std::array arguments{1, 1, 1}; + const auto argumentAddress = reinterpret_cast(arguments.data()); + if (indirect) commands.insert(commands.end(), {0xc0021600u, static_cast(argumentAddress), static_cast(argumentAddress >> 32u), 0x8041}); + else commands.insert(commands.end(), {0xc0031500u, 1, 1, 1, 0x8041}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + for (std::uint32_t iteration = 0; iteration < 3; ++iteration) { + sceAgcDriverSubmitAcb(0x20, &packet); + AgcDriverWaitIdle_nid_postfix(); + } + std::uint32_t destination = 0; + const auto destinationAddress = reinterpret_cast(&destination); + commands[threadRegisterIndex * 3 + 2] = 2; + commands.insert(commands.end(), {0xc0033700u, 0x00100200u, static_cast(destinationAddress), static_cast(destinationAddress >> 32u), 7}); + packet = Packet{commands.data(), static_cast(commands.size()), 0, {}}; + sceAgcDriverSubmitAcb(0x20, &packet); + ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "artifact is missing"); + ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "artifact is missing"); + ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "artifact is missing"); + Require(destination == 0, "failed dispatch executed a subsequent memory write"); +} + +} + +int main(int argc, char** argv) { + try { + Require(argc == 1 || (argc == 2 && (std::string_view(argv[1]) == "--indirect" || std::string_view(argv[1]) == "--fail-before-registration")), "invalid test arguments"); + auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Require(argc != 2 || std::string_view(argv[1]) != "--fail-before-registration", "injected failure before registration"); + PrepareMultisampledStorage(*device); + device.reset(); + Registration(argc == 2); + std::cout << "prepared shader and transactional registration tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + try { + AgcDriverShutdown_nid_postfix(); + } catch (const std::exception& shutdownError) { + std::cerr << shutdownError.what() << '\n'; + } + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ReadFirstLane.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ReadFirstLane.cpp index d8bbea7e0..b0f3e0af9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ReadFirstLane.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ReadFirstLane.cpp @@ -31,7 +31,7 @@ alignas(256) constexpr std::array ReadFirstLaneWave64Code{ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device, std::span code, std::uint32_t waveSize) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ReadLane.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ReadLane.cpp index ff98068ca..159f2705d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ReadLane.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ReadLane.cpp @@ -26,7 +26,7 @@ alignas(256) constexpr std::array ReadLaneCode{ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/RelativeIndexing.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/RelativeIndexing.cpp index 0c2d2e688..8b2ab8f12 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/RelativeIndexing.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/RelativeIndexing.cpp @@ -79,7 +79,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/RuntimeImages.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/RuntimeImages.cpp new file mode 100644 index 000000000..b2937b0ca --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/RuntimeImages.cpp @@ -0,0 +1,60 @@ +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; + +alignas(256) std::array, 2> textures{}; +alignas(256) std::array output{}; +constexpr std::array code{0x7e3c02ffu, 0x3fa00000u, 0x7e3e02ffu, 0x3f000000u, 0xf09c0108u, 0x00610a1eu, 0x34060082u, 0xe0701000u, 0x80000a03u, 0xbf810000u}; + +void Run(AgcDriver::VulkanDevice& device) { + ShaderRecompiler::CompiledShaderArtifact first; + const std::array formats{20u, 21u, 22u, 34u}; + for (std::uint32_t iteration = 0u; iteration < 17u; ++iteration) { + const auto format = formats[iteration % formats.size()]; + const bool clamped = (iteration & 4u) != 0u; + const bool swizzled = (iteration & 8u) != 0u; + auto& texels = textures[iteration % textures.size()]; + for (std::uint32_t pixel = 0u; pixel < 32u; ++pixel) { + texels[pixel] = format == 22u ? std::bit_cast(static_cast(pixel + 100u)) : format == 21u ? 0u - pixel - 100u : format == 34u ? (pixel + 100u) | ((pixel + 200u) << 11u) | ((pixel + 300u) << 22u) : pixel + 100u; + } + output.fill(0xdeadbeefu); + const auto bufferAddress = reinterpret_cast(output.data()); + const auto imageAddress = reinterpret_cast(texels.data()); + const auto swizzle = swizzled ? (format == 34u ? 0xfadu : 0xfa9u) : 0xfacu; + std::array userData{static_cast(bufferAddress), static_cast(bufferAddress >> 32u), 256u, 0x31016facu, static_cast(imageAddress >> 8u), static_cast(imageAddress >> 40u) | (format << 20u) | (3u << 30u), 7u, 0x90000000u | swizzle, 0u, 0u, 0u, 0u, clamped ? 2u : 0u, 0u, 0u, 0u}; + if (iteration == 16u) std::fill(userData.begin() + 4u, userData.begin() + 12u, 0u); + const ShaderRecompiler::ShaderComputeStageInfo compute{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + ShaderRecompiler::RecompileRequest request{{ShaderRecompiler::ShaderStage::Compute, reinterpret_cast(code.data()), code, 0u, {}}, {32u, 0u, userData, compute, std::nullopt, std::nullopt, {}}, device.Target(), {0u, 0u, 0u, 128u}}; + const auto shader = ShaderRecompiler::Recompile(request); + if (first.variantId == 0u) first = shader; + else Require(shader.cacheHit && shader.variantId == first.variantId, "T#/S# changed the compiled artifact"); + device.Dispatch(shader, 1u, 1u, 1u); + device.WaitIdle(); + const auto pixel = clamped ? 31u : 8u; + const auto expected = iteration == 16u ? 0u : swizzled ? (format == 34u ? pixel + 200u : format == 22u ? 0x3f800000u : 1u) : format == 34u ? pixel + 100u : texels[pixel]; + for (std::uint32_t lane = 0u; lane < 32u; ++lane) Require(output[lane] == expected, "runtime image/sampler selection failed: iteration=" + std::to_string(iteration) + " value=" + std::to_string(output[lane]) + " expected=" + std::to_string(expected)); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + std::cout << "runtime image and sampler tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp new file mode 100644 index 000000000..cf910ab2c --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp @@ -0,0 +1,362 @@ +#include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "VulkanTestDevice.hpp" +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace AgcDriver::Graphics; + +constexpr std::uint32_t Side = 256; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t TileR64KBX = 0x1b; +constexpr std::uint32_t Type2D = 9; +constexpr std::size_t SurfaceBytes = std::size_t{Side} * Side * 4u; +constexpr std::size_t Surfaces = 12; +constexpr std::size_t BlockBytes = SurfaceBytes * Surfaces; +constexpr std::uint64_t Budget = 1ull << 20u; + +class Device { +public: + Device() { +#ifdef _WIN32 + library = SDL_LoadObject("vulkan-1.dll"); +#else + library = SDL_LoadObject("libvulkan.so.1"); +#endif + Require(library != nullptr, "cannot load Vulkan"); + try { + instanceProc = reinterpret_cast(SDL_LoadFunction(library, "vkGetInstanceProcAddr")); + Require(instanceProc != nullptr, "missing Vulkan instance resolver"); + VkApplicationInfo application{VK_STRUCTURE_TYPE_APPLICATION_INFO}; + application.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo info{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; + info.pApplicationInfo = &application; + Check(function("vkCreateInstance")(&info, nullptr, &instance), "vkCreateInstance"); + std::uint32_t count = 0; + const auto enumerate = function("vkEnumeratePhysicalDevices"); + Check(enumerate(instance, &count, nullptr), "vkEnumeratePhysicalDevices"); + Require(count != 0, "no Vulkan device"); + std::vector devices(count); + Check(enumerate(instance, &count, devices.data()), "vkEnumeratePhysicalDevices"); + context.physical = devices.front(); + const auto extensions = function("vkEnumerateDeviceExtensionProperties"); + Check(extensions(context.physical, nullptr, &count, nullptr), "vkEnumerateDeviceExtensionProperties"); + std::vector available(count); + Check(extensions(context.physical, nullptr, &count, available.data()), "vkEnumerateDeviceExtensionProperties"); + const auto hasExtension = [&](const char* name) { + for (const auto& extension : available) { + if (std::strcmp(extension.extensionName, name) == 0) return true; + } + return false; + }; + auto bytes = AgcDriver::QueryBdaByteFeatures(context.physical, function("vkGetPhysicalDeviceFeatures2"), available); + auto address = AgcDriver::QueryBdaFeatures(context.physical, function("vkGetPhysicalDeviceFeatures2"), available); + const auto queues = function("vkGetPhysicalDeviceQueueFamilyProperties"); + queues(context.physical, &count, nullptr); + std::vector families(count); + queues(context.physical, &count, families.data()); + std::uint32_t family = 0; + while (family < count && (families[family].queueFlags & VK_QUEUE_COMPUTE_BIT) == 0) ++family; + Require(family < count, "no Vulkan compute queue"); + const float priority = 1; + VkDeviceQueueCreateInfo queue{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + queue.queueFamilyIndex = family; + queue.queueCount = 1; + queue.pQueuePriorities = &priority; + VkPhysicalDeviceFeatures enabled{}; + enabled.shaderInt64 = VK_TRUE; + address.pNext = &bytes; + std::vector extensionsEnabled{VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, VK_KHR_8BIT_STORAGE_EXTENSION_NAME}; + if (hasExtension(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME)) { + extensionsEnabled.push_back(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME); + VkPhysicalDeviceExternalMemoryHostPropertiesEXT hostProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &hostProperties}; + function("vkGetPhysicalDeviceProperties2")(context.physical, &properties); + context.hostImportAlignment = hostProperties.minImportedHostPointerAlignment; + } + VkDeviceCreateInfo device{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, &address}; + device.queueCreateInfoCount = 1; + device.pQueueCreateInfos = &queue; + device.enabledExtensionCount = static_cast(extensionsEnabled.size()); + device.ppEnabledExtensionNames = extensionsEnabled.data(); + device.pEnabledFeatures = &enabled; + Check(function("vkCreateDevice")(context.physical, &device, nullptr, &context.device), "vkCreateDevice"); + context.deviceProc = function("vkGetDeviceProcAddr"); + function("vkGetPhysicalDeviceMemoryProperties")(context.physical, &context.memory); + VkPhysicalDeviceProperties properties{}; + function("vkGetPhysicalDeviceProperties")(context.physical, &properties); + context.limits = properties.limits; + context.bufferDeviceAddress = true; + context.formatProperties = function("vkGetPhysicalDeviceFormatProperties"); + context.imageFormatProperties = function("vkGetPhysicalDeviceImageFormatProperties"); + if (hasExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) context.memoryProperties2 = function("vkGetPhysicalDeviceMemoryProperties2"); + context.Function("vkGetDeviceQueue")(context.device, family, 0, &context.queue); + VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pool.queueFamilyIndex = family; + Check(context.Function("vkCreateCommandPool")(context.device, &pool, nullptr, &context.pool), "vkCreateCommandPool"); + } catch (...) { + release(); + throw; + } + } + + ~Device() { release(); } + const Context& GetContext() const { return context; } + +private: + template + TFunction function(const char* name) const { + const auto result = reinterpret_cast(instanceProc(instance, name)); + Require(result != nullptr, name); + return result; + } + + void release() noexcept { + if (context.pool != VK_NULL_HANDLE) context.Function("vkDestroyCommandPool")(context.device, context.pool, nullptr); + context.bufferPool.reset(); + if (context.device != VK_NULL_HANDLE) function("vkDestroyDevice")(context.device, nullptr); + if (instance != VK_NULL_HANDLE) function("vkDestroyInstance")(instance, nullptr); + if (library != nullptr) SDL_UnloadObject(library); + } + + void* library = nullptr; + PFN_vkGetInstanceProcAddr instanceProc = nullptr; + VkInstance instance = VK_NULL_HANDLE; + Context context{}; +}; + +class Block { +public: + explicit Block(const Context& context) : context(context) { +#ifdef _WIN32 + memory = VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + memory = std::aligned_alloc(65536, BlockBytes); +#endif + Require(memory != nullptr, "cannot allocate the texture block"); + std::memset(memory, 0x5a, BlockBytes); + GuestAllocations::Mutation().Add(memory, BlockBytes, true, true); + } + + ~Block() { + ClearCachedTextures(context.device); + GuestAllocations::Mutation().Remove(memory); + HostImportFor(context, Surface(0), BlockBytes); +#ifdef _WIN32 + VirtualFree(memory, 0, MEM_RELEASE); +#else + std::free(memory); +#endif + } + + std::uint64_t Surface(std::size_t surface) const { return reinterpret_cast(memory) + surface * SurfaceBytes; } + +private: + const Context& context; + void* memory = nullptr; +}; + +std::array Descriptor(std::uint64_t address, std::uint32_t side = Side) { + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((side - 1u) & 3u) << 30u), + ((side - 1u) >> 2u) | ((side - 1u) << 14u), + 0xfacu | (TileR64KBX << 20u) | (Type2D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +std::shared_ptr Sampled(const Context& context, std::uint64_t address, std::uint32_t side = Side) { + const auto words = Descriptor(address, side); + return CachedSampledTexture(context, words); +} + +GuestTextureResource Resource(std::uint64_t address, std::uint32_t side = Side) { + const auto words = Descriptor(address, side); + return DecodeTextureResource(words); +} + +void viewTests(const Context& context) { + if (context.hostImportAlignment == 0) { + std::puts("host imports unavailable: views of storage images not tested"); + return; + } + Block block(context); + if (HostImportFor(context, block.Surface(0), BlockBytes) == nullptr) { + std::puts("host import of the texture block refused: views of storage images not tested"); + return; + } + const auto before = TextureCacheUsage(); + auto view = Sampled(context, block.Surface(0)); + Require(view->ViewsStorageImage(), "a sampled texture in host-imported memory is not a view of its storage image"); + const std::weak_ptr viewed = view; + const auto* image = view->StorageSource(); + const auto after = TextureCacheUsage(); + Require(after.sampledEntries == before.sampledEntries + 1u, "the view did not enter the sampled texture cache"); + Require(after.sampledBytes == before.sampledBytes, "a view of a cached storage image counted " + std::to_string(after.sampledBytes - before.sampledBytes) + " bytes against the sampled texture cache"); + Require(after.storageBytes > before.storageBytes, "the storage image of the view is not counted by the storage image cache"); + Require(Sampled(context, block.Surface(0)) == view, "the second lookup of the surface did not hit its view"); + view.reset(); + for (std::size_t surface = 1; surface < Surfaces; ++surface) CachedStorageSurface(context, Resource(block.Surface(surface))); + Require(!StorageImageCached(context, image), "the storage image cache never evicted the viewed image under a 1 MiB budget"); + const auto evicted = TextureCacheUsage(); + Require(viewed.expired(), "the sampled texture cache kept the view of an evicted storage image"); + Require(evicted.sampledEntries == after.sampledEntries - 1u && evicted.sampledBytes == before.sampledBytes, "dropping the view of the evicted storage image changed other sampled entries"); + auto again = Sampled(context, block.Surface(0)); + Require(again->ViewsStorageImage() && again->StorageSource() != nullptr && StorageImageCached(context, again->StorageSource()), "the surface's next lookup does not view its new cached storage image"); +} + +void reportedBudgetTests(const Context& context) { + if (context.memoryProperties2 == nullptr) { + std::puts("VK_EXT_memory_budget unavailable: the sampled texture cache keeps a quarter of the device-local heap"); + return; + } + VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2}; + properties.pNext = &reported; + context.memoryProperties2(context.physical, &properties); + const auto budget = SampledTextureBudget(context.memory, &reported, 0); + Require(budget >= (2048ull << 20u), "the sampled texture cache budget is below the 2 GiB floor"); + for (std::uint32_t heap = 0; heap < context.memory.memoryHeapCount; ++heap) { + if ((context.memory.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) continue; + std::printf("device-local heap %u: %llu MiB, VK_EXT_memory_budget budget %llu MiB, usage %llu MiB\n", heap, static_cast(context.memory.memoryHeaps[heap].size >> 20u), static_cast(reported.heapBudget[heap] >> 20u), static_cast(reported.heapUsage[heap] >> 20u)); + } + std::printf("sampled texture cache budget %llu MiB, storage image cache budget %llu MiB\n", static_cast(budget >> 20u), static_cast(TextureCacheBudget(context.memory) >> 20u)); +} + +void snapshotTests(const Context& context) { + Block block(context); + auto first = Sampled(context, block.Surface(0)); + Require(!first->ViewsStorageImage(), "a sampled texture outside host-imported memory is not a snapshot"); + const auto held = TextureCacheUsage().sampledBytes; + Require(held == first->AllocationBytes(), "a snapshot counted " + std::to_string(held) + " bytes, not its image allocation of " + std::to_string(first->AllocationBytes())); + const std::weak_ptr oldest = first; + first.reset(); + const auto capacity = static_cast(Budget / held); + Require(capacity >= 2u && capacity + 1u < Surfaces, "a snapshot of the 256 KiB surface counts " + std::to_string(held) + " bytes, which the test's 1 MiB budget does not fit as planned"); + for (std::size_t surface = 1; surface < capacity; ++surface) Sampled(context, block.Surface(surface)); + Require(!oldest.expired() && TextureCacheUsage().sampledEntries == capacity, "the sampled texture cache did not keep " + std::to_string(capacity) + " snapshots within its budget"); + Sampled(context, block.Surface(capacity)); + Require(oldest.expired(), "the least recently used snapshot was not evicted past the budget"); + Require(TextureCacheUsage().sampledBytes <= Budget, "the sampled texture cache holds more than its budget"); + ClearCachedTextures(context.device); + constexpr std::uint32_t TinySide = 16; + const auto tiny = Sampled(context, block.Surface(0), TinySide); + const auto guestBytes = DescribeSurface(Resource(block.Surface(0), TinySide)).guestBytes; + if (tiny->AllocationBytes() >= guestBytes) { + std::printf("the 16x16 snapshot's image takes %llu bytes, not less than its %llu guest bytes: counting the allocation alone not tested\n", static_cast(tiny->AllocationBytes()), static_cast(guestBytes)); + return; + } + Require(TextureCacheUsage().sampledBytes == tiny->AllocationBytes(), "a snapshot whose " + std::to_string(tiny->AllocationBytes()) + "-byte image is smaller than its " + std::to_string(guestBytes) + " guest bytes counted " + std::to_string(TextureCacheUsage().sampledBytes) + " bytes, not its allocation"); +} + +constexpr std::uint64_t ReportedBudget = 64ull << 30u; +constexpr std::uint64_t ReportedUsage = 12ull << 30u; + +void VKAPI_PTR ReportBudget(VkPhysicalDevice, VkPhysicalDeviceMemoryProperties2* properties) { + auto* reported = static_cast(properties->pNext); + for (std::uint32_t heap = 0; heap < VK_MAX_MEMORY_HEAPS; ++heap) { + reported->heapBudget[heap] = ReportedBudget; + reported->heapUsage[heap] = ReportedUsage; + } +} + +std::uint64_t ExpectedBudget(const Context& context) { + VkPhysicalDeviceMemoryBudgetPropertiesEXT reported{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, &reported}; + ReportBudget(context.physical, &properties); + return SampledTextureBudget(context.memory, &reported, SampledTextureMemory() + TextureCacheUsage().storageBytes); +} + +std::uint64_t ReadBudget(const Context& context) { + std::this_thread::sleep_for(std::chrono::milliseconds(1100)); + return SampledTextureCacheBudget(context); +} + +void memoryTests(Context context) { + context.memoryProperties2 = &ReportBudget; + const auto before = SampledTextureMemory(); + Block block(context); + auto held = Sampled(context, block.Surface(0)); + Require(!held->ViewsStorageImage() && held->AllocationBytes() != 0, "the test texture is not a snapshot with an image of its own"); + Require(SampledTextureMemory() == before + held->AllocationBytes(), "the sampled texture memory grew by " + std::to_string(SampledTextureMemory() - before) + " bytes, not by the snapshot's allocation of " + std::to_string(held->AllocationBytes())); + Require(ReadBudget(context) == ExpectedBudget(context), "the sampled texture cache budget does not count its cached texture as the cache's own memory"); + ClearCachedTextures(context.device); + Require(TextureCacheUsage().sampledEntries == 0 && SampledTextureMemory() == before + held->AllocationBytes(), "a texture its caller still holds stopped counting as sampled texture memory when it left the cache"); + const auto budget = ReadBudget(context); + Require(budget == ExpectedBudget(context), "the sampled texture cache budget counts a texture its caller still holds as memory in use outside the texture caches"); + std::printf("reported heap budget %llu MiB with %llu MiB in use: sampled texture cache budget %llu MiB with a %llu KiB texture held outside the cache\n", static_cast(ReportedBudget >> 20u), static_cast(ReportedUsage >> 20u), static_cast(budget >> 20u), static_cast(held->AllocationBytes() >> 10u)); + held.reset(); + Require(SampledTextureMemory() == before, "a released texture still counts as sampled texture memory"); +} + +} + +int main(int argc, char** argv) { + try { + const bool memory = argc > 1 && std::string(argv[1]) == "memory"; +#ifdef _WIN32 + if (!memory) _putenv_s("APS5_TEXTURE_CACHE_MIB", "1"); +#else + if (!memory) setenv("APS5_TEXTURE_CACHE_MIB", "1", 1); +#endif + std::unique_ptr device; + try { + device = std::make_unique(); + } catch (const std::exception& error) { + if (std::getenv("ANYPS5_REQUIRE_VULKAN") != nullptr) throw; + std::printf("skipped, no usable Vulkan device: %s\n", error.what()); + return VulkanTestSkipped; + } + auto context = device->GetContext(); + TextureDetiler detiler(context); + context.detiler = &detiler; + if (memory) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + memoryTests(context); + ClearCachedTextures(context.device); + std::puts("sampled texture memory budget tests passed"); + return 0; + } + { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + reportedBudgetTests(context); + snapshotTests(context); + viewTests(context); + ClearCachedTextures(context.device); + } + std::puts("sampled texture cache budget tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarBitCount.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarBitCount.cpp index 0624c8e50..8d24de96a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarBitCount.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarBitCount.cpp @@ -47,7 +47,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarGetreg.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarGetreg.cpp index ae8c61581..06c8439e0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarGetreg.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarGetreg.cpp @@ -27,7 +27,7 @@ alignas(256) constexpr std::array Code{ std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramEnd.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramEnd.cpp index 335a99a9a..9edb51a7e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramEnd.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramEnd.cpp @@ -41,7 +41,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { words[0] = tid * 0x01010101 std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramEndSaved.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramEndSaved.cpp index 100b4552e..2a657f384 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramEndSaved.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramEndSaved.cpp @@ -34,7 +34,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { words[0] = tid * 0x01010101 std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramFlow.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramFlow.cpp index 2ffaf0609..ed2807756 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramFlow.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarProgramFlow.cpp @@ -252,7 +252,7 @@ constexpr bool CrossLane[Checked] = {true, true, true, true, true, true, true, t std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarRelativeIndexing.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarRelativeIndexing.cpp index a786bf3ff..5a237a3e1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarRelativeIndexing.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarRelativeIndexing.cpp @@ -48,7 +48,7 @@ void Fill(std::uint32_t, std::uint32_t*) {} std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSethalt.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSethalt.cpp new file mode 100644 index 000000000..cda66e707 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSethalt.cpp @@ -0,0 +1,92 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084, 0x34060086, 0xe0301000, 0x80000401, 0xbf8c3f70, 0xbf8d0000, 0x4a0a0881, 0xbf820003, + 0xe0701000, 0x80010503, 0xbf810000, 0xbf82fffc, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, +}; + +void Fill(std::uint32_t tid, std::uint32_t* words) { words[0] = tid * 0x01010101u + 7u; } + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) { + Require(actual == expected, std::string("scalar sethalt: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* in = &Input[tid * Inputs]; + const std::uint32_t* out = &Output[tid * Results]; + Expect(tid, out[0], in[0] + 1u, "store after s_sethalt 0"); + Expect(tid, out[1], 0xdeadbeefu, "word after the store"); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("scalar sethalt tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSetpc.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSetpc.cpp index bc390fee3..669e12598 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSetpc.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSetpc.cpp @@ -239,7 +239,7 @@ constexpr const char* Names[Checked] = {"jump_table", "jump_table_offset", "skip std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSop1.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSop1.cpp index c26c22609..e65a91fa3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSop1.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSop1.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace { @@ -211,7 +212,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -247,12 +248,18 @@ void Run(AgcDriver::VulkanDevice& device) { device.WaitIdle(); } -void Check() { +bool ReadsExecLo(std::string_view name) { + if (name.starts_with("scc") || (name.find("saveexec") == std::string_view::npos && name.find("wrexec") == std::string_view::npos)) return false; + return name.find("_b32 ") != std::string_view::npos || name.ends_with(" s12") || name.ends_with(" s14"); +} + +void Check(std::uint32_t subgroupSize) { + const std::uint32_t hostLanes = subgroupSize < 32u ? (1u << subgroupSize) - 1u : 0xffffffffu; for (std::uint32_t tid = 0; tid < Threads; ++tid) { const std::uint32_t* in = &Input[tid * Inputs]; const std::uint32_t* out = &Output[tid * Results]; for (std::uint32_t i = 0; i < Results; ++i) { - if (Names[i] != nullptr) Expect(tid, out[i], Expected[tid][i], Names[i]); + if (Names[i] != nullptr) Expect(tid, out[i], ReadsExecLo(Names[i]) ? Expected[tid][i] & hostLanes : Expected[tid][i], Names[i]); } } } @@ -264,7 +271,7 @@ int main() { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; Run(*device); - Check(); + Check(device->Target().subgroupSize); std::puts("scalar sop1 tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSop2.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSop2.cpp index ed6a6bbe0..6449b8e9d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSop2.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSop2.cpp @@ -135,7 +135,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkCompare.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkCompare.cpp index 1b6340993..7cb3743eb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkCompare.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkCompare.cpp @@ -126,7 +126,7 @@ alignas(256) constexpr std::array CompareCode = BuildC std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkMisc.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkMisc.cpp index 4760ee0ea..1bcc932bc 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkMisc.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkMisc.cpp @@ -39,7 +39,7 @@ void Fill(std::uint32_t, std::uint32_t*) {} std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkWaitcnt.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkWaitcnt.cpp index 4afa85b94..014739d63 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkWaitcnt.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSopkWaitcnt.cpp @@ -122,7 +122,7 @@ alignas(256) constexpr std::array WaitCode = BuildCode std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarStaticCall.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarStaticCall.cpp new file mode 100644 index 000000000..fe8872d9c --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarStaticCall.cpp @@ -0,0 +1,134 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t MaxThreads = 64; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 4; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array SwappcForward{0x34020084u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0xbe8a1f00u, 0x800aff0au, 0x00000014u, 0x820b800bu, 0xbe88210au, 0xbf820008u, 0x4a0808ffu, 0x13579bdfu, 0x3a0a0affu, 0xa5a5a5a5u, 0x4a0c0f06u, 0x3a0e0effu, 0x5a5a5a5au, 0xbe802008u, 0xe0781000u, 0x80010401u, 0xbf810000u}; +alignas(256) constexpr std::array CallForward{0x34020084u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0xbb080001u, 0xbf820008u, 0x4a0808ffu, 0x13579bdfu, 0x3a0a0affu, 0xa5a5a5a5u, 0x4a0c0f06u, 0x3a0e0effu, 0x5a5a5a5au, 0xbe802008u, 0xe0781000u, 0x80010401u, 0xbf810000u}; +alignas(256) constexpr std::array CallBackward{0x34020084u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0xbf820008u, 0x4a0808ffu, 0x13579bdfu, 0x3a0a0affu, 0xa5a5a5a5u, 0x4a0c0f06u, 0x3a0e0effu, 0x5a5a5a5au, 0xbe802008u, 0xbb08fff7u, 0xbf820000u, 0xe0781000u, 0x80010401u, 0xbf810000u}; +alignas(256) constexpr std::array CallAfterEnd{0x34020084u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0xbb080003u, 0xe0781000u, 0x80010401u, 0xbf810000u, 0x4a0808ffu, 0x13579bdfu, 0x3a0a0affu, 0xa5a5a5a5u, 0x4a0c0f06u, 0x3a0e0effu, 0x5a5a5a5au, 0xbe802008u, 0xbf810000u}; +alignas(256) constexpr std::array MixedNested{0x34020084u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0xbe8a1f00u, 0x800aff0au, 0x00000014u, 0x820b800bu, 0xbe88210au, 0xbf82000bu, 0x4a0808ffu, 0x13579bdfu, 0xbb0c0001u, 0xbf820006u, 0x3a0a0affu, 0xa5a5a5a5u, 0x4a0c0f06u, 0x3a0e0effu, 0x5a5a5a5au, 0xbe80200cu, 0xbe802008u, 0xe0781000u, 0x80010401u, 0xbf810000u}; + +constexpr std::array Selectors{0u, 1u, 2u, 3u, 4u, 7u, 0x80000000u, 0xffffffffu}; +constexpr const char* Names[Results] = {"add", "xor", "sum", "xor_second"}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +std::uint32_t Expected(std::uint32_t lane, std::uint32_t result) { + const auto selector = Input[lane * Inputs]; + const auto value = Input[lane * Inputs + 1u]; + switch (result) { + case 0: return selector + 0x13579bdfu; + case 1: return value ^ 0xa5a5a5a5u; + case 2: return Input[lane * Inputs + 2u] + Input[lane * Inputs + 3u]; + default: return Input[lane * Inputs + 3u] ^ 0x5a5a5a5au; + } +} + +ShaderRecompiler::RecompileResult Compile(std::span code, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target) { + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{waveSize, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory}, + target, + {0, 0, 0, 128} + }; + request.useCache = false; + return ShaderRecompiler::Recompile(request); +} + +void Run(AgcDriver::VulkanDevice& device, std::span code, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target) { + for (std::uint32_t tid = 0; tid < MaxThreads; ++tid) { + Input[tid * Inputs] = Selectors[tid % Selectors.size()]; + Input[tid * Inputs + 1u] = 0x9e3779b9u * (tid + 1u); + Input[tid * Inputs + 2u] = 0xffffffffu - tid; + Input[tid * Inputs + 3u] = 17u * tid + 1u; + } + Output.fill(0xdeadbeefu); + const auto result = Compile(code, waveSize, target); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(const char* variant, std::uint32_t lanes) { + for (std::uint32_t tid = 0; tid < lanes; ++tid) { + for (std::uint32_t index = 0; index < Results; ++index) { + const auto actual = Output[tid * Results + index]; + const auto expected = Expected(tid, index); + Require(actual == expected, std::string(variant) + " wave" + std::to_string(lanes) + ": lane " + std::to_string(tid) + " " + Names[index] + " is " + Hex(actual) + ", expected " + Hex(expected)); + } + } + for (std::uint32_t index = lanes * Results; index < Output.size(); ++index) { + Require(Output[index] == 0xdeadbeefu, std::string(variant) + ": inactive lane wrote output"); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const auto defaultTarget = device->Target(); + const auto wave32Target = device->ComputeTarget(32); + const bool distinctTarget = defaultTarget.subgroupSize != wave32Target.subgroupSize; + std::printf("static call targets: default subgroup %u, ComputeTarget(32) subgroup %u\n", defaultTarget.subgroupSize, wave32Target.subgroupSize); + std::uint32_t lanes = 0u, dispatches = 0u; + const std::array, 5> variants{SwappcForward, CallForward, CallBackward, CallAfterEnd, MixedNested}; + const std::array names{"SWAPPC forward", "CALL forward", "CALL backward", "CALL after endpgm", "mixed nested calls"}; + for (std::size_t index = 0; index < variants.size(); ++index) { + const auto code = variants[index]; + Run(*device, code, 32, wave32Target); + Check(names[index], 32); + Run(*device, code, 64, defaultTarget); + Check(names[index], 64); + if (distinctTarget) { + Run(*device, code, 64, wave32Target); + Check(names[index], 64); + } + const std::uint32_t variantLanes = distinctTarget ? 160u : 96u; + lanes += variantLanes; + dispatches += distinctTarget ? 3u : 2u; + std::printf("passed %s: %u lanes, %u output comparisons\n", names[index], variantLanes, variantLanes * Results); + } + std::printf("static call execution passed: 5 variants, %u dispatches, %u lanes, %u output comparisons\n", dispatches, lanes, lanes * Results); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSwappc.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSwappc.cpp index 161d76cea..17288fd47 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSwappc.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSwappc.cpp @@ -63,7 +63,7 @@ constexpr const char* Names[Results] = {"dword_table_getpc_first", "dword_table_ std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSwappcFetch.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSwappcFetch.cpp new file mode 100644 index 000000000..158fb8d50 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSwappcFetch.cpp @@ -0,0 +1,168 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 32; +constexpr std::uint32_t IdentitySelect = 0xfacu; +constexpr std::uint32_t ReverseSelect = 0x977u; +alignas(256) std::array Pixels{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array VertexCode{ + 0xbe822100, 0x34020a84, 0xe0781000, 0x80010801, 0xf80008cf, 0x0b0a0908, 0xbf810000, +}; + +alignas(256) constexpr std::array PixelCode{ + 0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000, +}; + +constexpr std::array, 3> Triangle{{ + {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.25f, 1.0f}, {-1.0f, 3.0f, 0.75f, 1.0f} +}}; + +std::array VertexBufferDescriptor(std::uint32_t select) { + const auto address = reinterpret_cast(Triangle.data()); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (16u << 16u), static_cast(Triangle.size()), 0x0104d000u | select}; +} + +std::array OutputDescriptor() { + const auto address = reinterpret_cast(Output.data()); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), static_cast(sizeof(Output)), 0x31016facu}; +} + +ShaderRecompiler::RecompileRequest VertexRequest(const AgcDriver::VulkanDevice& device, std::span userData, const ShaderRecompiler::ShaderVertexStageInfo& info, std::span memory) { + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {64u, 0, userData, std::nullopt, std::nullopt, info, memory}, + device.Target(), + {0, 0, 0, 64} + }; + request.useCache = false; + return request; +} + +ShaderRecompiler::ShaderVertexStageInfo VertexInfo(std::uint32_t select, std::int32_t registers) { + ShaderRecompiler::ShaderVertexStageInfo info{}; + info.resourcesNum = 1; + info.resources[0].fields = VertexBufferDescriptor(select); + info.resourcesDst[0] = {8, registers, 0, 0}; + info.fetchAttribReg = 8; + info.fetchBufferReg = 10; + info.fetchEmbedded = true; + return info; +} + +std::vector UserData() { + std::vector userData(12, 0u); + userData[0] = 0x10000000u; + userData[1] = 0x00000001u; + const auto output = OutputDescriptor(); + std::copy(output.begin(), output.end(), userData.begin() + 4); + return userData; +} + +void Draw(AgcDriver::VulkanDevice& device, std::uint32_t select) { + Pixels.fill(std::byte{0x40}); + Output.fill(0xdeadbeefu); + const auto userData = UserData(); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + const auto vertexResult = ShaderRecompiler::Recompile(VertexRequest(device, userData, VertexInfo(select, 4), vertexMemory)); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData(8, 0u); + const std::array pixelMemory{{{reinterpret_cast(PixelCode.data()), std::as_bytes(std::span(PixelCode))}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, + {64u, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + device.Target(), + {0, 0, vertexPush, 128 - vertexPush} + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, vertexPush} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 64u, 64u, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Triangle.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); +} + +void Check(std::uint32_t select, const std::string& what) { + for (std::uint32_t vertex = 0; vertex < Triangle.size(); ++vertex) { + for (std::uint32_t component = 0; component < 4u; ++component) { + const auto source = select == ReverseSelect ? 3u - component : component; + const auto expected = std::bit_cast(Triangle[vertex][source]); + const auto actual = Output[vertex * 4u + component]; + Require(actual == expected, what + ": vertex " + std::to_string(vertex) + " v" + std::to_string(8u + component) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected)); + } + } +} + +void CheckRegisterLimit(AgcDriver::VulkanDevice& device) { + const auto userData = UserData(); + const std::array memory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + try { + static_cast(ShaderRecompiler::Recompile(VertexRequest(device, userData, VertexInfo(IdentitySelect, 5), memory))); + } catch (const std::exception& error) { + Require(std::string(error.what()).find("the inline model supports one to four") != std::string::npos, std::string("fetch-shader register limit: unexpected error: ") + error.what()); + return; + } + throw std::runtime_error("fetch-shader register limit: five destination registers compiled"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Draw(*device, IdentitySelect); + Check(IdentitySelect, "fetch-shader call, identity dst_sel"); + Draw(*device, ReverseSelect); + Check(ReverseSelect, "fetch-shader call, reversed dst_sel"); + CheckRegisterLimit(*device); + std::puts("scalar swappc fetch tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarWideLoads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarWideLoads.cpp index c38e112d1..97f04ea30 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarWideLoads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarWideLoads.cpp @@ -182,7 +182,7 @@ private: std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaDestination.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaDestination.cpp index 029da07e4..59a1b5490 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaDestination.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaDestination.cpp @@ -64,7 +64,7 @@ std::string Hex(std::uint32_t value) { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaFloatSelectors.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaFloatSelectors.cpp index 10863bc09..6efd68b5d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaFloatSelectors.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaFloatSelectors.cpp @@ -127,7 +127,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaIntegerClamp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaIntegerClamp.cpp index 3174ec093..dbd726c83 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaIntegerClamp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaIntegerClamp.cpp @@ -125,7 +125,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaIntegerSelectors.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaIntegerSelectors.cpp index 2d9b96b79..e6a4be581 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaIntegerSelectors.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaIntegerSelectors.cpp @@ -126,7 +126,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaLdexpF16.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaLdexpF16.cpp index fa99d9da1..ec46c5b46 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaLdexpF16.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaLdexpF16.cpp @@ -127,7 +127,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -155,6 +155,7 @@ ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std:: {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; return ShaderRecompiler::Recompile(request); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaMoveSelectors.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaMoveSelectors.cpp index 67a89cf6d..d020e56e7 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaMoveSelectors.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaMoveSelectors.cpp @@ -204,7 +204,7 @@ std::string Describe(std::uint32_t sourceSelector, bool signExtend, std::uint32_ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device, std::span code, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ShaderClock.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ShaderClock.cpp index 73e876521..88557c5e5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ShaderClock.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ShaderClock.cpp @@ -43,7 +43,7 @@ std::uint64_t Pair(const std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -56,29 +56,6 @@ void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, con Require(actual == expected, std::string("shader clock: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); } -void Run(AgcDriver::VulkanDevice& device) { - for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); - Output.fill(0xdeadbeefu); - std::vector userData(8, 0u); - const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); - const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); - std::copy(input.begin(), input.end(), userData.begin()); - std::copy(output.begin(), output.end(), userData.begin() + 4); - const std::span code(Code); - const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; - const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; - ShaderRecompiler::RecompileRequest request{ - {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, - {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), - {0, 0, 0, 128} - }; - request.useCache = false; - const auto result = ShaderRecompiler::Recompile(request); - device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); - device.WaitIdle(); -} - void Check() { for (std::uint32_t tid = 0; tid < Threads; ++tid) { const std::uint32_t* in = &Input[tid * Inputs]; @@ -94,12 +71,43 @@ void Check() { } } +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + constexpr std::uint32_t Passes = 128; + for (std::uint32_t pass = 0; pass < Passes; ++pass) { + Output.fill(0xdeadbeefu); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + Check(); + } +} + } int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (!TargetHasCapability(device->Target(), spv::CapabilityShaderClockKHR)) { + std::puts("skipped, the device lacks shaderSubgroupClock or shaderDeviceClock"); + return VulkanTestSkipped; + } Run(*device); Check(); std::puts("shader clock tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Srgb8ColorTarget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Srgb8ColorTarget.cpp new file mode 100644 index 000000000..c1315cbbd --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Srgb8ColorTarget.cpp @@ -0,0 +1,354 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 512; +constexpr std::uint32_t Height = 256; +constexpr std::uint32_t ScissorWidth = 256; +constexpr std::size_t BlockBytes = 65536; +constexpr std::size_t SurfaceBytes = 2 * BlockBytes; +constexpr std::size_t KeyBytes = SurfaceBytes / 256; +constexpr std::size_t LinearBytes = std::size_t{Width} * Height * 4; +constexpr std::uint32_t Srgb8Info = 0x8604u; +constexpr std::uint32_t Rgba8Info = 0x8028u; +constexpr std::uint32_t DccEnable = 0x10000000u; +constexpr std::uint32_t TiledAttrib3 = 0x4dc6c000u; +constexpr std::uint32_t LinearAttrib3 = 0x09000000u; +constexpr std::uint8_t ClearCode = 0x5d; +constexpr std::uint8_t Kept = 0x40; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +constexpr std::array, 3> Triangle{{ + {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f} +}}; + +alignas(256) std::array, 8> Programs{}; +std::size_t programCount = 0; +alignas(256) std::array Keys{}; + +double Eotf(std::uint32_t code) { + const double encoded = code / 255.0; + return encoded <= 0.04045 ? encoded / 12.92 : std::pow((encoded + 0.055) / 1.055, 2.4); +} + +int Oetf(double linear) { + linear = std::clamp(linear, 0.0, 1.0); + const double encoded = linear <= 0.0031308 ? linear * 12.92 : 1.055 * std::pow(linear, 1.0 / 2.4) - 0.055; + return static_cast(std::lround(encoded * 255.0)); +} + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +std::span Program(std::array srgb8, bool rgba8) { + Require(programCount < Programs.size(), "too many test programs"); + auto& code = Programs[programCount++]; + std::size_t at = 0; + const auto move = [&](std::uint32_t vgpr, float value) { + code[at++] = 0x7e0002ffu | (vgpr << 17u); + code[at++] = std::bit_cast(value); + }; + for (std::uint32_t channel = 0; channel < 4; ++channel) move(4 + channel, srgb8[channel]); + if (rgba8) { + for (std::uint32_t channel = 0; channel < 4; ++channel) move(8 + channel, channel % 2 == 0 ? 1.0f : 0.0f); + code[at++] = 0xf800000fu; + code[at++] = 0x0b0a0908u; + code[at++] = 0xf800181fu; + } else { + code[at++] = 0xf800180fu; + } + code[at++] = 0x07060504u; + code[at++] = 0xbf810000u; + return std::span(code.data(), at); +} + +struct Block { + explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) { +#ifdef _WIN32 + if (watched) { + data = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes)); + GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes); + } else { + data = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + } +#else + if (watched) { + void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED, "cannot map the color target"); + const auto begin = reinterpret_cast(raw); + const auto aligned = (begin + BlockBytes - 1) & ~(static_cast(BlockBytes) - 1); + if (aligned != begin) munmap(raw, aligned - begin); + if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast(aligned + bytes), begin + BlockBytes - aligned); + data = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes); + } else { + data = static_cast(std::aligned_alloc(BlockBytes, bytes)); + } +#endif + Require(data != nullptr, "cannot allocate the color target"); + Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched"); + GuestAllocations::Mutation mutation; + mutation.Add(data, bytes, true, true); + } + ~Block() { + AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(data); + } +#ifdef _WIN32 + if (watched) { + GuestArena::GuestArenaReset_nid_postfix(data, bytes); + GuestArena::GuestArenaRelease_nid_postfix(data, bytes); + } else { + VirtualFree(data, 0, MEM_RELEASE); + } +#else + if (watched) { + munmap(data, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes); + } else { + std::free(data); + } +#endif + } + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + std::uint64_t Address() const { return reinterpret_cast(data); } + std::size_t bytes; + bool watched; + std::uint8_t* data = nullptr; +}; + +AgcDriver::Graphics::ColorTarget DecodeTarget(std::uint32_t slot, const Block& block, std::uint32_t info, std::uint32_t attrib3, std::uint32_t clear = 0) { + const auto stride = slot * 0xfu; + AgcDriver::Registers cx; + cx[0x318 + stride] = static_cast(block.Address() >> 8u); + cx[0x31b + stride] = 0; + cx[0x31c + stride] = info; + cx[0x31d + stride] = 0; + cx[0x323 + stride] = clear; + cx[0x324 + stride] = 0; + cx[0x3b0 + slot] = ((Width - 1u) << 14u) | (Height - 1u); + cx[0x3b8 + slot] = attrib3; + cx[0x390 + slot] = static_cast(block.Address() >> 40u); + if ((info & DccEnable) != 0) { + const auto keys = reinterpret_cast(Keys.data()); + cx[0x325 + stride] = static_cast(keys >> 8u); + cx[0x3a8 + slot] = static_cast(keys >> 40u); + } + auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, slot); + color.exportIndex = slot; + return color; +} + +VkPipelineColorBlendAttachmentState Blend(bool enable, VkBlendFactor source, VkBlendFactor destination) { + VkPipelineColorBlendAttachmentState blend{}; + blend.blendEnable = enable ? VK_TRUE : VK_FALSE; + blend.srcColorBlendFactor = source; + blend.dstColorBlendFactor = destination; + blend.colorBlendOp = VK_BLEND_OP_ADD; + blend.srcAlphaBlendFactor = source; + blend.dstAlphaBlendFactor = destination; + blend.alphaBlendOp = VK_BLEND_OP_ADD; + blend.colorWriteMask = 0xf; + return blend; +} + +const VkPipelineColorBlendAttachmentState BlendOff = Blend(false, VK_BLEND_FACTOR_ONE, VK_BLEND_FACTOR_ZERO); +const VkPipelineColorBlendAttachmentState Keep = Blend(true, VK_BLEND_FACTOR_ZERO, VK_BLEND_FACTOR_ONE); +const VkPipelineColorBlendAttachmentState Multiply = Blend(true, VK_BLEND_FACTOR_ZERO, VK_BLEND_FACTOR_SRC_COLOR); + +void Draw(AgcDriver::VulkanDevice& device, std::vector colors, std::vector blends, std::span pixelCode) { + const auto target = device.Target(); + constexpr std::uint32_t waveSize = 64; + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast(Triangle.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + for (std::size_t index = 0; index < colors.size(); ++index) pixel.targetOutputMode.at(index) = 9; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData(8, 0u); + const std::array pixelMemory{{{reinterpret_cast(pixelCode.data()), std::as_bytes(pixelCode)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, + {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + {0, 0, vertexPush, 128 - vertexPush} + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, vertexPush} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.colors = std::move(colors); + state.color = state.colors.front(); + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {ScissorWidth, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blends = std::move(blends); + state.blend = state.blends.front(); + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Triangle.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); +} + +std::vector ReadBack(AgcDriver::VulkanDevice& device, std::uint64_t address, std::size_t bytes) { + AgcDriver::Graphics::StorageTexture::FlushPending(address, bytes, nullptr, "test read-back"); + device.WaitIdle(); + std::vector stored(bytes); + AgcDriver::GuestMemory::Read(address, std::as_writable_bytes(std::span(stored)), 1); + return stored; +} + +void ExpectSurface(AgcDriver::VulkanDevice& device, const Block& block, const std::vector& before, const std::function& drawn, int tolerance, const std::string& what) { + const auto stored = ReadBack(device, block.Address(), SurfaceBytes); + for (std::size_t offset = 0; offset < SurfaceBytes; ++offset) { + const bool inside = offset < BlockBytes; + const int expected = inside ? drawn(before[offset]) : before[offset]; + const int actual = stored[offset]; + if (std::abs(actual - expected) <= (inside ? tolerance : 0)) continue; + throw std::runtime_error(what + ": byte " + std::to_string(offset) + (inside ? " inside" : " outside") + " the scissor is " + std::to_string(actual) + ", expected " + std::to_string(expected) + " from " + std::to_string(before[offset])); + } +} + +std::vector Fill(Block& block) { + for (std::size_t offset = 0; offset < block.bytes; ++offset) block.data[offset] = static_cast((offset * 7u + offset / 256u) & 0xffu); + return std::vector(block.data, block.data + block.bytes); +} + +void RecordedDrawTests(AgcDriver::VulkanDevice& device, Block& block) { + auto before = Fill(block); + const auto color = DecodeTarget(0, block, Srgb8Info, TiledAttrib3); + Require(color.format == VK_FORMAT_R8_SRGB && color.elementBytes == 1 && color.bytes == SurfaceBytes, "the 8_SRGB target did not decode as a 512x256 R8_SRGB surface"); + const auto keep = Program({0.0f, 0.25f, 0.5f, 0.75f}, false); + Draw(device, {color}, {Keep}, keep); + ExpectSurface(device, block, before, [](std::uint8_t code) { return code; }, 0, "a ZERO/ONE blend into an 8_SRGB target"); + before = ReadBack(device, block.Address(), SurfaceBytes); + constexpr float factor = 0.6f; + Draw(device, {color}, {Multiply}, Program({factor, 0.25f, 0.5f, 0.75f}, false)); + ExpectSurface(device, block, before, [](std::uint8_t code) { return Oetf(Eotf(code) * factor); }, 1, "a multiply blend into an 8_SRGB target"); + before = ReadBack(device, block.Address(), SurfaceBytes); + const auto written = static_cast(Eotf(150)); + Draw(device, {color}, {BlendOff}, Program({written, 0.25f, 0.5f, 0.75f}, false)); + Draw(device, {color}, {Multiply}, Program({0.5f, 0.25f, 0.5f, 0.75f}, false)); + ExpectSurface(device, block, before, [&](std::uint8_t) { return Oetf(static_cast(written) * 0.5); }, 1, "a write and a multiply blend in one pass over an 8_SRGB target"); + before = ReadBack(device, block.Address(), SurfaceBytes); + Draw(device, {color}, {BlendOff}, Program({written, 0.25f, 0.5f, 0.75f}, false)); + Draw(device, {color}, {Blend(true, VK_BLEND_FACTOR_ZERO, VK_BLEND_FACTOR_DST_ALPHA)}, Program({0.5f, 0.25f, 0.5f, 0.75f}, false)); + ExpectSurface(device, block, before, [](std::uint8_t) { return 150; }, 1, "a destination-alpha blend after a write in one pass over an 8_SRGB target"); +} + +void ClearTests(AgcDriver::VulkanDevice& device, Block& block) { + Fill(block); + Keys.fill(0x20); + const auto color = DecodeTarget(0, block, Srgb8Info | DccEnable, TiledAttrib3, ClearCode); + Require(color.dccAddress == reinterpret_cast(Keys.data()), "the 8_SRGB target lost its DCC keys"); + const auto written = static_cast(Eotf(200)); + Draw(device, {color}, {BlendOff}, Program({written, 0.25f, 0.5f, 0.75f}, false)); + const std::vector cleared(SurfaceBytes, ClearCode); + ExpectSurface(device, block, cleared, [](std::uint8_t) { return 200; }, 1, "a draw over a fast-cleared 8_SRGB target"); + std::array keys{}; + AgcDriver::GuestMemory::Read(reinterpret_cast(Keys.data()), std::as_writable_bytes(std::span(keys)), 1); + Require(std::all_of(keys.begin(), keys.end(), [](std::uint8_t key) { return key == 0xff; }), "the drawn 8_SRGB target kept compressed DCC keys"); +} + +void SynchronousDrawTests(AgcDriver::VulkanDevice& device, Block& linear, Block& block) { + Require(AgcDriver::Graphics::ColorTargetLayout(Width, Height, AgcDriver::Graphics::ColorTileMode::Linear, 4).Bytes() == LinearBytes, "the linear RGBA8 target is padded"); + std::memset(linear.data, Kept, LinearBytes); + const auto before = Fill(block); + const auto rgba8 = DecodeTarget(0, linear, Rgba8Info, LinearAttrib3); + const auto srgb8 = DecodeTarget(1, block, Srgb8Info, TiledAttrib3); + constexpr float factor = 0.35f; + Draw(device, {rgba8, srgb8}, {BlendOff, Multiply}, Program({factor, 0.25f, 0.5f, 0.75f}, true)); + ExpectSurface(device, block, before, [](std::uint8_t code) { return Oetf(Eotf(code) * factor); }, 1, "a multiply blend into an 8_SRGB target beside a linear target"); + const auto pixels = ReadBack(device, linear.Address(), LinearBytes); + for (std::size_t offset = 0; offset < LinearBytes; ++offset) { + const bool inside = (offset / 4u) % Width < ScissorWidth; + const std::uint8_t expected = inside ? (offset % 2u == 0 ? 255 : 0) : Kept; + Require(pixels[offset] == expected, "the linear RGBA8 target beside an 8_SRGB target holds " + std::to_string(pixels[offset]) + " at byte " + std::to_string(offset) + ", expected " + std::to_string(expected)); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Block recorded(SurfaceBytes); + Block cleared(SurfaceBytes); + Block linear(LinearBytes); + Block synchronous(SurfaceBytes); + RecordedDrawTests(*device, recorded); + ClearTests(*device, cleared); + SynchronousDrawTests(*device, linear, synchronous); + std::puts("8_SRGB color target tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StorageCacheBudget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StorageCacheBudget.cpp new file mode 100644 index 000000000..8cd839635 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StorageCacheBudget.cpp @@ -0,0 +1,214 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Check; +using AgcDriver::Graphics::Require; +using AgcDriver::Graphics::StorageTexture; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Side = 4096; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t TileR64KBX = 0x1b; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint64_t SurfaceBytes = std::uint64_t{Side} * Side * 4u; +constexpr std::uint64_t Stride = 0x10000; +constexpr std::uint64_t FixedBudget = 2048ull << 20u; +constexpr std::uint64_t EvictionLimit = 4ull << 30u; +constexpr std::uint32_t MaxSurfaces = 80; + +alignas(256) constexpr std::array LoadStoreCode{ + 0x7e040280, 0x7e060280, 0xf0001108, 0x00010402, 0xbf8c3f70, 0xf0200108, 0x00010402, 0xe0700000, 0x80000400, 0xbf810000, +}; + +alignas(256) std::array Output{}; + +std::uint32_t Marker(std::uint32_t surface) { + return 0x5a000000u | surface; +} + +template +TFunction InstanceFunction(PFN_vkGetInstanceProcAddr resolve, VkInstance instance, const char* name) { + const auto result = reinterpret_cast(resolve(instance, name)); + Require(result != nullptr, name); + return result; +} + +struct SelectedDevice { + VkPhysicalDeviceMemoryProperties memory{}; + VkPhysicalDeviceType type = VK_PHYSICAL_DEVICE_TYPE_OTHER; +}; + +SelectedDevice SelectDevice() { +#ifdef _WIN32 + void* library = SDL_LoadObject("vulkan-1.dll"); +#else + void* library = SDL_LoadObject("libvulkan.so.1"); +#endif + Require(library != nullptr, "cannot load Vulkan"); + const auto resolve = reinterpret_cast(SDL_LoadFunction(library, "vkGetInstanceProcAddr")); + VkInstance instance = VK_NULL_HANDLE; + SelectedDevice chosen; + try { + Require(resolve != nullptr, "missing Vulkan instance resolver"); + VkApplicationInfo application{VK_STRUCTURE_TYPE_APPLICATION_INFO}; + application.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo info{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; + info.pApplicationInfo = &application; + Check(InstanceFunction(resolve, VK_NULL_HANDLE, "vkCreateInstance")(&info, nullptr, &instance), "vkCreateInstance"); + const auto enumerate = InstanceFunction(resolve, instance, "vkEnumeratePhysicalDevices"); + std::uint32_t count = 0; + Check(enumerate(instance, &count, nullptr), "vkEnumeratePhysicalDevices"); + std::vector devices(count); + Check(enumerate(instance, &count, devices.data()), "vkEnumeratePhysicalDevices"); + devices.resize(count); + const auto properties = InstanceFunction(resolve, instance, "vkGetPhysicalDeviceProperties"); + const auto queues = InstanceFunction(resolve, instance, "vkGetPhysicalDeviceQueueFamilyProperties"); + constexpr auto graphicsCompute = VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT; + VkPhysicalDevice selected = VK_NULL_HANDLE; + int selectedRank = -1; + for (const auto physical : devices) { + VkPhysicalDeviceProperties described{}; + properties(physical, &described); + const int rank = described.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? 3 : described.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? 2 : described.deviceType == VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU ? 1 : 0; + if (described.apiVersion < VK_API_VERSION_1_1 || rank <= selectedRank) continue; + std::uint32_t familyCount = 0; + queues(physical, &familyCount, nullptr); + std::vector families(familyCount); + queues(physical, &familyCount, families.data()); + if (std::none_of(families.begin(), families.end(), [](const VkQueueFamilyProperties& family) { return family.queueCount != 0 && (family.queueFlags & graphicsCompute) == graphicsCompute; })) continue; + selected = physical; + selectedRank = rank; + chosen.type = described.deviceType; + } + Require(selected != VK_NULL_HANDLE, "no Vulkan 1.1 graphics and compute device"); + InstanceFunction(resolve, instance, "vkGetPhysicalDeviceMemoryProperties")(selected, &chosen.memory); + } catch (...) { + if (instance != VK_NULL_HANDLE) InstanceFunction(resolve, instance, "vkDestroyInstance")(instance, nullptr); + SDL_UnloadObject(library); + throw; + } + InstanceFunction(resolve, instance, "vkDestroyInstance")(instance, nullptr); + SDL_UnloadObject(library); + return chosen; +} + +std::array TextureDescriptor(std::uint64_t address) { + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((Side - 1u) & 3u) << 30u), + ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), + 0xfacu | (TileR64KBX << 20u) | (Type2D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +std::shared_ptr Load(AgcDriver::VulkanDevice& device, std::uint64_t base, std::uint32_t surface) { + const auto address = base + surface * Stride; + Output.fill(0xdeadbeefu); + std::vector userData(16, 0u); + const auto output = reinterpret_cast(Output.data()); + const std::array buffer{static_cast(output), static_cast((output >> 32u) & 0xffffu), static_cast(Output.size() * 4u), 0x31016facu}; + const auto texture = TextureDescriptor(address); + std::copy(buffer.begin(), buffer.end(), userData.begin()); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + const std::span code(LoadStoreCode); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{32, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + StorageTexture::FlushPending(address, static_cast(SurfaceBytes), nullptr, "test"); + device.WaitIdle(); + Require(Output[0] == Marker(surface), "surface " + std::to_string(surface) + ": the shader read " + std::to_string(Output[0]) + " instead of its first texel"); + return StorageTexture::FindLive(address, SurfaceBytes); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const auto selected = SelectDevice(); + const auto budget = AgcDriver::Graphics::TextureCacheBudget(selected.memory); + std::vector storage((SurfaceBytes + MaxSurfaces * Stride + Stride) / 4u, 0u); + auto* texels = reinterpret_cast((reinterpret_cast(storage.data()) + Stride - 1u) & ~std::uintptr_t{Stride - 1u}); + for (std::uint32_t surface = 0; surface < MaxSurfaces; ++surface) texels[surface * Stride / 4u] = Marker(surface); + const auto base = static_cast(reinterpret_cast(texels)); + const auto blockBytes = static_cast(SurfaceBytes + (MaxSurfaces - 1u) * Stride); + { + GuestAllocations::Mutation mutation; + mutation.Add(texels, blockBytes, true, true); + } + const auto release = [&] { + AgcDriver::Graphics::ClearCachedTextures(device->Device()); + GuestAllocations::Mutation mutation; + mutation.Remove(texels); + }; + try { + const auto first = Load(*device, base, 0); + Require(first != nullptr, "surface 0 has no cached storage image"); + const auto held = std::max(first->AllocationBytes(), first->GuestBytes()); + const auto reused = static_cast(FixedBudget / held) + 1u; + std::printf("texture cache budget %llu MiB; each 4096x4096 32_UINT surface holds %llu MiB; cyclic set of %u surfaces (%llu MiB)\n", static_cast(budget >> 20u), static_cast(held >> 20u), reused, static_cast((reused * held) >> 20u)); + if (budget < reused * held) { + std::printf("skipped, the device-local heap gives the texture caches less than the cyclic set\n"); + release(); + return VulkanTestSkipped; + } + std::vector> images(reused); + images[0] = first; + for (std::uint32_t surface = 1; surface < reused; ++surface) images[surface] = Load(*device, base, surface); + std::uint32_t remade = 0; + for (std::uint32_t surface = 0; surface < reused; ++surface) { + const auto again = Load(*device, base, surface); + if (again == nullptr || again != images[surface].lock()) ++remade; + } + Require(remade == 0, std::to_string(remade) + " of " + std::to_string(reused) + " storage images were made again on the second pass over a " + std::to_string((reused * held) >> 20u) + " MiB cyclic set under a " + std::to_string(budget >> 20u) + " MiB budget"); + const auto capacity = static_cast(budget / held); + if (selected.type == VK_PHYSICAL_DEVICE_TYPE_CPU || budget > EvictionLimit || capacity + 2u > MaxSurfaces) { + std::printf("eviction past the budget not tested on a CPU device or over a 4 GiB budget\n"); + } else { + for (std::uint32_t surface = reused; surface < capacity + 2u; ++surface) Load(*device, base, surface); + for (std::uint32_t surface = 0; surface < capacity + 2u; ++surface) { + const bool cached = StorageTexture::FindLive(base + surface * Stride, SurfaceBytes) != nullptr; + Require(cached == (surface >= 2u), "surface " + std::to_string(surface) + (cached ? " is still cached" : " was evicted") + " after " + std::to_string(capacity + 2u) + " surfaces under a budget of " + std::to_string(capacity) + " of them"); + } + } + } catch (...) { + release(); + throw; + } + release(); + std::puts("storage cache budget tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/TbufferFormatD16.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/TbufferFormatD16.cpp index 8367d9e9f..14588e3c6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/TbufferFormatD16.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/TbufferFormatD16.cpp @@ -24,8 +24,8 @@ alignas(256) std::array Input{}; alignas(256) std::array Output{}; alignas(256) constexpr std::array Code{ - 0x34020084, 0x34060086, 0xbe880300, 0xbe890301, 0xbe8a0302, 0xbe8b03ff, 0x0104bfac, 0xbe8c0300, - 0xbe8d0301, 0xbe8e0302, 0xbe8f03ff, 0x0104b80f, 0x7e1402ff, 0xdeadbeef, 0x7e1602ff, 0xdeadbeef, + 0x34020084, 0x34060086, 0xbe880300, 0xbe890301, 0xbe8a0302, 0xbe8b03ff, 0x3104bfac, 0xbe8c0300, + 0xbe8d0301, 0xbe8e0302, 0xbe8f03ff, 0x3104b80f, 0x7e1402ff, 0xdeadbeef, 0x7e1602ff, 0xdeadbeef, 0x7e1802ff, 0xdeadbeef, 0x7e1a02ff, 0xdeadbeef, 0x7e1c02ff, 0xdeadbeef, 0x7e1e02ff, 0xdeadbeef, 0x7e2002ff, 0xdeadbeef, 0x7e2202ff, 0xdeadbeef, 0x7e2402ff, 0xdeadbeef, 0x7e2602ff, 0xdeadbeef, 0x7e2802ff, 0xdeadbeef, 0x7e2a02ff, 0xdeadbeef, 0x7e2c02ff, 0xdeadbeef, 0x7e2e02ff, 0xdeadbeef, @@ -122,7 +122,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/TypedHeaps.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/TypedHeaps.cpp new file mode 100644 index 000000000..77b95c736 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/TypedHeaps.cpp @@ -0,0 +1,71 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +alignas(256) std::array, 4> texels{}; +alignas(256) std::array, 4> outputs{}; +constexpr std::array code{0x7e3c0300u, 0x7e3e0280u, 0xf0001108u, 0x00010a1eu, 0x34060082u, 0xe0701000u, 0x80000a03u, 0xbf810000u}; +constexpr std::array storeCode{0x7e3c0300u, 0x7e3e0280u, 0x7e1402ffu, 123u, 0xf0201108u, 0x00010a1eu, 0x34060082u, 0xe0701000u, 0x80000a03u, 0xbf810000u}; + +ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::uint32_t index, bool nullImage, bool store) { + const auto output = reinterpret_cast(outputs[index].data()); + const auto image = reinterpret_cast(texels[index].data()); + std::array userData{static_cast(output), static_cast(output >> 32u), 256u, 0x31016facu}; + if (!nullImage) { + const std::array descriptor{static_cast(image >> 8u), static_cast(image >> 40u) | (20u << 20u) | (3u << 30u), 7u, 0x90000facu, 0u, 0u, 0u, 0u}; + std::copy(descriptor.begin(), descriptor.end(), userData.begin() + 4u); + } + const ShaderRecompiler::ShaderComputeStageInfo compute{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + const auto instructions = store ? std::span(storeCode) : std::span(code); + ShaderRecompiler::RecompileRequest request{{ShaderStage::Compute, reinterpret_cast(instructions.data()), instructions, 0u, {}}, {32u, 0u, userData, compute, std::nullopt, std::nullopt, {}}, device.Target(), {0u, 0u, 0u, 128u}}; + auto result = ShaderRecompiler::Recompile(request); + Require(result.runtimeImageCount == 0u, "direct image allocated runtime image metadata"); + Require(result.shaderDataDwords < ShaderRecompiler::RuntimeAbi::ShaderDataDwords, "shader data was not compacted"); + for (const auto& binding : result.bindings) { + if (binding.role == ShaderRecompiler::DescriptorRole::GuestImages) Require(binding.count == 1u, "single image expanded into unused descriptor slots"); + } + return result; +} + +void Run(AgcDriver::VulkanDevice& device) { + std::vector shaders; + for (std::uint32_t batch = 0u; batch < outputs.size(); ++batch) { + outputs[batch].fill(0xdeadbeefu); + for (std::uint32_t pixel = 0u; pixel < 32u; ++pixel) texels[batch][pixel] = 1000u * (batch + 1u) + pixel; + const bool nullImage = batch >= 2u; + shaders.push_back(Compile(device, batch, nullImage, batch == 3u)); + device.Dispatch(shaders.back(), 1u, 1u, 1u); + device.SubmitRecorded(); + } + shaders.clear(); + device.WaitIdle(); + for (std::uint32_t batch = 0u; batch < outputs.size(); ++batch) { + for (std::uint32_t pixel = 0u; pixel < 32u; ++pixel) Require(outputs[batch][pixel] == (batch == 3u ? 123u : batch == 2u ? 0u : texels[batch][pixel]), "descriptor heap lifetime or null image semantics failed: batch=" + std::to_string(batch) + " pixel=" + std::to_string(pixel) + " value=" + std::to_string(outputs[batch][pixel])); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + std::cout << "typed heap execution tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/VertexFetch.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/VertexFetch.cpp new file mode 100644 index 000000000..4baa2d05b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/VertexFetch.cpp @@ -0,0 +1,129 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Extent = 64u; +alignas(256) std::array pixels{}; +alignas(256) std::array vertices{}; +alignas(256) std::array movedVertices{}; +constexpr std::array, 3> positions{{{-1.0f, -1.0f, 0.0f, 1.0f}, {3.0f, -1.0f, 0.0f, 1.0f}, {-1.0f, 3.0f, 0.0f, 1.0f}}}; +constexpr std::array vertexCode{0xf4080100u, 0xfa000000u, 0x4a0a0a02u, 0x4a0a0b08u, 0x4a0a0a03u, 0xe00c2000u, 0x80010005u, 0xbf8c3f70u, 0xf80008cfu, 0x03020100u, 0xbf810000u}; +constexpr std::array pixelCode{0x7e0002f2u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u}; + +void Run(AgcDriver::VulkanDevice& device) { + ShaderRecompiler::RecompileResult incompatible; + incompatible.runtimeAbiVersion = ShaderRecompiler::RuntimeAbi::Version + 1u; + const auto rejectAbi = [](auto action) { + try { + action(); + } catch (const std::runtime_error& error) { + Require(std::string(error.what()).find("incompatible version") != std::string::npos, "unexpected runtime ABI validation error"); + return; + } + throw std::runtime_error("driver accepted an incompatible shader runtime ABI"); + }; + rejectAbi([&] { device.PrepareDispatch(incompatible, {}); }); + rejectAbi([&] { device.Dispatch(incompatible, 1u, 1u, 1u); }); + rejectAbi([&] { device.DispatchIndirect(incompatible, 0u); }); + const std::array incompatibleShaders{{{ShaderStage::Vertex, &incompatible, 0u}}}; + rejectAbi([&] { device.Draw({}, {}, incompatibleShaders); }); + std::array descriptor{}; + const auto tableAddress = reinterpret_cast(descriptor.data()); + std::array userData{static_cast(tableAddress), static_cast(tableAddress >> 32u), 1u, 1u}; + const std::array memory{{{tableAddress, std::as_bytes(std::span(descriptor))}}}; + ShaderRecompiler::ShaderVertexStageInfo vertexInfo{}; + vertexInfo.fetchEmbedded = true; + vertexInfo.resourcesNum = 1u; + vertexInfo.resourcesDst[0] = {0, 4, 0, 0}; + ShaderRecompiler::RecompileRequest request{{ShaderStage::Vertex, reinterpret_cast(vertexCode.data()), vertexCode, 0, {}}, {32u, 0u, userData, std::nullopt, std::nullopt, vertexInfo, memory}, device.Target(), {0u, 0u, 0u, 96u}}; + ShaderRecompiler::RecompileResult first; + struct Case { + std::uint32_t stride; + std::uint32_t format; + bool moved; + std::uint32_t offset; + }; + constexpr std::array cases{{{16u, 77u, false, 1u}, {32u, 77u, true, 0u}, {8u, 71u, false, 1u}, {16u, 71u, true, 0u}}}; + for (const auto& value : cases) { + auto& storage = value.moved ? movedVertices : vertices; + storage.fill(std::byte{0}); + userData[2] = value.offset; + const auto firstRecord = value.offset + userData[3] + 2u; + for (std::size_t index = 0; index < positions.size(); ++index) { + auto* destination = storage.data() + (firstRecord + index) * value.stride; + if (value.format == 77u) std::memcpy(destination, positions[index].data(), 16u); + else { + for (std::size_t component = 0; component < 4u; ++component) { + const auto scalar = positions[index][component]; + const std::uint16_t half = scalar == -1.0f ? 0xbc00u : scalar == 3.0f ? 0x4200u : scalar == 1.0f ? 0x3c00u : 0u; + std::memcpy(destination + component * 2u, &half, sizeof(half)); + } + } + } + const auto address = reinterpret_cast(storage.data()); + descriptor = {static_cast(address), static_cast(address >> 32u) | (value.stride << 16u), 8u, 0x01000facu | (value.format << 12u)}; + request.context.vertex->resources[0].fields = descriptor; + const auto vertex = ShaderRecompiler::Recompile(request); + Require(vertex.vertexInputs.empty() && vertex.vertexAttributes.empty(), "runtime vertex fetch created Vulkan vertex attributes"); + Require(vertex.vertexOffsetSgpr == -1 && vertex.instanceOffsetSgpr == -1, "runtime vertex fetch requested draw offset folding"); + Require(vertex.runtimeAbiVersion == ShaderRecompiler::RuntimeAbi::Version, "runtime vertex fetch has an incompatible runtime ABI"); + if (first.spirv.empty()) first = vertex; + else Require(vertex.cacheHit && vertex.variantId == first.variantId, "runtime vertex descriptor changed the artifact"); + const auto pushBytes = static_cast(vertex.pushConstants.size()); + ShaderRecompiler::ShaderPixelStageInfo pixelInfo{}; + pixelInfo.wave32 = true; + pixelInfo.targetOutputMode[0] = 9u; + pixelInfo.targetExportMapping.fill(0xe4u); + ShaderRecompiler::RecompileRequest pixelRequest{{ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, {32u, 0u, {}, std::nullopt, pixelInfo, std::nullopt, {}}, device.Target(), {0u, 0u, pushBytes, 128u - pushBytes}}; + const auto pixel = ShaderRecompiler::Recompile(pixelRequest); + const std::array shaders{{{ShaderStage::Vertex, &vertex, 0u}, {ShaderStage::Fragment, &pixel, pushBytes}}}; + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(pixels.data()), {Extent, Extent}, VK_FORMAT_R8G8B8A8_UNORM, pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Extent, Extent}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0.0f, static_cast(Extent), static_cast(Extent), -static_cast(Extent), 0.0f, 1.0f}; + state.scissor = {{0, 0}, {Extent, Extent}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15u; + state.blends = {state.blend}; + AgcDriver::Pm4::DrawParameters draw{0u, 3u, 0u, 1u, 0u, false}; + draw.firstVertex = 1u; + draw.firstInstance = 1u; + pixels.fill(std::byte{0}); + device.Draw(state, draw, shaders); + device.WaitIdle(); + for (std::size_t index = 0; index < pixels.size(); index += 4u) { + Require(pixels[index] == std::byte{255} && pixels[index + 1u] == std::byte{0} && pixels[index + 2u] == std::byte{0} && pixels[index + 3u] == std::byte{255}, "runtime vertex fetch produced the wrong triangle"); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + std::cout << "runtime vertex fetch tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop1FloatUnary.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop1FloatUnary.cpp index 4832b5c96..4deb73f4d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop1FloatUnary.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop1FloatUnary.cpp @@ -155,7 +155,7 @@ void ExpectNear(std::uint32_t tid, std::uint32_t index, std::uint32_t actual, st std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { @@ -186,6 +186,7 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop1Vop2Integer.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop1Vop2Integer.cpp index 773fd3e38..4b66b7318 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop1Vop2Integer.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop1Vop2Integer.cpp @@ -112,7 +112,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop2ModifierForms.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop2ModifierForms.cpp index 9b07c7f68..79b89f448 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop2ModifierForms.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop2ModifierForms.cpp @@ -129,7 +129,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Aliases.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Aliases.cpp index 4b1889827..de1612097 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Aliases.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Aliases.cpp @@ -75,7 +75,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3AlignOpSel.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3AlignOpSel.cpp index 8a07028cb..6d13f2d29 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3AlignOpSel.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3AlignOpSel.cpp @@ -127,7 +127,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CarryOut.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CarryOut.cpp index 0517f1a51..fc85e504a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CarryOut.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CarryOut.cpp @@ -65,7 +65,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CompareModifiers.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CompareModifiers.cpp index 096295e0c..1cea63a13 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CompareModifiers.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CompareModifiers.cpp @@ -69,7 +69,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CubeClamp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CubeClamp.cpp index 07e25ea2b..d655f5f83 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CubeClamp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CubeClamp.cpp @@ -126,7 +126,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Integer.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Integer.cpp index 355272eaa..ab08a3e53 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Integer.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Integer.cpp @@ -54,7 +54,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Integer16TernaryOpSel.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Integer16TernaryOpSel.cpp index ba6169955..1558687a9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Integer16TernaryOpSel.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Integer16TernaryOpSel.cpp @@ -127,7 +127,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerAlu.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerAlu.cpp index 38c609632..23f1a1462 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerAlu.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerAlu.cpp @@ -120,7 +120,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerClamp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerClamp.cpp index 4bc1207d3..94f41e100 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerClamp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerClamp.cpp @@ -125,7 +125,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerClampWide.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerClampWide.cpp index adc82c757..0c6080743 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerClampWide.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3IntegerClampWide.cpp @@ -124,7 +124,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3LegacySourceModifiers.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3LegacySourceModifiers.cpp index 5562b5295..877cfd37a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3LegacySourceModifiers.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3LegacySourceModifiers.cpp @@ -123,7 +123,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Lerp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Lerp.cpp index dbfd4c211..a39f5bbb5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Lerp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Lerp.cpp @@ -52,7 +52,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3MadClamp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3MadClamp.cpp index 53aef39dc..26c69d11b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3MadClamp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3MadClamp.cpp @@ -106,7 +106,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3MadI64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3MadI64.cpp index 295d0c372..18c85a251 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3MadI64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3MadI64.cpp @@ -108,7 +108,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3PackLdexp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3PackLdexp.cpp index baa0da0a1..181109a98 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3PackLdexp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3PackLdexp.cpp @@ -108,7 +108,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3QuadSad.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3QuadSad.cpp index f8f913466..752ec94c3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3QuadSad.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3QuadSad.cpp @@ -109,7 +109,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Sad.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Sad.cpp index 8d4c14553..541e13104 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Sad.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3Sad.cpp @@ -55,7 +55,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } std::string Hex(std::uint32_t value) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompare.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompare.cpp index 969945ef3..81d7642aa 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompare.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompare.cpp @@ -120,7 +120,7 @@ void FillInput() { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompareF64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompareF64.cpp index 8a9f1b893..064d6a8bf 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompareF64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompareF64.cpp @@ -110,7 +110,7 @@ void FillInput(std::uint32_t round) { std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompareTable.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompareTable.cpp index 3cde131ad..98bf2e9b8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompareTable.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/VopcCompareTable.cpp @@ -728,7 +728,7 @@ constexpr const char* Names[Compares] = { std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } void Run(AgcDriver::VulkanDevice& device, std::span code, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp b/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp index a7908412e..bab6cab86 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp @@ -2,6 +2,9 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_VULKANTESTDEVICE_HPP #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include +#include +#include #include #include #include @@ -9,6 +12,11 @@ constexpr int VulkanTestSkipped = 77; +inline bool TargetHasCapability(const ShaderRecompiler::SpirvTarget& target, spv::Capability capability) { + const auto& capabilities = target.supportedCapabilities; + return std::find(capabilities.begin(), capabilities.end(), static_cast(capability)) != capabilities.end(); +} + inline std::unique_ptr OpenVulkanTestDevice() { try { return std::make_unique(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Wave32Subgroup.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Wave32Subgroup.cpp index afd510b19..127cca54c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Wave32Subgroup.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Wave32Subgroup.cpp @@ -28,7 +28,7 @@ alignas(256) constexpr std::array WaveCode{ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), count * 4u, 0x31016facu}; } void Run(AgcDriver::VulkanDevice& device, const ShaderRecompiler::SpirvTarget& target) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/WaveUniform.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/WaveUniform.cpp index f2341395f..3b5263ed4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/WaveUniform.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/WaveUniform.cpp @@ -25,7 +25,7 @@ alignas(256) constexpr std::array WaveCode{ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu), count * 4u, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), count * 4u, 0x31016facu}; } std::uint32_t Accumulated() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/WriteLane.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/WriteLane.cpp index 12258296f..50dbc453e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/WriteLane.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/WriteLane.cpp @@ -27,7 +27,7 @@ alignas(256) constexpr std::array WriteLaneCode{ std::array BufferDescriptor(const void* data, std::uint32_t count) { const auto address = reinterpret_cast(data); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x01016facu}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } void Run(AgcDriver::VulkanDevice& device) { diff --git a/core/libs/prx/libSceAgcDriver/tests/shaders/SampleArray.comp b/core/libs/prx/libSceAgcDriver/tests/shaders/SampleArray.comp new file mode 100644 index 000000000..15d0c8ee6 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/shaders/SampleArray.comp @@ -0,0 +1,10 @@ +#version 450 + +layout(local_size_x = 1) in; +layout(set = 0, binding = 0) uniform sampler2DArray source; +layout(set = 0, binding = 1, std430) buffer Result { vec4 texel; }; +layout(push_constant) uniform Parameters { float lod; float layer; } parameters; + +void main() { + texel = vec4(textureLod(source, vec3(0.5, 0.5, parameters.layer), parameters.lod)); +} diff --git a/core/libs/prx/libSceAjm.native/Export.cpp b/core/libs/prx/libSceAjm.native/Export.cpp index fcab9e41a..8f5edd6e9 100644 --- a/core/libs/prx/libSceAjm.native/Export.cpp +++ b/core/libs/prx/libSceAjm.native/Export.cpp @@ -55,15 +55,13 @@ const char* APS5_VABI sceAjmStrError(int error) { return nullptr; } -APS5_EXPORT("Z4HDpe1ZEa4", sceAjmWVorbisUnknown00); -int APS5_VABI sceAjmWVorbisUnknown00(void) { - NotImplemented_nid_no_patch("Z4HDpe1ZEa4"); +int APS5_VABI sceAjmDecWVorbisCreateHeaderPacket(void) { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("oXShHcqMcX0", sceAjmWVorbisUnknown01); -int APS5_VABI sceAjmWVorbisUnknown01(void) { - NotImplemented_nid_no_patch("oXShHcqMcX0"); +int APS5_VABI sceAjmDecWVorbisCreateSetupPacket(void) { + NotImplemented_nid_no_patch(__func__); return 0; } diff --git a/core/libs/prx/libSceAjm.native/src/Ajm.cpp b/core/libs/prx/libSceAjm.native/src/Ajm.cpp index bff995503..b7649e151 100644 --- a/core/libs/prx/libSceAjm.native/src/Ajm.cpp +++ b/core/libs/prx/libSceAjm.native/src/Ajm.cpp @@ -116,6 +116,9 @@ struct Instance { std::uint32_t codec = 0; std::uint64_t flags = 0; void* decoder = nullptr; + std::vector channelDecoders; + std::vector channelBytes; + std::vector blockPcm; Atrac9CodecInfo info{}; bool initialized = false; std::uint32_t superframeRemaining = 0; @@ -135,6 +138,7 @@ struct Instance { ~Instance() { if (decoder) Atrac9ReleaseHandle(decoder); + for (void* channelDecoder : channelDecoders) Atrac9ReleaseHandle(channelDecoder); if (mp3) avcodec_free_context(&mp3); if (opus) avcodec_free_context(&opus); } @@ -237,16 +241,113 @@ Instance* Find(std::uint32_t id) { return found == g_instances.end() ? nullptr : found->second.get(); } -// Replaces the instance's decoder with a fresh one for its configuration, at a superframe boundary. The +constexpr std::uint8_t AT9_MULTICHANNEL_HEADER = 0x30; + +struct At9MultichannelConfig { + std::uint32_t channels; + std::uint32_t frameBytes; + std::uint32_t framesInSuperframe; + unsigned char monoConfig[ATRAC9_CONFIG_DATA_SIZE]; +}; + +bool ParseMultichannelConfig(const std::uint8_t* config, At9MultichannelConfig& parsed) { + if (config[0] != AT9_MULTICHANNEL_HEADER) return false; + const std::uint32_t bits = (std::uint32_t{config[1]} << 16) | (std::uint32_t{config[2]} << 8) | config[3]; + if ((bits >> 13) & 1u) return false; + const std::uint32_t sampleRateIndex = (bits >> 20) & 0xFu; + const std::uint32_t superframeIndex = bits & 3u; + parsed.channels = ((bits >> 14) & 0x3Fu) + 1; + parsed.frameBytes = ((bits >> 2) & 0x7FFu) + 1; + parsed.framesInSuperframe = 1u << superframeIndex; + const std::uint32_t mono = (0xFEu << 24) | (sampleRateIndex << 20) | ((parsed.frameBytes - 1) << 5) | (superframeIndex << 3); + for (std::size_t i = 0; i < ATRAC9_CONFIG_DATA_SIZE; ++i) parsed.monoConfig[i] = static_cast(mono >> (24 - 8 * i)); + return true; +} + +bool DescribeAt9Config(const std::uint8_t* configData, Atrac9CodecInfo& info) { + At9MultichannelConfig multichannel{}; + const bool isMultichannel = ParseMultichannelConfig(configData, multichannel); + unsigned char config[ATRAC9_CONFIG_DATA_SIZE]; + std::memcpy(config, isMultichannel ? multichannel.monoConfig : configData, sizeof(config)); + void* decoder = Atrac9GetHandle(); + const bool valid = Atrac9InitDecoder(decoder, config) == 0; + if (valid) Atrac9GetCodecInfo(decoder, &info); + Atrac9ReleaseHandle(decoder); + if (!valid) return false; + if (isMultichannel) { + info.channels = static_cast(multichannel.channels); + info.channelConfigIndex = -1; + info.superframeSize = static_cast(multichannel.channels * multichannel.frameBytes * multichannel.framesInSuperframe); + } + std::memcpy(info.configData, configData, ATRAC9_CONFIG_DATA_SIZE); + return true; +} + // decoder handle keeps its position within the superframe, which a re-initialization does not clear. bool ResetDecoder(Instance& instance) { if (instance.decoder) Atrac9ReleaseHandle(instance.decoder); - instance.decoder = Atrac9GetHandle(); + instance.decoder = nullptr; + for (void* channelDecoder : instance.channelDecoders) Atrac9ReleaseHandle(channelDecoder); + instance.channelDecoders.clear(); instance.superframeRemaining = 0; instance.frameInSuperframe = 0; unsigned char config[ATRAC9_CONFIG_DATA_SIZE]; std::memcpy(config, instance.info.configData, sizeof(config)); - return Atrac9InitDecoder(instance.decoder, config) == 0; + At9MultichannelConfig multichannel{}; + if (!ParseMultichannelConfig(config, multichannel)) { + instance.decoder = Atrac9GetHandle(); + return Atrac9InitDecoder(instance.decoder, config) == 0; + } + instance.channelBytes.assign(multichannel.channels, 0); + for (std::uint32_t channel = 0; channel < multichannel.channels; ++channel) { + instance.channelDecoders.push_back(Atrac9GetHandle()); + if (Atrac9InitDecoder(instance.channelDecoders.back(), multichannel.monoConfig) != 0) return false; + } + return true; +} + +bool HasAt9Decoder(const Instance& instance) { + return instance.decoder || !instance.channelDecoders.empty(); +} + +int DecodeAt9Block(void* decoder, std::uint32_t encoding, const std::uint8_t* data, int size, std::uint8_t* pcm, int* used) { + switch (encoding) { + case 0: return Atrac9Decode(decoder, data, size, reinterpret_cast(pcm), used, 0); + case 1: return Atrac9DecodeS32(decoder, data, size, reinterpret_cast(pcm), used, 0); + default: return Atrac9DecodeF32(decoder, data, size, reinterpret_cast(pcm), used, 0); + } +} + +constexpr int AT9_MULTICHANNEL_BLOCK_OVERRUN = 0x7F000001; + +int DecodeAt9Frame(Instance& instance, const std::uint8_t* data, int size, std::uint8_t* pcm, std::uint32_t encoding, std::size_t sampleBytes, int* used) { + if (instance.channelDecoders.empty()) return DecodeAt9Block(instance.decoder, encoding, data, size, pcm, used); + const std::size_t channels = instance.channelDecoders.size(); + const auto frameSamples = static_cast(instance.info.frameSamples); + const auto framesInSuperframe = static_cast(std::max(1, instance.info.framesInSuperframe)); + const auto budget = static_cast(instance.info.superframeSize / static_cast(channels)); + const bool lastFrame = instance.frameInSuperframe + 1 >= framesInSuperframe; + if (instance.frameInSuperframe == 0) std::fill(instance.channelBytes.begin(), instance.channelBytes.end(), 0u); + instance.blockPcm.resize(frameSamples * sampleBytes); + int offset = 0; + for (std::size_t channel = 0; channel < channels; ++channel) { + const std::uint32_t left = budget - std::min(budget, instance.channelBytes[channel]); + const int available = std::min(static_cast(left), size - offset); + if (available <= 0) return AT9_MULTICHANNEL_BLOCK_OVERRUN; + int blockUsed = 0; + const int status = DecodeAt9Block(instance.channelDecoders[channel], encoding, data + offset, available, instance.blockPcm.data(), &blockUsed); + if (status != 0) return status; + if (blockUsed <= 0 || blockUsed > available) return AT9_MULTICHANNEL_BLOCK_OVERRUN; + const std::uint32_t blockBytes = lastFrame ? left : static_cast(blockUsed); + if (static_cast(blockBytes) > size - offset) return AT9_MULTICHANNEL_BLOCK_OVERRUN; + for (std::size_t sample = 0; sample < frameSamples; ++sample) { + std::memcpy(pcm + (sample * channels + channel) * sampleBytes, instance.blockPcm.data() + sample * sampleBytes, sampleBytes); + } + instance.channelBytes[channel] += blockBytes; + offset += static_cast(blockBytes); + } + *used = offset; + return 0; } void OpenOpus(Instance& instance); @@ -283,11 +384,12 @@ std::int32_t InitializeInstance(Instance& instance, const std::uint8_t* paramete } if (size < ATRAC9_CONFIG_DATA_SIZE) return AJM_RESULT_INVALID_PARAMETER; std::memcpy(instance.info.configData, parameters, ATRAC9_CONFIG_DATA_SIZE); - if (!ResetDecoder(instance)) { + std::uint8_t config[ATRAC9_CONFIG_DATA_SIZE]; + std::memcpy(config, parameters, sizeof(config)); + if (!ResetDecoder(instance) || !DescribeAt9Config(config, instance.info)) { instance.initialized = false; return AJM_RESULT_INVALID_PARAMETER; } - Atrac9GetCodecInfo(instance.decoder, &instance.info); instance.initialized = true; instance.totalDecodedSamples = 0; instance.gapless = {}; @@ -541,11 +643,7 @@ void RunAt9(Instance& instance, const JobHeader& job, const AjmBuffer* inputs, c instance.frameInSuperframe = 0; } int used = 0; - switch (encoding) { - case 0: decodeStatus = Atrac9Decode(instance.decoder, input.data() + consumed, static_cast(instance.superframeRemaining), reinterpret_cast(pcm.data()), &used, 0); break; - case 1: decodeStatus = Atrac9DecodeS32(instance.decoder, input.data() + consumed, static_cast(instance.superframeRemaining), reinterpret_cast(pcm.data()), &used, 0); break; - default: decodeStatus = Atrac9DecodeF32(instance.decoder, input.data() + consumed, static_cast(instance.superframeRemaining), reinterpret_cast(pcm.data()), &used, 0); break; - } + decodeStatus = DecodeAt9Frame(instance, input.data() + consumed, static_cast(instance.superframeRemaining), pcm.data(), encoding, sampleBytes, &used); if (decodeStatus != 0 || used <= 0 || static_cast(used) > instance.superframeRemaining) { result |= AJM_RESULT_INVALID_DATA; // The decoder tracks its position in the superframe; a fresh one starts the next superframe. @@ -962,7 +1060,7 @@ void RunOpus(Instance& instance, const JobHeader& job, const AjmBuffer* inputs, void ClearContext(Instance& instance) { instance.totalDecodedSamples = 0; instance.gapless.skippedSamples = 0; - if (instance.decoder && instance.initialized) ResetDecoder(instance); + if (HasAt9Decoder(instance) && instance.initialized) ResetDecoder(instance); if (instance.mp3) avcodec_flush_buffers(instance.mp3); if (instance.opus) avcodec_flush_buffers(instance.opus); instance.opusPending.clear(); @@ -1150,13 +1248,10 @@ int APS5_VABI sceAjmInstanceDestroy(uint32_t context, uint32_t instance) { int APS5_VABI sceAjmDecAt9ParseConfigData(const void* config_data, AjmDecAt9ConfigDataInfo* config_info) { if (!config_data || !config_info) return SCE_AJM_ERROR_INVALID_PARAMETER; - void* decoder = Atrac9GetHandle(); - unsigned char config[ATRAC9_CONFIG_DATA_SIZE]; + std::uint8_t config[ATRAC9_CONFIG_DATA_SIZE]; std::memcpy(config, config_data, sizeof(config)); - const bool valid = Atrac9InitDecoder(decoder, config) == 0; Atrac9CodecInfo info{}; - if (valid) Atrac9GetCodecInfo(decoder, &info); - Atrac9ReleaseHandle(decoder); + const bool valid = DescribeAt9Config(config, info); AJM_TRACE("[ajm] parse config %02x %02x %02x %02x -> %s, %d channels, %d Hz, superframe %d bytes (%d frames of %d samples)\n", config[0], config[1], config[2], config[3], valid ? "ok" : "invalid", info.channels, info.samplingRate, info.superframeSize, info.framesInSuperframe, info.frameSamples); if (!valid) return SCE_AJM_ERROR_INVALID_PARAMETER; config_info->channels = static_cast(info.channels); diff --git a/core/libs/prx/libSceAmpr/Export.cpp b/core/libs/prx/libSceAmpr/Export.cpp index 1c4c227f0..6105b3e46 100644 --- a/core/libs/prx/libSceAmpr/Export.cpp +++ b/core/libs/prx/libSceAmpr/Export.cpp @@ -320,12 +320,13 @@ int APS5_VABI sceAmprAprCommandBufferResetGatherScatterState(Apr::CommandBufferO return result; } -int APS5_VABI sceAmprCommandBufferClearBuffer(Apr::CommandBufferObject* buffer) { +void* APS5_VABI sceAmprCommandBufferClearBuffer(Apr::CommandBufferObject* buffer) { + void* memory = buffer->base; buffer->base = nullptr; buffer->size = 0; buffer->offset = 0; buffer->numCommands = 0; - return 0; + return memory; } int APS5_VABI sceAmprCommandBufferConstructMarker(Apr::CommandBufferObject* buffer, std::uint32_t type, const char* text, const std::uint32_t* color) { diff --git a/core/libs/prx/libSceAppContent/Export.cpp b/core/libs/prx/libSceAppContent/Export.cpp index b02215364..695437ff9 100644 --- a/core/libs/prx/libSceAppContent/Export.cpp +++ b/core/libs/prx/libSceAppContent/Export.cpp @@ -6,13 +6,16 @@ #include #include #include +#include #include #include static constexpr int SCE_APP_CONTENT_ERROR_PARAMETER = static_cast(0x80D90002); +static constexpr int SCE_APP_CONTENT_ERROR_BUSY = static_cast(0x80D90003); +static constexpr int SCE_APP_CONTENT_ERROR_NOT_MOUNTED = static_cast(0x80D90004); static constexpr int SCE_APP_CONTENT_ERROR_NOT_FOUND = static_cast(0x80D90005); static constexpr int SCE_APP_CONTENT_ERROR_DRM_NO_ENTITLEMENT = static_cast(0x80D90007); -static constexpr uint32_t APPPARAM_ID_SKU_FLAG = 1; +static constexpr uint32_t APPPARAM_ID_SKU_FLAG = 0; static constexpr int32_t SKU_FLAG_FULL = 3; static constexpr char TEMPORARY_MOUNT_POINT[] = "/temp0"; @@ -24,13 +27,30 @@ struct AppContentAddcontInfo { uint32_t status; }; -static std::filesystem::path TemporaryDirectory(const AppContentMountPoint* mount_point) { - if (!mount_point || std::strncmp(mount_point->data, TEMPORARY_MOUNT_POINT, sizeof(mount_point->data)) != 0) APS5_INVALID_ARG_EX; - return ResolvePath_nid_no_patch(TEMPORARY_MOUNT_POINT); +namespace { + +struct TemporaryData { + std::mutex mutex; + const std::filesystem::path directory = ResolvePath_nid_no_patch(TEMPORARY_MOUNT_POINT); + bool mounted = false; + + TemporaryData() { BlockPathAlias_nid_no_patch(TEMPORARY_MOUNT_POINT); } +}; + +TemporaryData& Temporary() { + static TemporaryData state; + return state; +} + +bool IsTemporaryMountPoint(const AppContentMountPoint* mountPoint) { + return std::strncmp(mountPoint->data, TEMPORARY_MOUNT_POINT, sizeof(mountPoint->data)) == 0; +} + } static void ClearDirectory(const std::filesystem::path& directory) { for (const auto& entry : std::filesystem::directory_iterator(directory)) std::filesystem::remove_all(entry.path()); + RecordWrittenPath_nid_no_patch(directory); } extern "C" { @@ -65,8 +85,8 @@ int APS5_VABI sceAppContentAppParamGetInt(uint32_t param_id, int32_t* value) { case APPPARAM_ID_SKU_FLAG: *value = SKU_FLAG_FULL; return 0; - case 2: case 3: case 4: case 5: - *value = 0; + case 1: case 2: case 3: case 4: + *value = GetAppUserDefinedParam_nid_postfix(param_id - 1); return 0; default: return SCE_APP_CONTENT_ERROR_PARAMETER; @@ -82,7 +102,7 @@ int APS5_VABI sceAppContentDownloadDataGetAvailableSpaceKb(const AppContentMount return 0; } const auto directory = ResolvePath_nid_no_patch(DOWNLOAD_MOUNT_POINT); - std::filesystem::create_directories(directory); + if (std::filesystem::create_directories(directory)) RecordWrittenPath_nid_no_patch(directory); std::uint64_t usedKb = 0; for (const auto& entry : std::filesystem::recursive_directory_iterator(directory)) { if (entry.is_regular_file()) usedKb += (entry.file_size() + 1023u) / 1024u; @@ -95,36 +115,56 @@ int APS5_VABI sceAppContentDownloadDataGetAvailableSpaceKb(const AppContentMount int APS5_VABI sceAppContentInitialize(const AppContentInitParam* init_param, AppContentBootParam* boot_param) { (void)init_param; if (!boot_param) return SCE_APP_CONTENT_ERROR_PARAMETER; + Temporary(); std::memset(boot_param, 0, sizeof(*boot_param)); return 0; } -int APS5_VABI sceAppContentTemporaryDataFormat(const AppContentMountPoint* mount_point) { - ClearDirectory(TemporaryDirectory(mount_point)); +int APS5_VABI sceAppContentTemporaryDataFormat(const AppContentMountPoint* mountPoint) { + if (!mountPoint) return SCE_APP_CONTENT_ERROR_PARAMETER; + auto& state = Temporary(); + std::lock_guard lock(state.mutex); + if (!state.mounted || !IsTemporaryMountPoint(mountPoint)) return SCE_APP_CONTENT_ERROR_NOT_MOUNTED; + ClearDirectory(state.directory); return 0; } -int APS5_VABI sceAppContentTemporaryDataGetAvailableSpaceKb(const AppContentMountPoint* mount_point, size_t* available_space_kb) { - if (!available_space_kb) return SCE_APP_CONTENT_ERROR_PARAMETER; - *available_space_kb = static_cast(std::filesystem::space(TemporaryDirectory(mount_point)).available / 1024); +int APS5_VABI sceAppContentTemporaryDataGetAvailableSpaceKb(const AppContentMountPoint* mountPoint, size_t* availableSpaceKb) { + if (!mountPoint || !availableSpaceKb) return SCE_APP_CONTENT_ERROR_PARAMETER; + auto& state = Temporary(); + std::lock_guard lock(state.mutex); + if (!state.mounted || !IsTemporaryMountPoint(mountPoint)) return SCE_APP_CONTENT_ERROR_NOT_MOUNTED; + *availableSpaceKb = static_cast(std::filesystem::space(state.directory).available / 1024); return 0; } -int APS5_VABI sceAppContentTemporaryDataMount2(uint32_t option, AppContentMountPoint* mount_point) { - if (!mount_point) return SCE_APP_CONTENT_ERROR_PARAMETER; - if (option > TEMPORARY_DATA_OPTION_FORMAT) throw std::invalid_argument(std::string(__func__) + ": unknown option " + std::to_string(option)); - std::memset(mount_point->data, 0, sizeof(mount_point->data)); - std::memcpy(mount_point->data, TEMPORARY_MOUNT_POINT, sizeof(TEMPORARY_MOUNT_POINT)); - const auto directory = TemporaryDirectory(mount_point); - std::filesystem::create_directories(directory); - if (option == TEMPORARY_DATA_OPTION_FORMAT) ClearDirectory(directory); +int APS5_VABI sceAppContentTemporaryDataMount2(uint32_t option, AppContentMountPoint* mountPoint) { + if (!mountPoint || option > TEMPORARY_DATA_OPTION_FORMAT) return SCE_APP_CONTENT_ERROR_PARAMETER; + auto& state = Temporary(); + std::lock_guard lock(state.mutex); + if (state.mounted) return SCE_APP_CONTENT_ERROR_BUSY; + if (std::filesystem::create_directories(state.directory)) RecordWrittenPath_nid_no_patch(state.directory); + if (option == TEMPORARY_DATA_OPTION_FORMAT) ClearDirectory(state.directory); + AddPathAlias_nid_no_patch(TEMPORARY_MOUNT_POINT, state.directory.string().c_str()); + state.mounted = true; + std::memset(mountPoint->data, 0, sizeof(mountPoint->data)); + std::memcpy(mountPoint->data, TEMPORARY_MOUNT_POINT, sizeof(TEMPORARY_MOUNT_POINT)); + return 0; +} + +int APS5_VABI sceAppContentTemporaryDataUnmount(const AppContentMountPoint* mountPoint) { + if (!mountPoint) return SCE_APP_CONTENT_ERROR_PARAMETER; + auto& state = Temporary(); + std::lock_guard lock(state.mutex); + if (!state.mounted || !IsTemporaryMountPoint(mountPoint)) return SCE_APP_CONTENT_ERROR_NOT_MOUNTED; + BlockPathAlias_nid_no_patch(TEMPORARY_MOUNT_POINT); + state.mounted = false; return 0; } -APS5_EXPORT("7gxh+5QubhY", sceAppContentUnknown00); -int APS5_VABI sceAppContentUnknown00(void) { - NotImplemented_nid_no_patch("7gxh+5QubhY"); +int APS5_VABI sceAppContentAddcontEnqueueDownload(void) { + NotImplemented_nid_no_patch(__func__); return 0; } } diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut2Mastering.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut2Mastering.cpp index 4cbf76ef1..0d437bbb8 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut2Mastering.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut2Mastering.cpp @@ -10,7 +10,6 @@ int APS5_VABI sceAudioOut2MasteringInit(uint32_t flags) { } int APS5_VABI sceAudioOut2MasteringTerm(void) { - NotImplemented_nid_no_patch(__func__); return 0; } diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp index 5db475551..03248d3e0 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp @@ -28,6 +28,9 @@ static constexpr std::uint32_t FORMAT_CHANNELS_SHIFT = 8; static constexpr std::uint32_t FORMAT_CHANNELS_MASK = 0xFu; static constexpr std::uint32_t FORMAT_TYPE_MASK = 0x7Fu; static constexpr std::uint32_t FORMAT_FIELDS_MASK = 0xFFFu; +static constexpr std::uint16_t PORT_TYPE_MAIN = 0; +static constexpr int PORT_ERROR_FORMAT_FLAG_WITHOUT_8_CHANNELS = static_cast(0x80268001); +static constexpr int PORT_ERROR_UNSUPPORTED_FORMAT = static_cast(0x8026800E); static std::mutex g_portsLock; // Grows on demand: the title opens its bed ports plus max_object_ports object ports at once. @@ -158,6 +161,19 @@ extern "C" { int APS5_VABI sceAudioOut2PortCreate(AudioOut2ContextHandle ctx, const AudioOut2PortParam* params, AudioOut2PortHandle* port) { if (!ctx) return SCE_AUDIO_OUT2_ERROR_INVALID_HANDLE; if (!params || !port) return SCE_AUDIO_OUT2_ERROR_INVALID_ARGUMENT; + const auto channels = (params->data_format >> FORMAT_CHANNELS_SHIFT) & FORMAT_CHANNELS_MASK; + const auto sampleType = params->data_format & FORMAT_TYPE_MASK; + const auto* fold = StereoFoldFor(channels); + if (params->port_type == PORT_TYPE_MAIN) { + if ((params->data_format & 0x80u) != 0 && channels != 8) { + *port = static_cast(-1); + return PORT_ERROR_FORMAT_FLAG_WITHOUT_8_CHANNELS; + } + if (fold == nullptr || channels == 6 || sampleType > 1) { + *port = static_cast(-1); + return PORT_ERROR_UNSUPPORTED_FORMAT; + } + } std::lock_guard lock(g_portsLock); std::size_t index = 0; while (index < g_ports.size() && g_ports[index].used) index++; @@ -171,10 +187,9 @@ int APS5_VABI sceAudioOut2PortCreate(AudioOut2ContextHandle ctx, const AudioOut2 entry.dataFormat = params->data_format; entry.samplingFreq = params->sampling_freq; entry.flags = params->flags; - entry.channels = (params->data_format >> FORMAT_CHANNELS_SHIFT) & FORMAT_CHANNELS_MASK; - const auto sampleType = params->data_format & FORMAT_TYPE_MASK; - entry.fold = StereoFoldFor(entry.channels); - if (entry.fold == nullptr || sampleType > 1 || (params->data_format & ~FORMAT_FIELDS_MASK) != 0) { + entry.channels = channels; + entry.fold = fold; + if (entry.fold == nullptr || sampleType > 1 || (params->port_type != PORT_TYPE_MAIN && (params->data_format & ~FORMAT_FIELDS_MASK) != 0)) { entry = AudioOut2Port{}; throw std::runtime_error("sceAudioOut2PortCreate: data format 0x" + [&] { char text[16]; std::snprintf(text, sizeof(text), "%x", params->data_format); return std::string(text); }() + " is not implemented"); } diff --git a/core/libs/prx/libSceAudioOut/tests/AudioOut2Latency.cpp b/core/libs/prx/libSceAudioOut/tests/AudioOut2Latency.cpp index 5c655c401..5f6eea7b1 100644 --- a/core/libs/prx/libSceAudioOut/tests/AudioOut2Latency.cpp +++ b/core/libs/prx/libSceAudioOut/tests/AudioOut2Latency.cpp @@ -9,6 +9,7 @@ extern "C" { int APS5_VABI sceAudioOut2Initialize(); int APS5_VABI sceAudioOut2Set3DLatency(int, std::uint32_t); int APS5_VABI sceAudioOut2MasteringInit(std::uint32_t); +int APS5_VABI sceAudioOut2MasteringTerm(); int APS5_VABI sceAudioOut2MasteringSetParam(const void*, std::uint32_t, std::uint32_t); } @@ -47,6 +48,12 @@ void TestMasteringInit() { Require(ThrowsRuntimeError([] { sceAudioOut2MasteringInit(1); }), "flags 1 must throw"); } +void TestMasteringTerm() { + Require(sceAudioOut2MasteringInit(0) == 0, "a second initialization must be accepted"); + Require(sceAudioOut2MasteringTerm() == 0, "termination must be accepted"); + Require(sceAudioOut2MasteringTerm() == 0, "termination must be accepted for each initialization"); +} + void TestMasteringSetParam() { const std::uint32_t params[4] = {1u, 0u, 0u, 0u}; Require(sceAudioOut2MasteringSetParam(params, 0, 0) == 0, "mastering parameters must be accepted"); @@ -59,6 +66,7 @@ int main() { Require(sceAudioOut2Initialize() == 0, "initialization must succeed"); TestSet3DLatency(); TestMasteringInit(); + TestMasteringTerm(); TestMasteringSetParam(); return 0; } diff --git a/core/libs/prx/libSceAudioOut/tests/AudioOut2PortLayouts.cpp b/core/libs/prx/libSceAudioOut/tests/AudioOut2PortLayouts.cpp index 65aa267f7..56648e1b0 100644 --- a/core/libs/prx/libSceAudioOut/tests/AudioOut2PortLayouts.cpp +++ b/core/libs/prx/libSceAudioOut/tests/AudioOut2PortLayouts.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include @@ -32,16 +32,6 @@ static void SetEnvironment(const char* name, const std::string& value) { #endif } -template -static bool ThrowsRuntimeError(TFunction function) { - try { - function(); - } catch (const std::runtime_error&) { - return true; - } - return false; -} - namespace { constexpr std::uint32_t grain = 256; @@ -64,7 +54,6 @@ struct Layout { const Layout mono{1, {1.0f}, {1.0f}}; const Layout stereo{2, {1.0f, 0.0f}, {0.0f, 1.0f}}; -const Layout surround51{6, {1.0f, 0.0f, fold, 0.0f, fold, 0.0f}, {0.0f, 1.0f, fold, 0.0f, 0.0f, fold}}; const Layout surround71{8, {1.0f, 0.0f, fold, 0.0f, fold, 0.0f, fold, 0.0f}, {0.0f, 1.0f, fold, 0.0f, 0.0f, fold, 0.0f, fold}}; const Layout surround714{12, {1.0f, 0.0f, fold, 0.0f, fold, 0.0f, fold, 0.0f, fold, 0.0f, fold * fold, 0.0f}, {0.0f, 1.0f, fold, 0.0f, 0.0f, fold, 0.0f, fold, 0.0f, fold, 0.0f, fold * fold}}; @@ -103,7 +92,7 @@ void SetVolume(AudioOut2PortHandle port, const std::vector& volume) { } std::vector Play(std::uint32_t format, const void* data, const std::vector& volume) { - const auto path = std::filesystem::temp_directory_path() / "anyps5_audio_out2_port_layouts.raw"; + const auto path = std::filesystem::temp_directory_path() / ("anyps5_audio_out2_port_layouts-" + std::to_string(std::random_device{}()) + ".raw"); std::filesystem::remove(path); SetEnvironment("SDL_DISKAUDIOFILE", path.string()); @@ -215,20 +204,49 @@ void TestDroppedLfe() { RequireFold(Play(Format(surround714.channels, formatFloat), data.data(), volume), surround714, expected, volume, 1.0f); } -void TestRejectedLayouts() { +void TestMeasuredMainPortFormats() { + struct Case { + std::uint32_t format; + std::uint32_t result; + }; + constexpr Case cases[] = { + {0x000, 0x8026800E}, {0x001, 0x8026800E}, {0x080, 0x80268001}, + {0x100, 0}, {0x101, 0}, {0x102, 0x8026800E}, {0x17F, 0x8026800E}, + {0x180, 0x80268001}, {0x181, 0x80268001}, {0x1FF, 0x80268001}, + {0x200, 0}, {0x201, 0}, {0x202, 0x8026800E}, {0x27F, 0x8026800E}, + {0x280, 0x80268001}, {0x281, 0x80268001}, {0x2FF, 0x80268001}, + {0x300, 0x8026800E}, {0x301, 0x8026800E}, {0x380, 0x80268001}, + {0x400, 0x8026800E}, {0x401, 0x8026800E}, {0x500, 0x8026800E}, + {0x600, 0x8026800E}, {0x601, 0x8026800E}, {0x680, 0x80268001}, + {0x700, 0x8026800E}, {0x701, 0x8026800E}, {0x780, 0x80268001}, + {0x800, 0}, {0x801, 0}, {0x802, 0x8026800E}, {0x87F, 0x8026800E}, + {0x880, 0}, {0x881, 0}, {0x882, 0x8026800E}, {0x8FF, 0x8026800E}, + {0x900, 0x8026800E}, {0x901, 0x8026800E}, {0x980, 0x80268001}, + {0xA00, 0x8026800E}, {0xB00, 0x8026800E}, {0xC00, 0}, {0xC01, 0}, + {0xC02, 0x8026800E}, {0xC7F, 0x8026800E}, {0xC80, 0x80268001}, + {0xC81, 0x80268001}, {0xD00, 0x8026800E}, {0xE00, 0x8026800E}, + {0xF00, 0x8026800E}, {0xFFF, 0x80268001}, + {0xFFFFF100, 0}, {0xFFFFF201, 0}, {0xFFFFF800, 0}, {0xFFFFFC01, 0}, + {0xFFFFF600, 0x8026800E}, {0xFFFFF202, 0x8026800E}, {0xFFFFF280, 0x80268001}, + }; const auto context = CreateContext(); - for (const std::uint32_t channels : {0u, 3u, 4u, 5u, 7u, 9u, 10u, 11u, 13u, 15u}) { - AudioOut2PortHandle port = 0; - Require(ThrowsRuntimeError([&] { CreatePort(context, Format(channels, formatFloat), &port); })); - Require(ThrowsRuntimeError([&] { CreatePort(context, Format(channels, formatS16), &port); })); + for (const auto& test : cases) { + AudioOut2PortHandle port = 0xAAAAAAAAAAAAAAAAull; + Require(static_cast(CreatePort(context, test.format, &port)) == test.result); + if (test.result == 0) { + Require(port != 0 && port != static_cast(-1)); + Require(sceAudioOut2PortDestroy(port) == 0); + } else { + Require(port == static_cast(-1)); + } + } + for (const std::uint32_t format : {0x100u, 0x201u, 0x800u, 0xC01u}) { + for (std::uint32_t bit = 12; bit < 32; bit++) { + AudioOut2PortHandle port = 0; + Require(CreatePort(context, format | (1u << bit), &port) == 0); + Require(sceAudioOut2PortDestroy(port) == 0); + } } - AudioOut2PortHandle port = 0; - Require(ThrowsRuntimeError([&] { CreatePort(context, Format(12, 2), &port); })); - Require(ThrowsRuntimeError([&] { CreatePort(context, Format(12, formatFloat) | 0x1000, &port); })); - Require(ThrowsRuntimeError([&] { CreatePort(context, Format(12, formatFloat) | 0x10000, &port); })); - Require(CreatePort(context, Format(12, formatFloat), &port) == 0); - Require(port != 0); - Require(sceAudioOut2PortDestroy(port) == 0); Require(sceAudioOut2ContextDestroy(context) == 0); } @@ -236,7 +254,7 @@ void TestRejectedLayouts() { int main() { SetEnvironment("SDL_AUDIODRIVER", "disk"); - for (const Layout* layout : {&mono, &stereo, &surround51, &surround71, &surround714}) { + for (const Layout* layout : {&mono, &stereo, &surround71, &surround714}) { TestFloatLayout(*layout); TestS16Layout(*layout); } @@ -245,8 +263,13 @@ int main() { const auto volume = Volume(8); RequireFold(Play(Format(8, formatFloat) | 0x80, data.data(), volume), surround71, data, volume, 1.0f); } + { + const auto data = FloatGrain(2); + const auto volume = Volume(2); + RequireFold(Play(0xFFFFF200u, data.data(), volume), stereo, data, volume, 1.0f); + } TestHeightChannels(); TestDroppedLfe(); - TestRejectedLayouts(); + TestMeasuredMainPortFormats(); return 0; } diff --git a/core/libs/prx/libSceAudioOut/tests/AudioOut2Timing.cpp b/core/libs/prx/libSceAudioOut/tests/AudioOut2Timing.cpp index b3f670c35..c16ffebbf 100644 --- a/core/libs/prx/libSceAudioOut/tests/AudioOut2Timing.cpp +++ b/core/libs/prx/libSceAudioOut/tests/AudioOut2Timing.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -52,7 +53,7 @@ struct Session { AudioOut2PortHandle port = 0; std::vector buffer = std::vector(static_cast(grain) * 2, 0.0f); - explicit Session(const char* name) : path(std::filesystem::temp_directory_path() / name) { + explicit Session(const char* name) : path(std::filesystem::temp_directory_path() / (std::to_string(std::random_device{}()) + "-" + name)) { std::filesystem::remove(path); SetEnvironment("SDL_DISKAUDIOFILE", path.string()); AudioOut2ContextParam params{}; diff --git a/core/libs/prx/libSceAudioOut/tests/AudioOutMixLevelPadSpk.cpp b/core/libs/prx/libSceAudioOut/tests/AudioOutMixLevelPadSpk.cpp index 6a889cb8b..9c7ac4e9a 100644 --- a/core/libs/prx/libSceAudioOut/tests/AudioOutMixLevelPadSpk.cpp +++ b/core/libs/prx/libSceAudioOut/tests/AudioOutMixLevelPadSpk.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -55,7 +56,7 @@ constexpr int invalidMixLevel = static_cast(0x80260014); template std::vector Play(int type, std::uint32_t format, const std::vector& block, TSetup setup) { - const auto path = std::filesystem::temp_directory_path() / "anyps5_audio_out_mix_level_pad_spk.raw"; + const auto path = std::filesystem::temp_directory_path() / ("anyps5_audio_out_mix_level_pad_spk-" + std::to_string(std::random_device{}()) + ".raw"); std::filesystem::remove(path); SetEnvironment("SDL_DISKAUDIOFILE", path.string()); diff --git a/core/libs/prx/libSceBluetoothHid/CMakeLists.txt b/core/libs/prx/libSceBluetoothHid/CMakeLists.txt new file mode 100644 index 000000000..1f7328c36 --- /dev/null +++ b/core/libs/prx/libSceBluetoothHid/CMakeLists.txt @@ -0,0 +1,15 @@ +add_library(libSceBluetoothHid SHARED EXCLUDE_FROM_ALL Export.cpp) +target_include_directories(libSceBluetoothHid PRIVATE ${LIBS_INCLUDE_DIR}) +configure_windows_unwind(libSceBluetoothHid) + +if(BUILD_TESTING) + add_test_executable(guest_bluetooth_hid_tests ${LIBS_INCLUDE_DIR}/tests/GuestBluetoothHid.cpp) + target_include_directories(guest_bluetooth_hid_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_bluetooth_hid_tests PRIVATE libSceBluetoothHid libc) + configure_windows_unwind(guest_bluetooth_hid_tests) + add_test(NAME guest_bluetooth_hid COMMAND guest_bluetooth_hid_tests) + if(WIN32) + set_tests_properties(guest_bluetooth_hid PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceBluetoothHid/Export.cpp b/core/libs/prx/libSceBluetoothHid/Export.cpp new file mode 100644 index 000000000..46e728e4f --- /dev/null +++ b/core/libs/prx/libSceBluetoothHid/Export.cpp @@ -0,0 +1,52 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +#include + +extern "C" { + +std::int32_t APS5_VABI sceBluetoothHidInit() { + return 0; +} + +std::int32_t APS5_VABI sceBluetoothHidRegisterCallback(void* callback, std::int32_t flags, void* param) { + (void)flags; + (void)param; + if (callback == nullptr) throw std::invalid_argument(std::string(__func__) + ": null callback"); + return 0; +} + +std::int32_t APS5_VABI sceBluetoothHidUnregisterCallback() { + return 0; +} + +std::int32_t APS5_VABI sceBluetoothHidRegisterDevice(std::uint16_t vendorId, std::uint16_t productId) { + (void)vendorId; + (void)productId; + return 0; +} + +std::int32_t APS5_VABI sceBluetoothHidUnregisterDevice(std::uint16_t vendorId, std::uint16_t productId) { + (void)vendorId; + (void)productId; + return 0; +} + +int APS5_VABI sceBluetoothHidGetDeviceName() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceBluetoothHidGetInputReport() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceBluetoothHidSetOutputReport() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libSceCdlgG2P/Export.cpp b/core/libs/prx/libSceCdlgG2P/Export.cpp new file mode 100644 index 000000000..5b46c1222 --- /dev/null +++ b/core/libs/prx/libSceCdlgG2P/Export.cpp @@ -0,0 +1,35 @@ +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +int APS5_VABI sceG2PDialogInitialize(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceG2PDialogTerminate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceG2PDialogOpen(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceG2PDialogGetStatus(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceG2PDialogUpdateStatus(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceG2PDialogGetResult(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} +} diff --git a/core/libs/prx/libSceFont/src/Library.cpp b/core/libs/prx/libSceFont/src/Library.cpp index 90f5cc8b1..e872a3fa6 100644 --- a/core/libs/prx/libSceFont/src/Library.cpp +++ b/core/libs/prx/libSceFont/src/Library.cpp @@ -367,6 +367,29 @@ int APS5_VABI sceFontClearDeviceCache(FontLibrary library) { return rc; } +int APS5_VABI sceFontDettachDeviceCacheBuffer(FontLibrary library, void** buffer, std::uint32_t* size) { + auto* lib = static_cast(library); + if (!lib || lib->magic != LIBRARY_MAGIC) return SCE_FONT_ERROR_INVALID_LIBRARY; + std::uint32_t previous = 0; + if (!AcquireLibraryLock(lib, previous)) return SCE_FONT_ERROR_INVALID_LIBRARY; + std::uint32_t* current = AcquireDeviceCache(lib); + if (!current) { + ReleaseDeviceCache(lib, nullptr); + ReleaseLibraryLock(lib, previous); + return SCE_FONT_ERROR_NOT_ATTACHED_CACHE_BUFFER; + } + if (size) *size = current[0]; + if (buffer) { + *buffer = current; + } else if ((lib->flags & 1u) != 0) { + lib->iface->dealloc(lib->alloc_ctx, current); + } + lib->flags &= ~1u; + ReleaseDeviceCache(lib, nullptr); + ReleaseLibraryLock(lib, previous); + return SCE_FONT_OK; +} + int APS5_VABI sceFontSupportExternalFonts(FontLibrary library, std::uint32_t fontMax, std::uint32_t formats) { return SupportFonts(library, &FontLibNative::external_fonts_ctx, fontMax, (fontMax << 6) | 0x20u, formats); } diff --git a/core/libs/prx/libSceFont/src/Render.cpp b/core/libs/prx/libSceFont/src/Render.cpp index 0da94912c..55d3de834 100644 --- a/core/libs/prx/libSceFont/src/Render.cpp +++ b/core/libs/prx/libSceFont/src/Render.cpp @@ -419,6 +419,13 @@ int APS5_VABI sceFontDeleteGlyph(const FontMemory* memory, FontGlyph* pGlyph) { return SCE_FONT_OK; } +int APS5_VABI sceFontGlyphDefineAttribute(FontGlyph glyph, std::uint32_t attribute, std::uint64_t value) { + (void)attribute; + (void)value; + if (!glyph || glyph->magic != GLYPH_MAGIC) return SCE_FONT_ERROR_INVALID_GLYPH; + return SCE_FONT_OK; +} + int APS5_VABI sceFontGlyphGetGlyphForm(FontGlyph glyph) { if (!glyph || glyph->magic != GLYPH_MAGIC) return SCE_FONT_ERROR_INVALID_GLYPH; return glyph->glyph_form; diff --git a/core/libs/prx/libSceFont/src/Unimplemented.cpp b/core/libs/prx/libSceFont/src/Unimplemented.cpp index e8fcd5a5a..87c3bbcd5 100644 --- a/core/libs/prx/libSceFont/src/Unimplemented.cpp +++ b/core/libs/prx/libSceFont/src/Unimplemented.cpp @@ -62,10 +62,6 @@ int APS5_VABI sceFontDestroyWords() { return 0; } -int APS5_VABI sceFontDettachDeviceCacheBuffer() { - NotImplemented_nid_no_patch(__func__); - return 0; -} int APS5_VABI sceFontGetAttribute() { NotImplemented_nid_no_patch(__func__); @@ -102,11 +98,6 @@ int APS5_VABI sceFontGetTypographicDesign() { return 0; } -int APS5_VABI sceFontGlyphDefineAttribute() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI sceFontGlyphGetAttribute() { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/prx/libSceHttp/Export.cpp b/core/libs/prx/libSceHttp/Export.cpp index 16eca0b4a..76a577451 100644 --- a/core/libs/prx/libSceHttp/Export.cpp +++ b/core/libs/prx/libSceHttp/Export.cpp @@ -4,9 +4,11 @@ #include "prx/libc/include/General.hpp" #include "prx/libSceHttp/src/HttpErrors.hpp" #include +#include #include #include #include +#include // No network is emulated: contexts, templates and requests can be created, but any request // that would touch the network fails with the library's network error. @@ -141,6 +143,20 @@ int APS5_VABI sceHttpSetAuthEnabled(int id, int enable) { return 0; } +int APS5_VABI sceHttpSetAuthInfoCallback(int id, HttpAuthInfoCallback callback, void* userArg) { + (void)id; + (void)callback; + (void)userArg; + return 0; +} + +int APS5_VABI sceHttpSetCookieEnabled(int id, int enable) { + (void)id; + if (static_cast(enable) > 1) return ERROR_INVALID_VALUE; + if (enable != 0) NotImplemented_nid_no_patch(__func__); + return 0; +} + int APS5_VABI sceHttpSetAutoRedirect(int id, int enable) { (void)id; (void)enable; @@ -226,11 +242,10 @@ int APS5_VABI sceHttpUnsetEpoll(int id) { } int APS5_VABI sceHttpWaitRequest(HttpEpollHandle eh, HttpNBEvent* nbev, int maxevents, int timeout) { - (void)eh; - (void)nbev; - (void)maxevents; - (void)timeout; - return ERROR_NETWORK; + if (!eh || !nbev || maxevents <= 0) return ERROR_INVALID_VALUE; + if (timeout < 0) NotImplemented_nid_no_patch(__func__); + std::this_thread::sleep_for(std::chrono::microseconds(timeout)); + return 0; } int APS5_VABI sceHttpCreateRequestWithURL(int conn_id, int method, const char* url, uint64_t content_length) { @@ -370,23 +385,33 @@ int APS5_VABI sceHttpGetLastErrno(int request_id, int* errno_out) { return 0; } -int APS5_VABI sceHttpsGetSslError(void) { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceHttpsGetSslError(int id, int* err_num, uint32_t* detail) { + (void)id; + if (err_num == nullptr || detail == nullptr) { + return ERROR_INVALID_VALUE; + } + *err_num = 0; + *detail = 0; return 0; } -int APS5_VABI sceHttpSetCookieRecvCallback(void) { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceHttpSetCookieRecvCallback(int id, HttpCookieRecvCallback cbfunc, void* user_arg) { + (void)id; + (void)cbfunc; + (void)user_arg; return 0; } -int APS5_VABI sceHttpsSetSslVersion(void) { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceHttpsSetSslVersion(int id, int ssl_version) { + (void)id; + (void)ssl_version; return 0; } -int APS5_VABI sceHttpSetRedirectCallback(void) { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceHttpSetRedirectCallback(int id, HttpRedirectCallback cbfunc, void* user_arg) { + (void)id; + (void)cbfunc; + (void)user_arg; return 0; } diff --git a/core/libs/prx/libSceHttp2/CMakeLists.txt b/core/libs/prx/libSceHttp2/CMakeLists.txt index 45ae68e16..7fbaa4014 100644 --- a/core/libs/prx/libSceHttp2/CMakeLists.txt +++ b/core/libs/prx/libSceHttp2/CMakeLists.txt @@ -1,11 +1,12 @@ add_library(libSceHttp2 SHARED EXCLUDE_FROM_ALL Export.cpp) target_include_directories(libSceHttp2 PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(libSceHttp2 PRIVATE libc libkernel) configure_windows_unwind(libSceHttp2) if(BUILD_TESTING) add_test_executable(guest_http2_tests ${LIBS_INCLUDE_DIR}/tests/GuestHttp2.cpp) target_include_directories(guest_http2_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_http2_tests PRIVATE libSceHttp2 libc) + target_link_libraries(guest_http2_tests PRIVATE libSceHttp2 libkernel libc) configure_windows_unwind(guest_http2_tests) add_test(NAME guest_http2 COMMAND guest_http2_tests) if(WIN32) diff --git a/core/libs/prx/libSceHttp2/Export.cpp b/core/libs/prx/libSceHttp2/Export.cpp index 56f59ebea..215d674e5 100644 --- a/core/libs/prx/libSceHttp2/Export.cpp +++ b/core/libs/prx/libSceHttp2/Export.cpp @@ -2,12 +2,47 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libkernel/Equeue/Equeue.hpp" #include +#include +#include +#include +#include +#include // No network is emulated: contexts, templates and requests can be created, but any request // that would touch the network fails with the library's network error. static constexpr int ERROR_NETWORK = static_cast(0x80436063); static std::atomic g_nextHandle{1}; +static std::mutex g_poolsMutex; +static std::map g_pools; + +struct Http2MemoryPoolStats { + size_t pool_size; + size_t max_inuse_size; + size_t current_inuse_size; + int32_t reserved; +}; + +static std::mutex g_completionsMutex; +static std::unordered_map> g_completions; + +static void CompleteAsync(const char* function, int req_id, const Http2AsyncOption* kqueue_option, const void* option) { + if (kqueue_option == nullptr) NotImplemented_nid_no_patch(function); + if (option != nullptr) NotImplemented_nid_no_patch(function); + { + std::lock_guard lock(g_completionsMutex); + Http2AsyncResult completion{}; + completion.req_id = req_id; + completion.result = ERROR_NETWORK; + g_completions[req_id].push_back(completion); + } + if (EqueueTriggerEvent_nid_postfix(kqueue_option->equeue, static_cast(kqueue_option->user_event_id), EVFILT_USER, kqueue_option->user_data) != 0) { + std::lock_guard lock(g_completionsMutex); + g_completions[req_id].pop_back(); + NotImplemented_nid_no_patch(function); + } +} extern "C" { @@ -36,7 +71,8 @@ int APS5_VABI sceHttp2CreateTemplate(int lib_http2_ctx_id, const char* user_agen } int APS5_VABI sceHttp2DeleteRequest(int req_id) { - (void)req_id; + std::lock_guard lock(g_completionsMutex); + g_completions.erase(req_id); return 0; } @@ -68,9 +104,11 @@ int APS5_VABI sceHttp2GetStatusCode(int req_id, int* status_code) { int APS5_VABI sceHttp2Init(int libnet_mem_id, int libssl_ctx_id, size_t pool_size, int max_concurrent_request) { (void)libnet_mem_id; (void)libssl_ctx_id; - (void)pool_size; (void)max_concurrent_request; - return g_nextHandle.fetch_add(1, std::memory_order_relaxed); + const int id = g_nextHandle.fetch_add(1, std::memory_order_relaxed); + std::lock_guard lock(g_poolsMutex); + g_pools[id] = pool_size; + return id; } int APS5_VABI sceHttp2ReadData(int req_id, void* data, size_t size) { @@ -96,13 +134,11 @@ int APS5_VABI sceHttp2SendRequest(int req_id, const void* post_data, size_t size return ERROR_NETWORK; } -int APS5_VABI sceHttp2SendRequestAsync(int req_id, const void* post_data, size_t size, void* kqueue_option, void* option) { - (void)req_id; +int APS5_VABI sceHttp2SendRequestAsync(int req_id, const void* post_data, size_t size, Http2AsyncOption* kqueue_option, void* option) { (void)post_data; (void)size; - (void)kqueue_option; - (void)option; - return ERROR_NETWORK; + CompleteAsync(__func__, req_id, kqueue_option, option); + return 0; } int APS5_VABI sceHttp2SetAuthEnabled(int id, int is_enable) { @@ -198,16 +234,20 @@ int APS5_VABI sceHttp2SetMinSslVersion(int id, uint32_t ssl_version) { } int APS5_VABI sceHttp2Term(int lib_http2_ctx_id) { - (void)lib_http2_ctx_id; + std::lock_guard lock(g_poolsMutex); + g_pools.erase(lib_http2_ctx_id); return 0; } int APS5_VABI sceHttp2WaitAsync(int req_id, Http2AsyncResult* result, uint32_t* timeout, void* option) { - (void)req_id; - (void)result; (void)timeout; - (void)option; - return ERROR_NETWORK; + if (result == nullptr || option != nullptr) NotImplemented_nid_no_patch(__func__); + std::lock_guard lock(g_completionsMutex); + const auto pending = g_completions.find(req_id); + if (pending == g_completions.end() || pending->second.empty()) NotImplemented_nid_no_patch("sceHttp2WaitAsync: waiting for an operation that has not completed"); + *result = pending->second.front(); + pending->second.pop_front(); + return 0; } int APS5_VABI sceHttp2AbortRequest(int req_id) { @@ -215,8 +255,12 @@ int APS5_VABI sceHttp2AbortRequest(int req_id) { return 0; } -int APS5_VABI sceHttp2GetMemoryPoolStats() { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceHttp2GetMemoryPoolStats(int lib_http2_ctx_id, Http2MemoryPoolStats* stats) { + if (stats == nullptr) APS5_INVALID_ARG_EX; + std::lock_guard lock(g_poolsMutex); + const auto pool = g_pools.find(lib_http2_ctx_id); + if (pool == g_pools.end()) throw std::invalid_argument("sceHttp2GetMemoryPoolStats: unknown context"); + *stats = {pool->second, 0, 0, 0}; return 0; } @@ -241,4 +285,31 @@ int APS5_VABI sceHttp2SetRequestNoContentLength(int id) { return 0; } + +int APS5_VABI sceHttp2SetResolveRetry(int id, int32_t retry) { + (void)id; + (void)retry; + return 0; +} + +int APS5_VABI sceHttp2WebSocketCreateRequest() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceHttp2WebSocketCloseAsync() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceHttp2WebSocketSendTextMessageAsync() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceHttp2WebSocketSendDataMessageAsync() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + } diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 0d7e33f95..70675ee05 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -500,6 +500,10 @@ int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5Va return 0; } +int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(void* self, NullAccessCallback callback, void* context) { + return _ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(self, callback, context); +} + void APS5_VABI _ZN3sce4Json12MemAllocatorC2Ev(void* self) { *static_cast(self) = &JsonMemAllocatorVtable[2]; } @@ -569,6 +573,7 @@ void APS5_VABI _ZN3sce4Json6Object8iteratorD1Ev(ObjectIterator* self) { (void)se ObjectIterator* APS5_VABI _ZN3sce4Json6Object8iteratorppEv(ObjectIterator* self) { ++self->it; return self; } Pair* APS5_VABI _ZNK3sce4Json6Object8iteratordeEv(const ObjectIterator* self) { return &*self->it; } bool APS5_VABI _ZNK3sce4Json6Object8iteratorneERKS2_(const ObjectIterator* self, const ObjectIterator* other) { return self->it != other->it; } +bool APS5_VABI _ZNK3sce4Json6Object5emptyEv(const Object* self) { return self->items->empty(); } std::size_t APS5_VABI _ZNK3sce4Json6Object4sizeEv(const Object* self) { return self->items->size(); } void APS5_VABI _ZN3sce4Json5ValueC1Ev(Value* self) { Construct(*self); } @@ -771,9 +776,8 @@ int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value* out, const char* } -APS5_EXPORT("6i18OJSvFWk", sceJson2Unknown00); -int APS5_VABI sceJson2Unknown00(void) { - NotImplemented_nid_no_patch("6i18OJSvFWk"); +int APS5_VABI _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(void) { + NotImplemented_nid_no_patch(__func__); return 0; } @@ -803,9 +807,9 @@ int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_14InitParameter2E(void return 0; } -int APS5_VABI _ZN3sce4Json5Value5clearEv(void) { - NotImplemented_nid_no_patch(__func__); - return 0; +void APS5_VABI _ZN3sce4Json5Value5clearEv(Value* self) { + if (self == nullptr) return; + Clear(NodeOf(*self)); } int APS5_VABI _ZN3sce4Json12MemAllocator11notifyErrorEimPv(void) { diff --git a/core/libs/prx/libSceLibcInternal/CMakeLists.txt b/core/libs/prx/libSceLibcInternal/CMakeLists.txt index 6443b72c5..2c0060756 100644 --- a/core/libs/prx/libSceLibcInternal/CMakeLists.txt +++ b/core/libs/prx/libSceLibcInternal/CMakeLists.txt @@ -17,7 +17,7 @@ if(BUILD_TESTING) if(NOT WIN32) add_test_executable(guest_force_tls_destructor_tests ${LIBS_INCLUDE_DIR}/tests/GuestForceTlsDestructor.cpp) target_include_directories(guest_force_tls_destructor_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_force_tls_destructor_tests PRIVATE libSceLibcInternal libkernel libc) + target_link_libraries(guest_force_tls_destructor_tests PRIVATE libSceLibcInternal libkernel libc ${CMAKE_DL_LIBS}) add_test(NAME guest_force_tls_destructor COMMAND guest_force_tls_destructor_tests) set_tests_properties(guest_force_tls_destructor PROPERTIES TIMEOUT 20) endif() diff --git a/core/libs/prx/libSceMbus/CMakeLists.txt b/core/libs/prx/libSceMbus/CMakeLists.txt new file mode 100644 index 000000000..7901f908f --- /dev/null +++ b/core/libs/prx/libSceMbus/CMakeLists.txt @@ -0,0 +1,15 @@ +add_library(libSceMbus SHARED EXCLUDE_FROM_ALL Export.cpp) +target_include_directories(libSceMbus PRIVATE ${LIBS_INCLUDE_DIR}) +configure_windows_unwind(libSceMbus) + +if(BUILD_TESTING) + add_test_executable(guest_mbus_tests ${LIBS_INCLUDE_DIR}/tests/GuestMbus.cpp) + target_include_directories(guest_mbus_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_mbus_tests PRIVATE libSceMbus libc) + configure_windows_unwind(guest_mbus_tests) + add_test(NAME guest_mbus COMMAND guest_mbus_tests) + if(WIN32) + set_tests_properties(guest_mbus PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceMbus/Export.cpp b/core/libs/prx/libSceMbus/Export.cpp new file mode 100644 index 000000000..9bfe48812 --- /dev/null +++ b/core/libs/prx/libSceMbus/Export.cpp @@ -0,0 +1,37 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +#include + +extern "C" { + +std::int32_t APS5_VABI sceDeviceServiceInitialize(std::int32_t mode, std::int32_t flags) { + (void)mode; + (void)flags; + return 0; +} + +std::int32_t APS5_VABI sceDeviceServiceTerminate() { + return 0; +} + +std::int32_t APS5_VABI sceDeviceServiceGetEventState(std::int32_t deviceType) { + (void)deviceType; + return 0; +} + +std::int32_t APS5_VABI sceDeviceServiceQueryDeviceInfo_(std::int32_t deviceClass, std::int32_t arg1, std::int32_t arg2, void* infos, std::int32_t maxInfos, std::int32_t* count, std::int32_t* reserved, std::uint64_t infoSize) { + (void)deviceClass; + (void)arg1; + (void)arg2; + (void)infos; + (void)reserved; + (void)infoSize; + if (count == nullptr || maxInfos < 0) throw std::invalid_argument(std::string(__func__) + ": invalid argument"); + *count = 0; + return 0; +} + +} diff --git a/core/libs/prx/libSceNet/CMakeLists.txt b/core/libs/prx/libSceNet/CMakeLists.txt index 0e2e8e419..2516da1e1 100644 --- a/core/libs/prx/libSceNet/CMakeLists.txt +++ b/core/libs/prx/libSceNet/CMakeLists.txt @@ -5,7 +5,7 @@ configure_windows_unwind(libSceNet) if(BUILD_TESTING) add_test_executable(guest_sce_net_tests ${LIBS_INCLUDE_DIR}/tests/GuestSceNet.cpp) target_include_directories(guest_sce_net_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_sce_net_tests PRIVATE libSceNet libSceNetCtl libc) + target_link_libraries(guest_sce_net_tests PRIVATE libSceNet libSceNetCtl libkernel libc) configure_windows_unwind(guest_sce_net_tests) add_test(NAME guest_sce_net COMMAND guest_sce_net_tests) set_tests_properties(guest_sce_net PROPERTIES TIMEOUT 20) diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index 0b279067d..05a06ea00 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -31,6 +31,7 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libkernel/Socket/include/SocketPoll.hpp" // Guest socket descriptors retain PS5 semantics while their transport is backed by host sockets. @@ -38,20 +39,28 @@ namespace { // FreeBSD errno numbers as reported through sceNetErrnoLoc (SCE_NET_ERROR_* is 0x80410100 + errno). constexpr int NET_ENOENT = 2; constexpr int NET_EBADF = 9; +constexpr int NET_EACCES = 13; constexpr int NET_EFAULT = 14; constexpr int NET_EINVAL = 22; constexpr int NET_ENOSPC = 28; constexpr int NET_EAGAIN = 35; +constexpr int NET_EINPROGRESS = 36; +constexpr int NET_EALREADY = 37; constexpr int NET_ENOTSOCK = 38; constexpr int NET_EOPNOTSUPP = 45; constexpr int NET_EPROTONOSUPPORT = 43; constexpr int NET_EAFNOSUPPORT = 47; constexpr int NET_EADDRINUSE = 48; +constexpr int NET_EADDRNOTAVAIL = 49; constexpr int NET_ENETUNREACH = 51; constexpr int NET_ECONNABORTED = 53; +constexpr int NET_ENOBUFS = 55; +constexpr int NET_EISCONN = 56; +constexpr int NET_ENOTCONN = 57; constexpr int NET_EMSGSIZE = 40; constexpr int NET_ETIMEDOUT = 60; constexpr int NET_ECONNREFUSED = 61; +constexpr int NET_EHOSTUNREACH = 65; constexpr int NET_ERROR_BASE = static_cast(0x80410100u); constexpr int NET_ERROR_RESOLVER_ENODNS = static_cast(0x804101E1u); @@ -84,11 +93,18 @@ void close_socket(NativeSocket socket) { closesocket(socket); } int native_error() { switch (WSAGetLastError()) { case WSAEWOULDBLOCK: return NET_EAGAIN; + case WSAEINPROGRESS: return NET_EINPROGRESS; + case WSAEALREADY: return NET_EALREADY; case WSAEADDRINUSE: return NET_EADDRINUSE; + case WSAEADDRNOTAVAIL: return NET_EADDRNOTAVAIL; case WSAEAFNOSUPPORT: return NET_EAFNOSUPPORT; case WSAENETUNREACH: return NET_ENETUNREACH; - case WSAEHOSTUNREACH: return NET_ENETUNREACH; + case WSAEHOSTUNREACH: return NET_EHOSTUNREACH; case WSAECONNABORTED: return NET_ECONNABORTED; + case WSAENOBUFS: return NET_ENOBUFS; + case WSAEISCONN: return NET_EISCONN; + case WSAENOTCONN: return NET_ENOTCONN; + case WSAEACCES: return NET_EACCES; case WSAECONNRESET: return 54; case WSAETIMEDOUT: return NET_ETIMEDOUT; case WSAECONNREFUSED: return NET_ECONNREFUSED; @@ -112,11 +128,18 @@ int native_error() { #if EWOULDBLOCK != EAGAIN case EWOULDBLOCK: return NET_EAGAIN; #endif + case EINPROGRESS: return NET_EINPROGRESS; + case EALREADY: return NET_EALREADY; case EADDRINUSE: return NET_EADDRINUSE; + case EADDRNOTAVAIL: return NET_EADDRNOTAVAIL; case EAFNOSUPPORT: return NET_EAFNOSUPPORT; case ENETUNREACH: return NET_ENETUNREACH; - case EHOSTUNREACH: return NET_ENETUNREACH; + case EHOSTUNREACH: return NET_EHOSTUNREACH; case ECONNABORTED: return NET_ECONNABORTED; + case ENOBUFS: return NET_ENOBUFS; + case EISCONN: return NET_EISCONN; + case ENOTCONN: return NET_ENOTCONN; + case EACCES: return NET_EACCES; case ECONNRESET: return 54; case ETIMEDOUT: return NET_ETIMEDOUT; case ECONNREFUSED: return NET_ECONNREFUSED; @@ -207,7 +230,13 @@ std::map g_socks; std::set g_epolls; std::map> g_epoll_socks; std::set g_epoll_aborted; -std::set g_pools; +std::map g_pools; + +struct NetMemoryPoolStats { + std::size_t pool_size; + std::size_t max_inuse_size; + std::size_t current_inuse_size; +}; std::map g_resolvers; int g_next_sock = 32; int g_next_epoll = 0x4000; @@ -287,10 +316,71 @@ std::int64_t message_length(const NetMsghdr* message) { extern "C" { +extern const std::uint32_t sce_net_in6addr_any[4] = {}; + int* APS5_VABI sceNetErrnoLoc(void) { return errno_slot(); } +int PollSockets(KernelSocketPoll::Entry* entries, int count, int timeoutMilliseconds) { + std::vector> natives(static_cast(count)); + { + std::lock_guard lk(g_mutex); + for (int index = 0; index < count; ++index) { + const auto socket = g_socks.find(entries[index].descriptor); + if (socket != g_socks.end()) natives[static_cast(index)] = socket->second.native; + } + } +#ifdef _WIN32 + std::vector descriptors; + constexpr short NativeReadable = POLLRDNORM; + constexpr short NativeUrgent = POLLRDBAND; + constexpr short NativeWritable = POLLWRNORM; +#else + std::vector descriptors; + constexpr short NativeReadable = POLLIN; + constexpr short NativeUrgent = POLLPRI; + constexpr short NativeWritable = POLLOUT; +#endif + std::vector owners; + for (int index = 0; index < count; ++index) { + auto& entry = entries[index]; + if (!natives[static_cast(index)]) { + entry.revents = KernelSocketPoll::Unknown; + continue; + } + decltype(descriptors)::value_type descriptor{}; + descriptor.fd = natives[static_cast(index)]->value; + if (entry.events & KernelSocketPoll::Readable) descriptor.events |= NativeReadable; + if (entry.events & KernelSocketPoll::Urgent) descriptor.events |= NativeUrgent; + if (entry.events & KernelSocketPoll::Writable) descriptor.events |= NativeWritable; + descriptors.push_back(descriptor); + owners.push_back(index); + } + if (descriptors.empty()) return 0; +#ifdef _WIN32 + const int result = WSAPoll(descriptors.data(), static_cast(descriptors.size()), timeoutMilliseconds); +#else + const int result = ::poll(descriptors.data(), descriptors.size(), timeoutMilliseconds); +#endif + if (result < 0) return -native_error(); + int ready = 0; + for (std::size_t index = 0; index < descriptors.size(); ++index) { + const short native = descriptors[index].revents; + short revents = 0; + if (native & NativeReadable) revents |= KernelSocketPoll::Readable; + if (native & NativeUrgent) revents |= KernelSocketPoll::Urgent; + if (native & NativeWritable) revents |= KernelSocketPoll::Writable; + if (native & (POLLERR | POLLNVAL)) revents |= KernelSocketPoll::Error; + if (native & POLLHUP) revents |= KernelSocketPoll::HangUp; + entries[owners[index]].revents = revents; + if (revents != 0) ++ready; + } + return ready; +} + +const bool g_socketPollerRegistered = (KernelSetSocketPoller_nid_no_patch(&PollSockets), true); + int APS5_VABI sceNetInit_nid_postfix(void) { std::lock_guard lk(g_mutex); g_net_inited = true; @@ -311,7 +401,7 @@ int APS5_VABI sceNetPoolCreate(const char* name, int size, int flags) { } std::lock_guard lk(g_mutex); const int id = g_next_pool++; - g_pools.insert(id); + g_pools.emplace(id, static_cast(size)); return id; } @@ -478,8 +568,11 @@ int APS5_VABI sceNetConnect(int s, const void* addr, uint32_t addrlen) { NativeLength native_length = 0; if (!guest_to_native_address(addr, addrlen, native, native_length)) return fail(NET_EINVAL); if (native.ss_family != (socket.family == NET_AF_INET ? AF_INET : AF_INET6)) return fail(NET_EAFNOSUPPORT); - return ::connect(socket.native->value, reinterpret_cast(&native), native_length) == 0 - ? 0 : fail(native_error()); + if (::connect(socket.native->value, reinterpret_cast(&native), native_length) == 0) return 0; +#ifdef _WIN32 + if (WSAGetLastError() == WSAEWOULDBLOCK) return fail(NET_EINPROGRESS); +#endif + return fail(native_error()); } int64_t APS5_VABI sceNetRecv(int s, void* buf, size_t len, int flags) { @@ -761,8 +854,16 @@ int APS5_VABI sceNetEpollCreate(const char* name, int flags) { return id; } -int APS5_VABI sceNetGetMemoryPoolStats() { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceNetGetMemoryPoolStats(int memid, NetMemoryPoolStats* stats) { + if (stats == nullptr) { + return fail(NET_EINVAL); + } + std::lock_guard lk(g_mutex); + const auto pool = g_pools.find(memid); + if (pool == g_pools.end()) { + return fail(NET_EBADF); + } + *stats = {pool->second, 0, 0}; return 0; } @@ -987,11 +1088,7 @@ int APS5_VABI sceNetResolverDestroy(int rid) { return g_resolvers.erase(rid) != 0 ? 0 : fail(NET_EBADF); } -int APS5_VABI sceNetResolverStartNtoa(int rid, const char* hostname, void* addr, int timeout, int retry, int flags) { - (void)timeout; - (void)retry; - (void)flags; - if (!hostname || !addr) return fail(NET_EINVAL); +int lookup_ipv4(int rid, const char* hostname, const char* func, std::vector& addresses) { { std::lock_guard lk(g_mutex); if (g_resolvers.count(rid) == 0) return fail(NET_EBADF); @@ -1003,17 +1100,64 @@ int APS5_VABI sceNetResolverStartNtoa(int rid, const char* hostname, void* addr, const int result = ::getaddrinfo(hostname, nullptr, &hints, &results); if (result != 0) { *errno_slot() = result == EAI_AGAIN ? NET_ETIMEDOUT : NET_ENOENT; - log_soft(__func__, "host DNS lookup failed"); + log_soft(func, "host DNS lookup failed"); set_resolver_error(rid, NET_ERROR_RESOLVER_ENODNS); return NET_ERROR_RESOLVER_ENODNS; } - const auto* address = reinterpret_cast(results->ai_addr); - std::memcpy(addr, &address->sin_addr, sizeof(address->sin_addr)); + for (const addrinfo* entry = results; entry != nullptr; entry = entry->ai_next) { + const auto address = reinterpret_cast(entry->ai_addr)->sin_addr.s_addr; + if (std::find(addresses.begin(), addresses.end(), address) == addresses.end()) addresses.push_back(address); + } ::freeaddrinfo(results); set_resolver_error(rid, 0); return 0; } +int APS5_VABI sceNetResolverStartNtoa(int rid, const char* hostname, void* addr, int timeout, int retry, int flags) { + (void)timeout; + (void)retry; + (void)flags; + if (!hostname || !addr) return fail(NET_EINVAL); + std::vector addresses; + if (const int error = lookup_ipv4(rid, hostname, __func__, addresses)) return error; + std::memcpy(addr, addresses.data(), sizeof(addresses[0])); + return 0; +} + +struct NetResolverRecord { + std::uint32_t address; + std::uint8_t address6[12]; + std::int32_t family; + std::int32_t reserved[3]; +}; + +struct NetResolverInfo { + NetResolverRecord records[10]; + std::int32_t count; + std::int32_t count4; + std::int32_t reserved[14]; +}; +static_assert(sizeof(NetResolverRecord) == 32 && offsetof(NetResolverRecord, family) == 16); +static_assert(sizeof(NetResolverInfo) == 384 && offsetof(NetResolverInfo, count) == 320 && offsetof(NetResolverInfo, count4) == 324); + +int APS5_VABI sceNetResolverStartNtoaMultipleRecordsEx(int rid, const char* hostname, NetResolverInfo* info, int timeout, + int retry, int flags) { + (void)timeout; + (void)retry; + if (flags != 0) throw std::runtime_error("sceNetResolverStartNtoaMultipleRecordsEx: flags " + std::to_string(flags) + " are not supported"); + if (!hostname || !info) return fail(NET_EINVAL); + std::vector addresses; + if (const int error = lookup_ipv4(rid, hostname, __func__, addresses)) return error; + *info = {}; + info->count = static_cast(std::min(addresses.size(), std::size(info->records))); + info->count4 = info->count; + for (std::int32_t index = 0; index < info->count; ++index) { + info->records[index].address = addresses[index]; + info->records[index].family = NET_AF_INET; + } + return 0; +} + int APS5_VABI sceNetResolverStartAton(int rid, const void* addr, char* hostname, int len, int timeout, int retry, int flags) { (void)timeout; (void)retry; @@ -1058,4 +1202,7 @@ int APS5_VABI sceNetResolverStartNtoaMultipleRecords() { return 0; } +extern const std::uint8_t in6addr_any_nid_postfix[16] = {}; +extern const std::uint8_t in6addr_loopback_nid_postfix[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}; + } diff --git a/core/libs/prx/libSceNgs2.native/CMakeLists.txt b/core/libs/prx/libSceNgs2.native/CMakeLists.txt index 0536d7a2d..8d100ce35 100644 --- a/core/libs/prx/libSceNgs2.native/CMakeLists.txt +++ b/core/libs/prx/libSceNgs2.native/CMakeLists.txt @@ -2,7 +2,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/Ngs2Library.cmake) add_sce_ngs2_library(libSceNgs2.native) if(BUILD_TESTING) - foreach(ngs2Test IN ITEMS Render Atrac9 Filter Custom Parse) + foreach(ngs2Test IN ITEMS Render Atrac9 Filter Custom Parse Stream UserFx) string(TOLOWER ${ngs2Test} testName) add_test_executable(ngs2_${testName}_tests ${CMAKE_CURRENT_SOURCE_DIR}/tests/Ngs2${ngs2Test}.cpp) target_include_directories(ngs2_${testName}_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/LibAtrac9/C/src) diff --git a/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp b/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp index a6a92c819..3496f022f 100644 --- a/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp +++ b/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp @@ -49,6 +49,7 @@ static constexpr std::uint32_t SCE_NGS2_SAMPLER_VOICE_PARAM_EXIT_LOOP = 0x100000 static constexpr std::uint32_t SCE_NGS2_SAMPLER_VOICE_PARAM_PITCH = 0x10000005; static constexpr std::uint32_t SCE_NGS2_SAMPLER_VOICE_PARAM_FILTER = 0x1000000a; static constexpr std::uint32_t SCE_NGS2_SUBMIXER_VOICE_PARAM_SETUP = 0x20000000; +static constexpr std::uint32_t SCE_NGS2_SUBMIXER_VOICE_PARAM_USER_FX = 0x20000004; static constexpr std::uint32_t SCE_NGS2_MASTERING_VOICE_PARAM_SETUP = 0x30000000; static constexpr std::uint32_t SCE_NGS2_MASTERING_VOICE_PARAM_GAIN = 0x30000004; static constexpr std::uint32_t SCE_NGS2_MASTERING_VOICE_PARAM_OUTPUT = 0x30000005; @@ -60,6 +61,7 @@ static constexpr std::uint32_t SCE_NGS2_CUSTOM_MAX_PORTS = 16; static constexpr std::uint32_t SCE_NGS2_CUSTOM_MODULE_ID_USER_FX2 = 0x1f; static constexpr std::uint32_t SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE = 1; +static constexpr std::uint32_t SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND = 2; static constexpr std::uint32_t SCE_NGS2_WAVEFORM_BLOCKS_FLAG_RESET = 4; static constexpr std::uint32_t SCE_NGS2_VOICE_CALLBACK_FLAG_BLOCK_END = 1; @@ -172,6 +174,20 @@ struct Ngs2UserFx2SetupContext { }; static_assert(sizeof(Ngs2UserFx2SetupContext) == 72); +struct Ngs2UserFxProcessContext { + float** channel_data; + std::uintptr_t user_data0; + std::uintptr_t user_data1; + std::uintptr_t user_data2; + std::uint32_t flags; + std::uint32_t num_channels; + std::uint32_t num_grain_samples; + std::uint32_t sample_rate; +}; +static_assert(sizeof(Ngs2UserFxProcessContext) == 48); + +using Ngs2UserFxProcessHandler = std::int32_t (APS5_VABI *)(Ngs2UserFxProcessContext*); + using Ngs2UserFx2CleanupContext = Ngs2UserFx2SetupContext; struct Ngs2UserFx2ControlContext { @@ -405,6 +421,15 @@ struct Ngs2SubmixerVoiceSetupParam { }; static_assert(sizeof(Ngs2SubmixerVoiceSetupParam) == 16); +struct Ngs2SubmixerVoiceUserFxParam { + Ngs2VoiceParamHeader header; + Ngs2UserFxProcessHandler handler; + std::uintptr_t user_data0; + std::uintptr_t user_data1; + std::uintptr_t user_data2; +}; +static_assert(sizeof(Ngs2SubmixerVoiceUserFxParam) == 40); + struct Ngs2CustomSubmixerVoiceSetupParam { Ngs2VoiceParamHeader header; std::uint32_t num_input_channels; diff --git a/core/libs/prx/libSceNgs2.native/src/Atrac9.cpp b/core/libs/prx/libSceNgs2.native/src/Atrac9.cpp index af2b13da2..6403f9c10 100644 --- a/core/libs/prx/libSceNgs2.native/src/Atrac9.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Atrac9.cpp @@ -127,16 +127,19 @@ struct RiffChunks { static int ReadChunks(const std::uint8_t* bytes, std::size_t size, RiffChunks& chunks) { if (size < 12) return SCE_NGS2_ERROR_INVALID_WAVEFORM_DATA; if (std::memcmp(bytes, "RIFF", 4) != 0 || std::memcmp(bytes + 8, "WAVE", 4) != 0) return SCE_NGS2_ERROR_UNKNOWN_WAVEFORM_FORMAT; - const std::size_t end = std::min(size, 8 + static_cast(ReadLe32(bytes + 4))); + const std::size_t riffEnd = 8 + static_cast(ReadLe32(bytes + 4)); + if (riffEnd < 12) return SCE_NGS2_ERROR_INVALID_WAVEFORM_DATA; + const std::size_t end = std::min(size, riffEnd); for (std::size_t offset = 12; offset + 8 <= end;) { const auto* chunk = bytes + offset; const std::size_t payload = offset + 8; const std::size_t chunkSize = ReadLe32(chunk + 4); + if (chunkSize > riffEnd - payload) return SCE_NGS2_ERROR_INVALID_WAVEFORM_DATA; const bool data = std::memcmp(chunk, "data", 4) == 0; if (data) { chunks.dataOffset = payload; chunks.dataSize = chunkSize; - break; + if (chunkSize > end - payload) break; } if (chunkSize > end - payload) return SCE_NGS2_ERROR_INVALID_WAVEFORM_DATA; if (std::memcmp(chunk, "fmt ", 4) == 0) { @@ -155,6 +158,47 @@ static int ReadChunks(const std::uint8_t* bytes, std::size_t size, RiffChunks& c return SCE_NGS2_OK; } +static bool AddAtrac9Range(Ngs2WaveformInfo& info, std::uint64_t begin, std::uint32_t samples, std::uint32_t repeats) { + if (samples == 0) return true; + const auto preroll = std::min(begin, info.num_audio_unit_samples); + const auto firstFrame = (begin - preroll) / info.num_audio_frame_samples; + const auto skip = begin - firstFrame * info.num_audio_frame_samples; + const auto frames = (skip + samples + info.num_audio_frame_samples - 1) / info.num_audio_frame_samples; + const auto offset = firstFrame * info.audio_frame_size; + const auto bytes = frames * info.audio_frame_size; + if (offset > info.data_size || bytes > info.data_size - offset) return false; + auto& block = info.block[info.num_blocks++]; + block = {info.data_offset + offset, bytes, repeats, static_cast(skip), samples, 0, 0}; + return true; +} + +static int ParseAtrac9Loop(const RiffChunks& chunks, Ngs2WaveformInfo& info) { + if (chunks.sampler == nullptr) return SCE_NGS2_OK; + if (chunks.samplerSize < 36) return SCE_NGS2_ERROR_INVALID_WAVEFORM_DATA; + const auto count = ReadLe32(chunks.sampler + 28); + if (count > (chunks.samplerSize - 36) / 24) return SCE_NGS2_ERROR_INVALID_WAVEFORM_DATA; + if (count == 0) return SCE_NGS2_OK; + if (count != 1) throw std::runtime_error("NGS2: multiple waveform loops are not implemented"); + const auto* loop = chunks.sampler + 36; + if (ReadLe32(loop + 4) != 0 || ReadLe32(loop + 16) != 0) + throw std::runtime_error("NGS2: non-forward or fractional waveform loops are not implemented"); + const std::uint64_t begin = ReadLe32(loop + 8); + const std::uint64_t end = static_cast(ReadLe32(loop + 12)) + 1; + const auto skip = info.block[0].num_skip_samples; + const std::uint64_t waveformEnd = static_cast(skip) + info.num_samples; + if (begin < skip || begin >= end || end > waveformEnd) return SCE_NGS2_ERROR_INVALID_WAVEFORM_DATA; + const auto plays = ReadLe32(loop + 20); + info.loop_begin_position = static_cast(begin - skip); + info.loop_end_position = static_cast(end - skip); + info.num_blocks = 0; + info.block[0] = {}; + if (!AddAtrac9Range(info, skip, info.loop_begin_position, 0) || + !AddAtrac9Range(info, begin, static_cast(end - begin), plays == 0 ? UINT32_MAX : plays - 1) || + !AddAtrac9Range(info, end, static_cast(waveformEnd - end), 0)) + return SCE_NGS2_ERROR_INVALID_WAVEFORM_DATA; + return SCE_NGS2_OK; +} + static int ParseAtrac9(const RiffChunks& chunks, Ngs2WaveformInfo& info) { const auto* format = chunks.format; if (chunks.formatSize < FMT_ATRAC9_SIZE || chunks.fact == nullptr || chunks.factSize < FACT_ATRAC9_SIZE) return SCE_NGS2_ERROR_INVALID_WAVEFORM_FORMAT; @@ -165,7 +209,6 @@ static int ParseAtrac9(const RiffChunks& chunks, Ngs2WaveformInfo& info) { ReadLe16(format + 12) != codec.superframeSize || ReadLe16(format + 18) != static_cast(codec.frameSamples * codec.framesInSuperframe)) { return SCE_NGS2_ERROR_INVALID_WAVEFORM_FORMAT; } - if (chunks.sampler != nullptr && chunks.samplerSize >= 32 && ReadLe32(chunks.sampler + 28) != 0) throw std::runtime_error("NGS2: parsing looped waveforms is not implemented"); info.format = {SCE_NGS2_WAVEFORM_TYPE_ATRAC9, static_cast(codec.channels), static_cast(codec.samplingRate), static_cast(config[0]) << 24 | static_cast(config[1]) << 16 | static_cast(config[2]) << 8 | config[3], 0, 0}; info.data_offset = static_cast(chunks.dataOffset); @@ -182,7 +225,7 @@ static int ParseAtrac9(const RiffChunks& chunks, Ngs2WaveformInfo& info) { info.block[0].data_size = chunks.dataSize; info.block[0].num_skip_samples = ReadLe32(chunks.fact + 8); info.block[0].num_samples = info.num_samples; - return SCE_NGS2_OK; + return ParseAtrac9Loop(chunks, info); } static int ParsePcm16(const RiffChunks& chunks, std::size_t size, Ngs2WaveformInfo& info) { @@ -224,6 +267,7 @@ int APS5_VABI sceNgs2ParseWaveformData(const void* data, size_t data_size, Ngs2W RiffChunks chunks; const int result = ReadChunks(static_cast(data), data_size, chunks); if (result != SCE_NGS2_OK) return result; + if (chunks.formatSize < 2) return SCE_NGS2_ERROR_INVALID_WAVEFORM_FORMAT; const std::uint16_t tag = ReadLe16(chunks.format); if (tag == WAVE_FORMAT_EXTENSIBLE && chunks.formatSize >= FMT_GUID_OFFSET + sizeof(ATRAC9_GUID) && std::memcmp(chunks.format + FMT_GUID_OFFSET, ATRAC9_GUID, sizeof(ATRAC9_GUID)) == 0) { diff --git a/core/libs/prx/libSceNgs2.native/src/Custom.cpp b/core/libs/prx/libSceNgs2.native/src/Custom.cpp index d97ae633c..c77dc5205 100644 --- a/core/libs/prx/libSceNgs2.native/src/Custom.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Custom.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -109,3 +110,15 @@ void Ngs2ProcessUserFx(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t samp fx.flags = 0; } } + +void Ngs2ProcessLegacyUserFx(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate) { + if (voice.userFxHandler == nullptr) return; + std::array channels{}; + for (std::uint32_t c = 0; c < voice.channels; ++c) channels[c] = voice.samples.data() + static_cast(c) * grain; + Ngs2UserFxProcessContext context{channels.data(), voice.userFxData[0], voice.userFxData[1], voice.userFxData[2], + voice.userFxFlags, voice.channels, grain, sampleRate}; + const int result = voice.userFxHandler(&context); + if (result != SCE_NGS2_OK) throw std::runtime_error("NGS2: the UserFx process handler failed with " + std::to_string(result)); + voice.userFxFlags = 0; + voice.hasSamples = std::any_of(voice.samples.begin(), voice.samples.end(), [](float sample) { return sample != 0.0f; }); +} diff --git a/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp b/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp index 8b3965619..ab1486207 100644 --- a/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp +++ b/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp @@ -2,6 +2,7 @@ #define CORE_LIBS_PRX_LIBSCENGS2_SRC_NGS2INTERNAL_HPP #include +#include #include #include #include @@ -25,12 +26,21 @@ enum class Ngs2PlayState : std::uint32_t { Stopped = SCE_NGS2_VOICE_STATE_FLAG_INUSE | SCE_NGS2_VOICE_STATE_FLAG_PLAYING | SCE_NGS2_VOICE_STATE_FLAG_STOPPED, }; +struct Ngs2Piece { + const std::uint8_t* data; + std::uint64_t firstFrame; + std::uint64_t frames; +}; + struct Ngs2Block { const std::uint8_t* data; Ngs2WaveformBlock info; std::uint32_t cursor = 0; std::uint32_t numRepeated = 0; std::size_t dataCursor = 0; + bool streaming = false; + std::uint64_t availableFrames = 0; + std::vector pieces; }; struct Ngs2Atrac9DecoderDeleter { @@ -105,6 +115,9 @@ struct Ngs2Voice { float fbwLevel = 1.0f; float lfeLevel = 1.0f; std::vector userFx; + Ngs2UserFxProcessHandler userFxHandler = nullptr; + std::array userFxData{}; + std::uint32_t userFxFlags = 0; std::vector samples; bool rendering = false; bool rendered = false; @@ -143,6 +156,7 @@ struct Ngs2System { }; std::string Ngs2Hex(std::uint32_t value); +const std::uint8_t* Ngs2StreamEnd(const Ngs2Voice& voice, const Ngs2Block& block); std::recursive_mutex& Ngs2Mutex(); Ngs2System* Ngs2FindSystem(Ngs2Handle handle); @@ -166,6 +180,7 @@ void Ngs2SetupUserFx(Ngs2Rack& rack, const Ngs2CustomRackOption& option); void Ngs2CleanupUserFx(Ngs2Rack& rack); void Ngs2ApplyCustomParam(Ngs2Voice& voice, const Ngs2VoiceParamHeader& param); void Ngs2ProcessUserFx(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate); +void Ngs2ProcessLegacyUserFx(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate); void Ngs2RenderSystem(Ngs2System& system, const Ngs2RenderBufferInfo* bufferInfo, std::uint32_t numBufferInfo); #endif diff --git a/core/libs/prx/libSceNgs2.native/src/Render.cpp b/core/libs/prx/libSceNgs2.native/src/Render.cpp index 1e81d0d6b..b0133ed2f 100644 --- a/core/libs/prx/libSceNgs2.native/src/Render.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Render.cpp @@ -14,10 +14,21 @@ static constexpr double PHASE_ONE = 4294967296.0; static constexpr std::uint64_t PHASE_FRACTION = 0xffffffff; static const std::uint8_t* FrameAt(const Ngs2Voice& voice, const Ngs2Block& block, std::uint32_t frame) { - return block.data + (static_cast(block.info.num_skip_samples) + frame) * voice.channels * sizeof(std::int16_t); + const std::size_t frameBytes = voice.channels * sizeof(std::int16_t); + if (block.streaming) { + for (const auto& piece : block.pieces) { + if (frame >= piece.firstFrame && frame - piece.firstFrame < piece.frames) return piece.data + (frame - piece.firstFrame) * frameBytes; + } + return nullptr; + } + return block.data + (static_cast(block.info.num_skip_samples) + frame) * frameBytes; } static void ReadPcm(const Ngs2Voice& voice, const std::uint8_t* frame, float* out) { + if (frame == nullptr) { + for (std::uint32_t channel = 0; channel < voice.channels; channel++) out[channel] = 0.0f; + return; + } for (std::uint32_t channel = 0; channel < voice.channels; channel++) { std::int16_t sample; std::memcpy(&sample, frame + channel * sizeof(sample), sizeof(sample)); @@ -34,7 +45,8 @@ static void ReadFrames(Ngs2Voice& voice, float* current, float* next) { return; } ReadPcm(voice, FrameAt(voice, block, block.cursor), current); - if (block.cursor + 1 < block.info.num_samples) ReadPcm(voice, FrameAt(voice, block, block.cursor + 1), next); + if (block.streaming && block.cursor + 1 >= block.availableFrames) std::memcpy(next, current, voice.channels * sizeof(float)); + else if (block.cursor + 1 < block.info.num_samples) ReadPcm(voice, FrameAt(voice, block, block.cursor + 1), next); else if (block.info.num_repeats != 0) ReadPcm(voice, FrameAt(voice, block, 0), next); else if (voice.blocks.size() > 1) ReadPcm(voice, FrameAt(voice, voice.blocks[1], 0), next); else std::memcpy(next, current, voice.channels * sizeof(float)); @@ -65,7 +77,7 @@ static void FinishBlock(Ngs2Voice& voice) { 0, }; if (!repeat) { - voice.waveformEnd = voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9 ? block.data + block.dataCursor : FrameAt(voice, block, block.info.num_samples); + voice.waveformEnd = voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9 ? block.data + block.dataCursor : block.streaming ? Ngs2StreamEnd(voice, block) : FrameAt(voice, block, block.info.num_samples); voice.blocks.pop_front(); } if (voice.callback != nullptr && (voice.callbackFlags & info.flag) != 0) voice.callback(&info); @@ -74,12 +86,17 @@ static void FinishBlock(Ngs2Voice& voice) { static void Advance(Ngs2Voice& voice, std::uint64_t frames) { while (frames != 0 && voice.state == Ngs2PlayState::Playing && !voice.blocks.empty()) { auto& block = voice.blocks.front(); - const auto step = std::min(frames, block.info.num_samples - block.cursor); + const std::uint64_t limit = block.streaming ? std::min(block.info.num_samples, block.availableFrames) : block.info.num_samples; + if (block.cursor >= limit && block.cursor != block.info.num_samples && !(block.streaming && !voice.acceptsBlocks)) break; + const auto step = std::min(frames, limit - block.cursor); block.cursor += static_cast(step); voice.decodedSamples += step; if (voice.waveformType != SCE_NGS2_WAVEFORM_TYPE_ATRAC9) voice.decodedBytes += step * voice.channels * sizeof(std::int16_t); frames -= step; - if (block.cursor == block.info.num_samples) FinishBlock(voice); + if (block.streaming) { + while (block.pieces.size() > 1 && block.cursor >= block.pieces.front().firstFrame + block.pieces.front().frames) block.pieces.erase(block.pieces.begin()); + } + if (block.cursor == block.info.num_samples || (block.streaming && !voice.acceptsBlocks && block.cursor == block.availableFrames)) FinishBlock(voice); } } @@ -171,14 +188,18 @@ static void RenderVoice(Ngs2Voice& voice, const std::vector& voices, if (voice.rendered) return; if (voice.rendering) throw std::invalid_argument("NGS2: the voice patches form a cycle"); voice.rendering = true; + struct RenderingScope { + bool& active; + ~RenderingScope() { active = false; } + } renderingScope{voice.rendering}; voice.samples.assign(static_cast(grain) * voice.channels, 0.0f); voice.hasSamples = false; if (voice.state == Ngs2PlayState::Playing && voice.channels != 0) { if (voice.rack->rackId == SCE_NGS2_RACK_ID_SAMPLER) RenderSampler(voice, grain, systemRate); else MixInputs(voice, voices, grain, systemRate); + Ngs2ProcessLegacyUserFx(voice, grain, systemRate); if (voice.hasSamples) Ngs2ProcessUserFx(voice, grain, systemRate); } - voice.rendering = false; voice.rendered = true; } diff --git a/core/libs/prx/libSceNgs2.native/src/Voice.cpp b/core/libs/prx/libSceNgs2.native/src/Voice.cpp index af69ff6fa..fb21bc22d 100644 --- a/core/libs/prx/libSceNgs2.native/src/Voice.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Voice.cpp @@ -25,6 +25,7 @@ void Ngs2Voice::SetEvent(std::uint32_t eventId) { if (state == Ngs2PlayState::Empty || state == Ngs2PlayState::Stopped) { state = Ngs2PlayState::Playing; stateFlags |= SCE_NGS2_VOICE_STATE_FLAG_INUSE; + if (userFxHandler) userFxFlags = 1; } break; case SCE_NGS2_VOICE_EVENT_STOP: @@ -62,12 +63,26 @@ void Ngs2Voice::ResetSetup() { decodedSamples = 0; decodedBytes = 0; waveformEnd = nullptr; + userFxHandler = nullptr; + userFxData = {}; + userFxFlags = 0; +} + +const std::uint8_t* Ngs2StreamEnd(const Ngs2Voice& voice, const Ngs2Block& block) { + const auto& last = block.pieces.back(); + return last.data + last.frames * voice.channels * sizeof(std::int16_t); } const std::uint8_t* Ngs2Voice::WaveformData() const { if (blocks.empty()) return waveformEnd; const auto& block = blocks.front(); if (waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9) return block.data + block.dataCursor; + if (block.streaming) { + for (const auto& piece : block.pieces) { + if (block.cursor >= piece.firstFrame && block.cursor - piece.firstFrame < piece.frames) return piece.data + (block.cursor - piece.firstFrame) * channels * sizeof(std::int16_t); + } + return Ngs2StreamEnd(*this, block); + } return block.data + (static_cast(block.info.num_skip_samples) + block.cursor) * channels * sizeof(std::int16_t); } @@ -150,8 +165,22 @@ static void SetupSampler(Ngs2Voice& voice, const Ngs2WaveformFormat& format) { if (format.waveform_type == SCE_NGS2_WAVEFORM_TYPE_ATRAC9) Ngs2SetupAtrac9(voice, format); } +static void AppendStreamData(Ngs2Voice& voice, const Ngs2SamplerVoiceWaveformBlocksParam& param, std::size_t frameBytes) { + if (voice.blocks.empty() || !voice.blocks.back().streaming) throw std::runtime_error("NGS2: appending waveform data to a voice without a streaming waveform is not implemented"); + auto& stream = voice.blocks.back(); + for (std::uint32_t i = 0; i < param.num_blocks; i++) { + const auto& block = param.blocks[i]; + if (block.data_size == 0) continue; + if (block.data_size % frameBytes != 0 || block.num_skip_samples != 0 || block.num_repeats != 0) { + throw std::runtime_error("NGS2: appending a waveform block with skipped samples, repeats or a partial frame is not implemented"); + } + stream.pieces.push_back({static_cast(param.data) + block.data_offset, stream.availableFrames, block.data_size / frameBytes}); + stream.availableFrames += block.data_size / frameBytes; + } +} + static void AddWaveformBlocks(Ngs2Voice& voice, const Ngs2SamplerVoiceWaveformBlocksParam& param) { - constexpr std::uint32_t knownFlags = SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE | SCE_NGS2_WAVEFORM_BLOCKS_FLAG_RESET; + constexpr std::uint32_t knownFlags = SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE | SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND | SCE_NGS2_WAVEFORM_BLOCKS_FLAG_RESET; if ((param.flags & ~knownFlags) != 0) throw std::runtime_error("NGS2: waveform block flags " + Ngs2Hex(param.flags) + " are not implemented"); if (voice.channels == 0 || (param.num_blocks != 0 && (param.blocks == nullptr || param.data == nullptr))) APS5_INVALID_ARG_EX; const bool reset = (param.flags & SCE_NGS2_WAVEFORM_BLOCKS_FLAG_RESET) != 0; @@ -162,17 +191,31 @@ static void AddWaveformBlocks(Ngs2Voice& voice, const Ngs2SamplerVoiceWaveformBl voice.waveformEnd = nullptr; if (voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9) Ngs2RestartAtrac9(voice); } - voice.acceptsBlocks = (param.flags & SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE) != 0; const std::size_t frameBytes = voice.channels * sizeof(std::int16_t); + if ((param.flags & SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND) != 0) { + if (voice.waveformType != SCE_NGS2_WAVEFORM_TYPE_PCM_I16L) throw std::runtime_error("NGS2: appending waveform data to a voice that is not 16-bit PCM is not implemented"); + AppendStreamData(voice, param, frameBytes); + voice.acceptsBlocks = (param.flags & SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE) != 0; + return; + } + voice.acceptsBlocks = (param.flags & SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE) != 0; for (std::uint32_t i = 0; i < param.num_blocks; i++) { const auto& block = param.blocks[i]; if (block.num_samples == 0 && block.data_size == 0) continue; const std::uint64_t bytes = voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9 ? Ngs2Atrac9BlockBytes(voice, block) : (static_cast(block.num_skip_samples) + block.num_samples) * frameBytes; - if (block.num_samples == 0 || bytes > block.data_size) { + const bool streaming = block.num_samples != 0 && bytes > block.data_size && voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_PCM_I16L && + block.num_skip_samples == 0 && block.num_repeats == 0 && block.data_size >= frameBytes && block.data_size % frameBytes == 0; + if (!streaming && (block.num_samples == 0 || bytes > block.data_size)) { throw std::invalid_argument("NGS2: waveform block " + std::to_string(i) + " does not fit its data"); } voice.blocks.push_back({static_cast(param.data) + block.data_offset, block}); + if (streaming) { + auto& stream = voice.blocks.back(); + stream.streaming = true; + stream.availableFrames = block.data_size / frameBytes; + stream.pieces.push_back({stream.data, 0, stream.availableFrames}); + } } } @@ -250,6 +293,13 @@ static void ApplyParam(Ngs2Voice& voice, const Ngs2VoiceParamHeader& param) { switch (rackId) { case SCE_NGS2_RACK_ID_SAMPLER: ApplySamplerParam(voice, param); return; case SCE_NGS2_RACK_ID_SUBMIXER: + if (param.id == SCE_NGS2_SUBMIXER_VOICE_PARAM_USER_FX) { + const auto& fx = ParamAs(param); + voice.userFxHandler = fx.handler; + voice.userFxData = {fx.user_data0, fx.user_data1, fx.user_data2}; + voice.userFxFlags = fx.handler ? 1 : 0; + return; + } if (param.id != SCE_NGS2_SUBMIXER_VOICE_PARAM_SETUP) break; if (ParamAs(param).flags != 0) throw std::runtime_error("NGS2: submixer setup flags are not implemented"); SetupMixer(voice, ParamAs(param).num_io_channels); diff --git a/core/libs/prx/libSceNgs2.native/tests/Ngs2Atrac9.cpp b/core/libs/prx/libSceNgs2.native/tests/Ngs2Atrac9.cpp index 739d8ae3c..44290013c 100644 --- a/core/libs/prx/libSceNgs2.native/tests/Ngs2Atrac9.cpp +++ b/core/libs/prx/libSceNgs2.native/tests/Ngs2Atrac9.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #ifdef _WIN32 @@ -41,16 +42,18 @@ static const std::uint8_t Superframe[SuperframeBytes] = { 0x09, 0x31, 0x5d, 0x92, 0x94, 0x0f, 0x8a, 0xae, 0x55, 0x49, 0xf7, 0xe9, 0x25, 0xea, 0x1c, 0xe6 }; -static std::vector Reference() { +static std::vector Reference(std::uint32_t superframes = 1) { std::uint8_t config[4] = {0xFE, 0x70, 0x07, 0xF0}; void* decoder = Atrac9GetHandle(); Require(Atrac9InitDecoder(decoder, config) == 0); - std::vector pcm(SuperframeSamples); - int offset = 0; - for (std::uint32_t frame = 0; frame < 4; frame++) { - int used = 0; - Require(Atrac9DecodeF32(decoder, Superframe + offset, static_cast(SuperframeBytes) - offset, pcm.data() + frame * 256, &used, 0) == 0); - offset += used; + std::vector pcm(SuperframeSamples * superframes); + for (std::uint32_t s = 0; s < superframes; ++s) { + int offset = 0; + for (std::uint32_t frame = 0; frame < 4; frame++) { + int used = 0; + Require(Atrac9DecodeF32(decoder, Superframe + offset, static_cast(SuperframeBytes) - offset, pcm.data() + s * SuperframeSamples + frame * 256, &used, 0) == 0); + offset += used; + } } Atrac9ReleaseHandle(decoder); return pcm; @@ -190,6 +193,107 @@ static void TestCalcBlock() { Require(sceNgs2CalcWaveformBlock(&silent, 0, 1, &block) == SCE_NGS2_ERROR_INVALID_WAVEFORM_FORMAT); } +static void Set32(std::vector& bytes, std::size_t offset, std::uint32_t value) { + for (int i = 0; i < 4; ++i) bytes[offset + i] = static_cast(value >> (i * 8)); +} + +static std::vector LoopedFile(std::uint32_t begin, std::uint32_t end, std::uint32_t plays, bool trailingSampler = false) { + auto file = At9File(48000); + Set32(file, 80, 4 * SuperframeSamples - 256); + Set32(file, 96, 4 * SuperframeBytes); + for (int i = 0; i < 3; ++i) file.insert(file.end(), Superframe, Superframe + SuperframeBytes); + std::vector sampler; + PutTag(sampler, "smpl"); + Put32(sampler, 60); + for (int i = 0; i < 7; ++i) Put32(sampler, 0); + Put32(sampler, 1); + Put32(sampler, 24); + Put32(sampler, 0); + Put32(sampler, 0); + Put32(sampler, begin + 256); + Put32(sampler, end + 256 - 1); + Put32(sampler, 0); + Put32(sampler, plays); + file.insert(trailingSampler ? file.end() : file.begin() + 92, sampler.begin(), sampler.end()); + Set32(file, 4, static_cast(file.size() - 8)); + return file; +} + +static void TestParseLoops() { + constexpr std::uint32_t samples = 4 * SuperframeSamples - 256; + const auto reference = Reference(4); + for (bool trailingSampler : {false, true}) { + for (const auto [begin, end] : {std::pair{0u, 800u}, {100u, 700u}, {768u, 2600u}, {850u, 2600u}, {1024u, 2600u}, {3072u, samples}, {0u, samples}}) { + const auto file = LoopedFile(begin, end, 2, trailingSampler); + Ngs2WaveformInfo info{}; + Require(sceNgs2ParseWaveformData(file.data(), file.size(), &info) == SCE_NGS2_OK); + Require(info.loop_begin_position == begin && info.loop_end_position == end && info.num_samples == samples); + Require(info.num_blocks == 1 + (begin != 0) + (end != samples)); + Require(info.block[begin != 0].num_repeats == 1); + const auto system = CreateSystem(); + const auto voice = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SAMPLER)); + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_SETUP, Ngs2SamplerVoiceSetupParam{{}, info.format}); + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, + Ngs2SamplerVoiceWaveformBlocksParam{{}, file.data(), 0, info.num_blocks, info.block}); + Patch(voice, Mastering(system, 1)); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + const auto rendered = Render(system, samples + end - begin + Grain); + for (std::uint32_t i = 0; i < rendered.size(); ++i) { + const auto index = i < end ? i : i - (end - begin); + Require(rendered[i] == (i < samples + end - begin ? reference[256 + index] : 0.0f)); + } + Require(Flags(voice) == 0); + Require(sceNgs2SystemDestroy(system, nullptr) == 0); + } + } + const auto file = LoopedFile(100, 700, 0); + Ngs2WaveformInfo info{}; + Require(sceNgs2ParseWaveformData(file.data(), 168, &info) == SCE_NGS2_OK); + Require(info.block[1].num_repeats == UINT32_MAX); + const auto system = CreateSystem(); + const auto voice = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SAMPLER)); + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_SETUP, Ngs2SamplerVoiceSetupParam{{}, info.format}); + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, + Ngs2SamplerVoiceWaveformBlocksParam{{}, file.data(), 0, info.num_blocks, info.block}); + Patch(voice, Mastering(system, 1)); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + auto rendered = Render(system, 1024); + const Ngs2VoiceParamHeader exitLoop{sizeof(Ngs2VoiceParamHeader), 0, SCE_NGS2_SAMPLER_VOICE_PARAM_EXIT_LOOP}; + Require(sceNgs2VoiceControl(voice, &exitLoop) == SCE_NGS2_OK); + const auto tail = Render(system, samples + 600 - 1024 + Grain); + rendered.insert(rendered.end(), tail.begin(), tail.end()); + for (std::uint32_t i = 0; i < rendered.size(); ++i) { + const auto index = i < 700 ? i : i - 600; + Require(rendered[i] == (i < samples + 600 ? reference[256 + index] : 0.0f)); + } + Require(Flags(voice) == 0); + Require(sceNgs2SystemDestroy(system, nullptr) == 0); +} + +static void TestMalformedLoops() { + const auto original = LoopedFile(100, 700, 0); + Ngs2WaveformInfo info{}; + for (const auto [offset, value] : {std::pair{128, 2}, {144, 0}, {148, 255}, {148, 5000}}) { + auto file = original; + Set32(file, offset, value); + Require(sceNgs2ParseWaveformData(file.data(), file.size(), &info) == SCE_NGS2_ERROR_INVALID_WAVEFORM_DATA); + } + for (const auto offset : {140u, 152u}) { + auto file = original; + Set32(file, offset, 1); + bool rejected = false; + try { sceNgs2ParseWaveformData(file.data(), file.size(), &info); } catch (const std::runtime_error&) { rejected = true; } + Require(rejected); + } + for (std::size_t size = 92; size < 168; ++size) + Require(sceNgs2ParseWaveformData(original.data(), size, &info) == SCE_NGS2_ERROR_INVALID_WAVEFORM_DATA); + auto shortSampler = original; + shortSampler.erase(shortSampler.begin() + 100 + 35, shortSampler.begin() + 160); + Set32(shortSampler, 96, 35); + Set32(shortSampler, 4, static_cast(shortSampler.size() - 8)); + Require(sceNgs2ParseWaveformData(shortSampler.data(), shortSampler.size(), &info) == SCE_NGS2_ERROR_INVALID_WAVEFORM_DATA); +} + static std::uint8_t* SuperframeBeforeGuardPage() { #ifdef _WIN32 SYSTEM_INFO info{}; @@ -235,6 +339,8 @@ int main() { TestSkipAndBlockEnd(reference); TestRepeatAndState(reference); TestParse(); + TestParseLoops(); + TestMalformedLoops(); TestCalcBlock(); TestCorruptSuperframeAtPageEnd(); return 0; diff --git a/core/libs/prx/libSceNgs2.native/tests/Ngs2Stream.cpp b/core/libs/prx/libSceNgs2.native/tests/Ngs2Stream.cpp new file mode 100644 index 000000000..4b983d3e7 --- /dev/null +++ b/core/libs/prx/libSceNgs2.native/tests/Ngs2Stream.cpp @@ -0,0 +1,136 @@ +#include "Ngs2Test.hpp" + +#include +#include +#include + +static std::vector RenderI16(uintptr_t system) { + std::vector out(Grain, -1); + const Ngs2RenderBufferInfo info{out.data(), out.size() * sizeof(std::int16_t), SCE_NGS2_WAVEFORM_TYPE_PCM_I16L, 1}; + Require(sceNgs2SystemRender(system, &info, 1) == SCE_NGS2_OK); + return out; +} + +static uintptr_t RackVoice(uintptr_t rack, std::uint32_t index) { + uintptr_t voice = 0; + Require(sceNgs2RackGetVoiceHandle(rack, index, &voice) == SCE_NGS2_OK && voice != 0); + return voice; +} + +static uintptr_t StreamVoice(uintptr_t rack, std::uint32_t index, uintptr_t master) { + const auto voice = RackVoice(rack, index); + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_SETUP, Ngs2SamplerVoiceSetupParam{{}, {SCE_NGS2_WAVEFORM_TYPE_PCM_I16L, 1, 48000, 0, 0, 0}}); + Patch(voice, master); + return voice; +} + +static void AddBlock(uintptr_t voice, const void* data, std::uint64_t bytes, std::uint32_t samples, std::uint32_t flags) { + const Ngs2WaveformBlock block{0, bytes, 0, 0, samples, 0, 0}; + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, Ngs2SamplerVoiceWaveformBlocksParam{{}, data, flags, 1, &block}); +} + +template +static bool Throws(TFunction function) { + try { + function(); + } catch (const std::exception&) { + return true; + } + return false; +} + +static void TestStreamEndsAtDeclaredSamples() { + const auto system = CreateSystem(); + const auto master = Mastering(system, 1); + const auto rack = CreateRack(system, SCE_NGS2_RACK_ID_SAMPLER); + const auto voice = StreamVoice(rack, 0, master); + const std::vector first(6, 16384); + const std::vector second(8, 8192); + const std::vector last(6, 4096); + + AddBlock(voice, first.data(), first.size() * 2, 20, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + auto out = RenderI16(system); + for (std::uint32_t i = 0; i < Grain; i++) Require(out[i] == (i < 6 ? 16384 : 0)); + Require(Flags(voice) == (SCE_NGS2_VOICE_STATE_FLAG_INUSE | SCE_NGS2_VOICE_STATE_FLAG_PLAYING)); + Ngs2SamplerVoiceState state{}; + Require(sceNgs2VoiceGetState(voice, &state.voice_state, sizeof(state)) == SCE_NGS2_OK); + Require(state.num_decoded_samples == 6 && state.decoded_data_size == 12 && state.waveform_data == first.data() + 6); + + out = RenderI16(system); + for (std::uint32_t i = 0; i < Grain; i++) Require(out[i] == 0); + Require(sceNgs2VoiceGetState(voice, &state.voice_state, sizeof(state)) == SCE_NGS2_OK); + Require(state.num_decoded_samples == 6 && state.decoded_data_size == 12); + + AddBlock(voice, second.data(), second.size() * 2, 20, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE | SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND); + Require(sceNgs2VoiceGetState(voice, &state.voice_state, sizeof(state)) == SCE_NGS2_OK && state.waveform_data == second.data()); + out = RenderI16(system); + for (std::uint32_t i = 0; i < Grain; i++) Require(out[i] == 8192); + Require(sceNgs2VoiceGetState(voice, &state.voice_state, sizeof(state)) == SCE_NGS2_OK); + Require(state.num_decoded_samples == 14 && state.decoded_data_size == 28 && state.waveform_data == second.data() + 8); + + AddBlock(voice, last.data(), last.size() * 2, 20, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND); + out = RenderI16(system); + for (std::uint32_t i = 0; i < Grain; i++) Require(out[i] == (i < 6 ? 4096 : 0)); + Require(Flags(voice) == 0); + Require(sceNgs2VoiceGetState(voice, &state.voice_state, sizeof(state)) == SCE_NGS2_OK); + Require(state.num_decoded_samples == 20 && state.decoded_data_size == 40 && state.waveform_data == last.data() + 6); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); +} + +static void TestStreamClosedBeforeDeclaredSamples() { + const auto system = CreateSystem(); + const auto master = Mastering(system, 1); + const auto rack = CreateRack(system, SCE_NGS2_RACK_ID_SAMPLER); + const auto voice = StreamVoice(rack, 0, master); + const std::vector first(6, 16384); + const std::vector closing(4, 8192); + + AddBlock(voice, first.data(), first.size() * 2, 1000, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + AddBlock(voice, closing.data(), closing.size() * 2, 1000, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND); + auto out = RenderI16(system); + for (std::uint32_t i = 0; i < Grain; i++) Require(out[i] == (i < 6 ? 16384 : 8192)); + Require(Flags(voice) == (SCE_NGS2_VOICE_STATE_FLAG_INUSE | SCE_NGS2_VOICE_STATE_FLAG_PLAYING)); + out = RenderI16(system); + for (std::uint32_t i = 0; i < Grain; i++) Require(out[i] == (i < 2 ? 8192 : 0)); + Require(Flags(voice) == 0); + Ngs2SamplerVoiceState state{}; + Require(sceNgs2VoiceGetState(voice, &state.voice_state, sizeof(state)) == SCE_NGS2_OK); + Require(state.num_decoded_samples == 10 && state.decoded_data_size == 20); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); +} + +static void TestStreamRejections() { + const auto system = CreateSystem(); + const auto master = Mastering(system, 1); + const auto rack = CreateRack(system, SCE_NGS2_RACK_ID_SAMPLER); + const std::vector pcm(6, 1000); + + const auto empty = StreamVoice(rack, 0, master); + Require(Throws([&] { AddBlock(empty, pcm.data(), pcm.size() * 2, 6, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND); })); + + const auto complete = StreamVoice(rack, 1, master); + AddBlock(complete, pcm.data(), pcm.size() * 2, 6, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE); + Require(Throws([&] { AddBlock(complete, pcm.data(), pcm.size() * 2, 6, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE | SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND); })); + + const auto repeating = StreamVoice(rack, 2, master); + const Ngs2WaveformBlock repeated{0, pcm.size() * 2, 1, 0, 20, 0, 0}; + Require(Throws([&] { Control(repeating, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, Ngs2SamplerVoiceWaveformBlocksParam{{}, pcm.data(), SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE, 1, &repeated}); })); + + const auto skipping = StreamVoice(rack, 3, master); + const Ngs2WaveformBlock skipped{0, pcm.size() * 2, 0, 1, 20, 0, 0}; + Require(Throws([&] { Control(skipping, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, Ngs2SamplerVoiceWaveformBlocksParam{{}, pcm.data(), SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE, 1, &skipped}); })); + + const auto partial = StreamVoice(rack, 4, master); + AddBlock(partial, pcm.data(), pcm.size() * 2, 20, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE); + Require(Throws([&] { AddBlock(partial, pcm.data(), 3, 20, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE | SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND); })); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); +} + +int main() { + TestStreamEndsAtDeclaredSamples(); + TestStreamClosedBeforeDeclaredSamples(); + TestStreamRejections(); + return 0; +} diff --git a/core/libs/prx/libSceNgs2.native/tests/Ngs2Test.hpp b/core/libs/prx/libSceNgs2.native/tests/Ngs2Test.hpp index 3f48fb8c9..97843654b 100644 --- a/core/libs/prx/libSceNgs2.native/tests/Ngs2Test.hpp +++ b/core/libs/prx/libSceNgs2.native/tests/Ngs2Test.hpp @@ -49,10 +49,11 @@ inline void Check(bool value, int line) { inline constexpr std::uint32_t Grain = 8; -inline std::vector buffers[32]; +inline std::vector> buffers; inline std::size_t usedBuffers = 0; inline Ngs2ContextBufferInfo Buffer(const Ngs2ContextBufferInfo& query) { + if (usedBuffers == buffers.size()) buffers.emplace_back(); auto& storage = buffers[usedBuffers++]; storage.resize(query.host_buffer_size / sizeof(std::uint64_t) + 1); Ngs2ContextBufferInfo info{}; diff --git a/core/libs/prx/libSceNgs2.native/tests/Ngs2UserFx.cpp b/core/libs/prx/libSceNgs2.native/tests/Ngs2UserFx.cpp new file mode 100644 index 000000000..e72e1dc86 --- /dev/null +++ b/core/libs/prx/libSceNgs2.native/tests/Ngs2UserFx.cpp @@ -0,0 +1,117 @@ +#include "Ngs2Test.hpp" + +#include +#include + +struct State { + std::uint32_t calls = 0; + std::uint32_t flags = 0; + std::uint32_t channels = 0; + float gain = 1.0f; + float bias = 0.0f; + int result = 0; +}; + +static int APS5_VABI Process(Ngs2UserFxProcessContext* context) { + auto& state = *reinterpret_cast(context->user_data0); + Require(context->user_data1 == 123 && context->user_data2 == 456); + Require(context->num_channels == state.channels && context->num_grain_samples == Grain && context->sample_rate == 48000); + state.calls++; + state.flags = context->flags; + for (std::uint32_t c = 0; c < context->num_channels; ++c) + for (std::uint32_t i = 0; i < context->num_grain_samples; ++i) + context->channel_data[c][i] = context->channel_data[c][i] * state.gain + state.bias * (c + 1); + return state.result; +} + +static void Install(uintptr_t voice, State& state) { + Control(voice, SCE_NGS2_SUBMIXER_VOICE_PARAM_USER_FX, + Ngs2SubmixerVoiceUserFxParam{{}, Process, reinterpret_cast(&state), 123, 456}); +} + +static std::vector Render(uintptr_t system, std::uint32_t channels) { + std::vector output(Grain * channels, -1.0f); + const Ngs2RenderBufferInfo buffer{output.data(), output.size() * sizeof(float), SCE_NGS2_WAVEFORM_TYPE_PCM_F32L, channels}; + Require(sceNgs2SystemRender(system, &buffer, 1) == 0); + return output; +} + +static void TestGenerator() { + const auto system = CreateSystem(); + const auto master = Mastering(system, 8); + const auto voice = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SUBMIXER)); + Control(voice, SCE_NGS2_SUBMIXER_VOICE_PARAM_SETUP, Ngs2SubmixerVoiceSetupParam{{}, 8, 0}); + State state{0, 0, 8, 1.0f, 0.125f}; + Install(voice, state); + Patch(voice, master); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + auto output = Render(system, 8); + Require(state.calls == 1 && state.flags == 1); + for (std::size_t i = 0; i < output.size(); ++i) Require(output[i] == 0.125f * (i % 8 + 1)); + Render(system, 8); + Require(state.calls == 2 && state.flags == 0); + Event(voice, SCE_NGS2_VOICE_EVENT_PAUSE); + for (float sample : Render(system, 8)) Require(sample == 0.0f); + Require(state.calls == 2); + Event(voice, SCE_NGS2_VOICE_EVENT_RESUME); + Render(system, 8); + Require(state.calls == 3 && state.flags == 0); + Event(voice, SCE_NGS2_VOICE_EVENT_STOP_IMM); + for (float sample : Render(system, 8)) Require(sample == 0.0f); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + Render(system, 8); + Require(state.calls == 4 && state.flags == 1); + Control(voice, SCE_NGS2_SUBMIXER_VOICE_PARAM_USER_FX, Ngs2SubmixerVoiceUserFxParam{}); + for (float sample : Render(system, 8)) Require(sample == 0.0f); + Require(state.calls == 4); + Install(voice, state); + state.result = -5; + bool rejected = false; + try { Render(system, 8); } catch (const std::runtime_error&) { rejected = true; } + Require(rejected); + state.result = 0; + Render(system, 8); + Require(state.calls == 6 && state.flags == 1); + Control(voice, SCE_NGS2_SUBMIXER_VOICE_PARAM_SETUP, Ngs2SubmixerVoiceSetupParam{{}, 8, 0}); + Patch(voice, master); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + for (float sample : Render(system, 8)) Require(sample == 0.0f); + Require(state.calls == 6); + Require(sceNgs2SystemDestroy(system, nullptr) == 0); +} + +static void TestEffectAndIndependentVoices() { + const auto system = CreateSystem(); + const auto master = Mastering(system, 2); + const auto first = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SUBMIXER)); + const auto second = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SUBMIXER)); + for (auto voice : {first, second}) { + Control(voice, SCE_NGS2_SUBMIXER_VOICE_PARAM_SETUP, Ngs2SubmixerVoiceSetupParam{{}, 2, 0}); + Patch(voice, master); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + } + State firstState{0, 0, 2, 0.5f, 0.0f}; + State secondState{0, 0, 2, 1.0f, 0.125f}; + Install(first, firstState); + Install(second, secondState); + const auto sampler = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SAMPLER)); + Control(sampler, SCE_NGS2_SAMPLER_VOICE_PARAM_SETUP, Ngs2SamplerVoiceSetupParam{{}, {SCE_NGS2_WAVEFORM_TYPE_PCM_I16L, 2, 48000, 0, 0, 0}}); + std::array pcm{}; + for (std::size_t i = 0; i < pcm.size(); ++i) pcm[i] = i % 2 == 0 ? 16384 : -8192; + const Ngs2WaveformBlock block{0, sizeof(pcm), 0, 0, Grain, 0, 0}; + Control(sampler, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, Ngs2SamplerVoiceWaveformBlocksParam{{}, pcm.data(), 0, 1, &block}); + Patch(sampler, first); + Event(sampler, SCE_NGS2_VOICE_EVENT_PLAY); + const auto mixed = Render(system, 2); + for (std::size_t i = 0; i < mixed.size(); ++i) Require(mixed[i] == (i % 2 == 0 ? 0.375f : 0.125f)); + Require(firstState.calls == 1 && secondState.calls == 1 && Flags(sampler) == 0); + const auto tail = Render(system, 2); + for (std::size_t i = 0; i < tail.size(); ++i) Require(tail[i] == (i % 2 == 0 ? 0.125f : 0.25f)); + Require(firstState.calls == 2 && secondState.calls == 2); + Require(sceNgs2SystemDestroy(system, nullptr) == 0); +} + +int main() { + TestGenerator(); + TestEffectAndIndependentVoices(); +} diff --git a/core/libs/prx/libSceNpEntitlementAccess/Export.cpp b/core/libs/prx/libSceNpEntitlementAccess/Export.cpp index ccf92707c..0cb5f3dd0 100644 --- a/core/libs/prx/libSceNpEntitlementAccess/Export.cpp +++ b/core/libs/prx/libSceNpEntitlementAccess/Export.cpp @@ -91,6 +91,16 @@ int APS5_VABI sceNpEntitlementAccessGetSkuFlag(uint32_t* sku_flag) { return 0; } +int APS5_VABI sceNpEntitlementAccessGetEntitlementKey( + uint32_t serviceLabel, const NpUnifiedEntitlementLabel* entitlementLabel, + NpEntitlementAccessEntitlementKey* key) { + (void)serviceLabel; + constexpr int errorParameter = static_cast(0x817D0002); + constexpr int errorNoEntitlement = static_cast(0x817D0007); + if (!entitlementLabel || !key) return errorParameter; + return errorNoEntitlement; +} + int APS5_VABI sceNpEntitlementAccessInitialize(const NpEntitlementAccessInitParam* init_param, NpEntitlementAccessBootParam* boot_param) { (void)init_param; (void)boot_param; @@ -150,23 +160,19 @@ int APS5_VABI sceNpEntitlementAccessGetPftFlag(void) { } int APS5_VABI sceNpEntitlementAccessPollServiceEntitlementInfoList() { - NotImplemented_nid_no_patch(__func__); - return 0; + return SCE_NP_ERROR_SIGNED_OUT; } int APS5_VABI sceNpEntitlementAccessPollUnifiedEntitlementInfoList() { - NotImplemented_nid_no_patch(__func__); - return 0; + return SCE_NP_ERROR_SIGNED_OUT; } int APS5_VABI sceNpEntitlementAccessRequestServiceEntitlementInfoList() { - NotImplemented_nid_no_patch(__func__); - return 0; + return SCE_NP_ERROR_SIGNED_OUT; } int APS5_VABI sceNpEntitlementAccessRequestUnifiedEntitlementInfoList() { - NotImplemented_nid_no_patch(__func__); - return 0; + return SCE_NP_ERROR_SIGNED_OUT; } } diff --git a/core/libs/prx/libSceNpManager/CMakeLists.txt b/core/libs/prx/libSceNpManager/CMakeLists.txt index 7406159c1..f0f629d89 100644 --- a/core/libs/prx/libSceNpManager/CMakeLists.txt +++ b/core/libs/prx/libSceNpManager/CMakeLists.txt @@ -9,6 +9,15 @@ if(MINGW) endif() if(BUILD_TESTING) + add_test_executable(guest_np_reachability_tests ${LIBS_INCLUDE_DIR}/tests/GuestNpReachability.cpp) + target_include_directories(guest_np_reachability_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_np_reachability_tests PRIVATE libSceNpManager) + configure_windows_unwind(guest_np_reachability_tests) + add_test(NAME guest_np_reachability COMMAND guest_np_reachability_tests) + if(WIN32) + set_tests_properties(guest_np_reachability PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_np_manager_tests ${LIBS_INCLUDE_DIR}/tests/GuestNpManager.cpp) target_include_directories(guest_np_manager_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_np_manager_tests PRIVATE libSceNpManager) diff --git a/core/libs/prx/libSceNpManager/Export.cpp b/core/libs/prx/libSceNpManager/Export.cpp index a9d412be4..511383f4c 100644 --- a/core/libs/prx/libSceNpManager/Export.cpp +++ b/core/libs/prx/libSceNpManager/Export.cpp @@ -3,16 +3,25 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" #include +#include // PSN is not emulated: the user is reported as signed out and online queries fail. static constexpr int SCE_NP_ERROR_INVALID_ARGUMENT = static_cast(0x80550003); static constexpr int SCE_NP_ERROR_SIGNED_OUT = static_cast(0x80550006); +static constexpr int SCE_NP_ERROR_CALLBACK_ALREADY_REGISTERED = static_cast(0x80550008); +static constexpr int SCE_NP_ERROR_CALLBACK_NOT_REGISTERED = static_cast(0x80550009); static constexpr uint32_t NP_STATE_SIGNED_OUT = 1; static constexpr int NP_POLL_ASYNC_FINISHED = 0; static constexpr uint32_t NP_REACHABILITY_STATE_UNAVAILABLE = 0; static std::atomic g_nextRequest{1}; +namespace { +std::mutex reachabilityMutex; +void* reachabilityCallback = nullptr; +void* reachabilityUserdata = nullptr; +} + extern "C" { int APS5_VABI sceNpAbortRequest(int req_id) { @@ -124,8 +133,19 @@ void APS5_VABI sceNpRegisterGamePresenceCallback(void* callback, void* userdata) } int APS5_VABI sceNpRegisterNpReachabilityStateCallback(void* callback, void* userdata) { - (void)userdata; if (!callback) return SCE_NP_ERROR_INVALID_ARGUMENT; + std::lock_guard lock(reachabilityMutex); + if (reachabilityCallback) return SCE_NP_ERROR_CALLBACK_ALREADY_REGISTERED; + reachabilityCallback = callback; + reachabilityUserdata = userdata; + return 0; +} + +int APS5_VABI sceNpUnregisterNpReachabilityStateCallback(void) { + std::lock_guard lock(reachabilityMutex); + if (!reachabilityCallback) return SCE_NP_ERROR_CALLBACK_NOT_REGISTERED; + reachabilityCallback = nullptr; + reachabilityUserdata = nullptr; return 0; } diff --git a/core/libs/prx/libSceNpSessionSignaling/Export.cpp b/core/libs/prx/libSceNpSessionSignaling/Export.cpp index 996613a42..b28941f1b 100644 --- a/core/libs/prx/libSceNpSessionSignaling/Export.cpp +++ b/core/libs/prx/libSceNpSessionSignaling/Export.cpp @@ -88,4 +88,9 @@ int APS5_VABI sceNpSessionSignalingGetConnectionStatistics(void) { NotImplemented_nid_no_patch(__func__); return 0; } + +int APS5_VABI sceNpSessionSignalingGetConnectionFromPeerAddress(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} } diff --git a/core/libs/prx/libScePad/Export.cpp b/core/libs/prx/libScePad/Export.cpp index c426e5067..26bc9347e 100644 --- a/core/libs/prx/libScePad/Export.cpp +++ b/core/libs/prx/libScePad/Export.cpp @@ -12,6 +12,7 @@ namespace { +constexpr int PAD_ERROR_DEVICE_NOT_CONNECTED = static_cast(0x80920007); constexpr int PAD_ERROR_DEVICE_NO_HANDLE = static_cast(0x80920008); bool g_opened = false; @@ -139,6 +140,22 @@ int APS5_VABI scePadOpen_nid_postfix(int userId, int type, int index, const void return PAD_HANDLE; } +int APS5_VABI scePadOpenExt(int userId, int type, int index, const void* param) { + if (!ValidPort(userId, type, index) || param == nullptr) { + return PAD_ERROR_INVALID_ARG; + } + if (type != PAD_PORT_TYPE_SPECIAL) NotImplemented_nid_no_patch(__func__); + return PAD_ERROR_DEVICE_NOT_CONNECTED; +} + +int APS5_VABI scePadReadExt() { NotImplemented_nid_no_patch(__func__); return 0; } + +int APS5_VABI scePadGetFeatureReport() { NotImplemented_nid_no_patch(__func__); return 0; } + +int APS5_VABI scePadSetFeatureReport() { NotImplemented_nid_no_patch(__func__); return 0; } + +int APS5_VABI scePadOutputReport() { NotImplemented_nid_no_patch(__func__); return 0; } + int APS5_VABI scePadReadState(int handle, PadData* data); int APS5_VABI scePadRead_nid_postfix(int handle, PadData* data, int num) { @@ -223,7 +240,7 @@ int APS5_VABI scePadSetVibration(int handle, const PadVibrationParam* param) { int APS5_VABI scePadSetVibrationMode(int handle, int mode) { if (handle != PAD_HANDLE) return PAD_ERROR_INVALID_HANDLE; - if (mode != 0 && mode != 1) return PAD_ERROR_INVALID_ARG; + if (mode < 0 || mode > 2) return PAD_ERROR_INVALID_ARG; Pad::SetVibrationMode(mode); return PAD_OK; } @@ -243,8 +260,15 @@ int APS5_VABI scePadVrControllerRead() { return 0; } -APS5_EXPORT("fCWdlnmB1Ks", scePadUnknown_fCWdlnmB1Ks); -int APS5_VABI scePadUnknown_fCWdlnmB1Ks(void) { +int APS5_VABI scePadIsRemoteController(int handle, bool* remote) { + if (handle != PAD_HANDLE) return PAD_ERROR_INVALID_HANDLE; + if (remote == nullptr) return PAD_ERROR_INVALID_ARG; + *remote = false; + return 0; +} + + +int APS5_VABI scePadSetAngularVelocityBiasCorrectionState() { NotImplemented_nid_no_patch(__func__); return 0; } diff --git a/core/libs/prx/libScePsml_debug/Export.cpp b/core/libs/prx/libScePsml_debug/Export.cpp index dcace7377..3eaf69d18 100644 --- a/core/libs/prx/libScePsml_debug/Export.cpp +++ b/core/libs/prx/libScePsml_debug/Export.cpp @@ -30,8 +30,7 @@ std::int32_t APS5_VABI scePsmlMfsrGetDispatchMfsrPacket1100(void* context, void* return SCE_PSML_ERROR_NOT_INITIALIZED; } -APS5_EXPORT("+2KpvixvL6E", scePsmlUnknown__P2KpvixvL6E); -int APS5_VABI scePsmlUnknown__P2KpvixvL6E() { +int APS5_VABI scePsmlMfsrGetSharedResourcesInitRequirement() { NotImplemented_nid_no_patch(__func__); return 0; } @@ -46,26 +45,22 @@ int APS5_VABI scePsmlMfsrGetDispatchMfsrPacketSizeInDwords() { return 0; } -APS5_EXPORT("ArakEpzsZo0", scePsmlUnknown_ArakEpzsZo0); -int APS5_VABI scePsmlUnknown_ArakEpzsZo0() { +int APS5_VABI scePsmlMfsrGetContextBufferRequirement800M3_2() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("FSGaTQze0UY", scePsmlUnknown_FSGaTQze0UY); -int APS5_VABI scePsmlUnknown_FSGaTQze0UY() { +int APS5_VABI scePsmlMfsrSelectConfig() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("GHna9-DvnUk", scePsmlUnknown_GHna9_MDvnUk); -int APS5_VABI scePsmlUnknown_GHna9_MDvnUk() { +int APS5_VABI scePsmlMfsrGetMipmapBias() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("GJY0MvuTcs8", scePsmlUnknown_GJY0MvuTcs8); -int APS5_VABI scePsmlUnknown_GJY0MvuTcs8() { +int APS5_VABI scePsmlMfsrRequestCapture() { NotImplemented_nid_no_patch(__func__); return 0; } @@ -75,87 +70,73 @@ int APS5_VABI scePsmlMfsrReleaseContext() { return 0; } -APS5_EXPORT("LXq+6mIxpCw", scePsmlUnknown_LXq_P6mIxpCw); -int APS5_VABI scePsmlUnknown_LXq_P6mIxpCw() { +int APS5_VABI scePsmlMfsrIsCaptureInProgress() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("RUNLFro+qok", scePsmlUnknown_RUNLFro_Pqok); -int APS5_VABI scePsmlUnknown_RUNLFro_Pqok() { +int APS5_VABI scePsmlMfsrGetDispatchMfsrPacket900() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("eWoKNeB6V-k", scePsmlUnknown_eWoKNeB6V_Mk); -int APS5_VABI scePsmlUnknown_eWoKNeB6V_Mk() { +int APS5_VABI scePsmlMfsrCreateSharedResources() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("gxv3i+MTEzU", scePsmlUnknown_gxv3i_PMTEzU); -int APS5_VABI scePsmlUnknown_gxv3i_PMTEzU() { +int APS5_VABI scePsmlMfsrCreateContext800M3_2() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("jEevBXmagOQ", scePsmlUnknown_jEevBXmagOQ); -int APS5_VABI scePsmlUnknown_jEevBXmagOQ() { +int APS5_VABI scePsmlMfsrReleaseSharedResources() { NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("0GAw7SmkwII", scePsmlMfsr2Unknown_0GAw7SmkwII); -int APS5_VABI scePsmlMfsr2Unknown_0GAw7SmkwII(void) { - NotImplemented_nid_no_patch("0GAw7SmkwII"); +int APS5_VABI scePsmlMfsr2ReleaseSharedResources() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("1ic5q-kdOsc", scePsmlMfsr2Unknown_1ic5q_MkdOsc); -int APS5_VABI scePsmlMfsr2Unknown_1ic5q_MkdOsc(void) { - NotImplemented_nid_no_patch("1ic5q-kdOsc"); +int APS5_VABI scePsmlMfsr2GetSharedResourcesInitRequirement() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("5z2gBlqxJ+0", scePsmlMfsr2Unknown_5z2gBlqxJ_P0); -int APS5_VABI scePsmlMfsr2Unknown_5z2gBlqxJ_P0(void) { - NotImplemented_nid_no_patch("5z2gBlqxJ+0"); +int APS5_VABI scePsmlMfsr2GetContextInitRequirement() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("Kyy1baXgaVU", scePsmlMfsr2Unknown_Kyy1baXgaVU); -int APS5_VABI scePsmlMfsr2Unknown_Kyy1baXgaVU(void) { - NotImplemented_nid_no_patch("Kyy1baXgaVU"); +int APS5_VABI scePsmlMfsr2ReleaseContext() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("ZLL31lzzxr4", scePsmlMfsr2Unknown_ZLL31lzzxr4); -int APS5_VABI scePsmlMfsr2Unknown_ZLL31lzzxr4(void) { - NotImplemented_nid_no_patch("ZLL31lzzxr4"); +int APS5_VABI scePsmlMfsr2GetDispatchPacketsSizeInDwords() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("gMduXCLYrNg", scePsmlMfsr2Unknown_gMduXCLYrNg); -int APS5_VABI scePsmlMfsr2Unknown_gMduXCLYrNg(void) { - NotImplemented_nid_no_patch("gMduXCLYrNg"); +int APS5_VABI scePsmlMfsr2CreateSharedResources() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("lrJwpLjKXRc", scePsmlMfsr2Unknown_lrJwpLjKXRc); -int APS5_VABI scePsmlMfsr2Unknown_lrJwpLjKXRc(void) { - NotImplemented_nid_no_patch("lrJwpLjKXRc"); +int APS5_VABI scePsmlMfsr2GetDispatchPackets() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("m9JLPc3wOQw", scePsmlMfsr2Unknown_m9JLPc3wOQw); -int APS5_VABI scePsmlMfsr2Unknown_m9JLPc3wOQw(void) { - NotImplemented_nid_no_patch("m9JLPc3wOQw"); +int APS5_VABI scePsmlMfsr2CreateContext() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("o+NM86gEwFE", scePsmlMfsr2Unknown_o_PNM86gEwFE); -int APS5_VABI scePsmlMfsr2Unknown_o_PNM86gEwFE(void) { - NotImplemented_nid_no_patch("o+NM86gEwFE"); +int APS5_VABI scePsmlMfsr2Init() { + NotImplemented_nid_no_patch(__func__); return 0; } diff --git a/core/libs/prx/libSceRtc/Export.cpp b/core/libs/prx/libSceRtc/Export.cpp index a8986f228..7772cb1fb 100644 --- a/core/libs/prx/libSceRtc/Export.cpp +++ b/core/libs/prx/libSceRtc/Export.cpp @@ -149,110 +149,158 @@ bool parseDigits(const char*& cursor, int count, int& value) { return true; } -bool parseZoneSuffix(const char*& cursor, std::int64_t& offsetMinutes) { - offsetMinutes = 0; - if (*cursor == 'Z' || *cursor == 'z') { +RtcDateTime makeDateTime(int year, int month, int day, int hour, int minute, int second, std::uint32_t microsecond) { + return RtcDateTime{static_cast(year), static_cast(month), static_cast(day), + static_cast(hour), static_cast(minute), static_cast(second), microsecond}; +} + +int storeParsedTick(RtcTick* utc, const RtcDateTime& time, std::int64_t offsetMinutes) { + if (validate(&time) == 0) utc->tick = toTick(time); + utc->tick -= static_cast(offsetMinutes * TICKS_PER_MINUTE); + return 0; +} + +int parseRfc3339(RtcTick* utc, const char* cursor) { + int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; + if (!parseDigits(cursor, 4, year) || *cursor++ != '-' || !parseDigits(cursor, 2, month) || *cursor++ != '-' || !parseDigits(cursor, 2, day)) return SCE_RTC_ERROR_BAD_PARSE; + if (*cursor != 'T' && *cursor != 't') return SCE_RTC_ERROR_BAD_PARSE; + ++cursor; + if (!parseDigits(cursor, 2, hour) || *cursor++ != ':' || !parseDigits(cursor, 2, minute) || *cursor++ != ':' || !parseDigits(cursor, 2, second)) return SCE_RTC_ERROR_BAD_PARSE; + std::uint32_t microsecond = 0; + if (*cursor == '.') { ++cursor; - return true; + for (std::uint32_t scale = 100000; *cursor >= '0' && *cursor <= '9'; ++cursor, scale /= 10) microsecond += static_cast(*cursor - '0') * scale; } + std::int64_t offsetMinutes = 0; if (*cursor == '+' || *cursor == '-') { const int sign = *cursor++ == '-' ? -1 : 1; int offsetHours = 0, offsetMinute = 0; - if (!parseDigits(cursor, 2, offsetHours) || *cursor++ != ':' || !parseDigits(cursor, 2, offsetMinute)) return false; - if (offsetHours > 23 || offsetMinute > 59) return false; + if (!parseDigits(cursor, 2, offsetHours) || *cursor++ != ':' || !parseDigits(cursor, 2, offsetMinute)) return SCE_RTC_ERROR_BAD_PARSE; offsetMinutes = sign * (offsetHours * 60 + offsetMinute); - return true; + } else if (*cursor != 'Z' && *cursor != 'z') { + return SCE_RTC_ERROR_BAD_PARSE; } - return false; + const RtcDateTime time = makeDateTime(year, month, day, hour, minute, second, microsecond); + if (const int result = validate(&time); result != 0 && (result != SCE_RTC_ERROR_INVALID_SECOND || second != 60)) return result; + return storeParsedTick(utc, time, offsetMinutes); +} + +bool isDigit(char c) { + return c >= '0' && c <= '9'; +} + +char toUpper(char c) { + return c >= 'a' && c <= 'z' ? static_cast(c - 'a' + 'A') : c; +} + +int readDigits(const char*& cursor, int maxDigits, int& value) { + value = 0; + int count = 0; + for (; count < maxDigits && isDigit(*cursor); ++count, ++cursor) value = value * 10 + (*cursor - '0'); + return count; +} + +bool matchWord(const char*& cursor, const char* word, std::size_t length) { + for (std::size_t index = 0; index < length; ++index) { + if (toUpper(cursor[index]) != word[index]) return false; + } + cursor += length; + return true; } bool parseName(const char*& cursor, const char* const* names, int count, int& index) { for (index = 0; index < count; ++index) { - if (std::strncmp(cursor, names[index], 3) == 0) { - cursor += 3; + if (matchWord(cursor, names[index], std::strlen(names[index])) || matchWord(cursor, names[index], 3)) return true; + } + return false; +} + +bool parseWeekday(const char*& cursor) { + static constexpr const char* names[] = {"SUNDAY", "MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY", "SATURDAY"}; + int index = 0; + return parseName(cursor, names, 7, index); +} + +bool parseMonth(const char*& cursor, int& month) { + static constexpr const char* names[] = {"JANUARY", "FEBRUARY", "MARCH", "APRIL", "MAY", "JUNE", "JULY", "AUGUST", "SEPTEMBER", "OCTOBER", "NOVEMBER", "DECEMBER"}; + if (!parseName(cursor, names, 12, month)) return false; + ++month; + return true; +} + +bool parseZoneName(const char* cursor, std::int64_t& offsetMinutes) { + static constexpr struct { + const char* name; + std::int64_t minutes; + } zones[] = { + {"GMT", 0}, {"WET", 0}, {"BST", 60}, {"WAT", -60}, {"AST", -240}, {"ADT", -180}, {"EST", -300}, {"EDT", -240}, + {"CST", -360}, {"CDT", -300}, {"MST", -420}, {"MDT", -360}, {"PST", -480}, {"PDT", -420}, {"YST", -540}, {"YDT", -480}, + {"HDT", -540}, {"CAT", -600}, {"AHST", -600}, {"NST", -150}, {"NDT", -90}, {"IDLW", -720}, {"CET", 60}, {"MET", 60}, + {"MEWT", 60}, {"MEST", 120}, {"CEST", 120}, {"SST", 120}, {"FWT", 120}, {"FST", 60}, {"EET", 120}, {"IST", 120}, + {"WAST", 420}, {"WADT", 480}, {"CCT", 480}, {"AWST", 480}, {"JST", 540}, {"KST", 540}, {"ACST", 570}, {"EAST", 600}, + {"GST", 600}, {"AEST", 600}, {"NZT", 720}, {"NZST", 720}, {"NZDT", 780}, {"IDLE", 720}, {"JT", 450}, + }; + for (const auto& zone : zones) { + const char* name = cursor; + if (matchWord(name, zone.name, std::strlen(zone.name))) { + offsetMinutes = zone.minutes; return true; } } return false; } -bool parseWeekday(const char*& cursor) { - static constexpr const char* names[] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; - int index = 0; - return parseName(cursor, names, 7, index); -} - -bool parseMonth(const char*& cursor, int& month) { - static constexpr const char* names[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; - if (!parseName(cursor, names, 12, month)) return false; - ++month; - return true; -} - -bool parseClock(const char*& cursor, int& hour, int& minute, int& second) { - return parseDigits(cursor, 2, hour) && *cursor++ == ':' && parseDigits(cursor, 2, minute) && *cursor++ == ':' && parseDigits(cursor, 2, second); -} - -RtcDateTime makeDateTime(int year, int month, int day, int hour, int minute, int second, std::uint32_t microsecond) { - return RtcDateTime{static_cast(year), static_cast(month), static_cast(day), - static_cast(hour), static_cast(minute), static_cast(second), microsecond}; -} - -bool parseIsoDateTime(const char* cursor, RtcDateTime& time, std::int64_t& offsetMinutes) { - int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; - if (!parseDigits(cursor, 4, year) || *cursor++ != '-' || !parseDigits(cursor, 2, month) || *cursor++ != '-' || !parseDigits(cursor, 2, day)) return false; - if (*cursor != 'T' && *cursor != 't' && *cursor != ' ') return false; - ++cursor; - if (!parseClock(cursor, hour, minute, second)) return false; - std::uint32_t microsecond = 0; - if (*cursor == '.') { - ++cursor; - std::uint32_t scale = 100000; - if (*cursor < '0' || *cursor > '9') return false; - for (; *cursor >= '0' && *cursor <= '9'; ++cursor, scale /= 10) microsecond += static_cast(*cursor - '0') * scale; - } +bool parseZone(const char* cursor, std::int64_t& offsetMinutes) { offsetMinutes = 0; - if (*cursor != 0 && !parseZoneSuffix(cursor, offsetMinutes)) return false; - if (*cursor != 0) return false; - time = makeDateTime(year, month, day, hour, minute, second, microsecond); + if (*cursor == 0) return true; + if (*cursor == '+' || *cursor == '-') { + const std::int64_t sign = *cursor++ == '-' ? -1 : 1; + int hours = 0, minutes = 0; + if (parseDigits(cursor, 2, hours) && parseDigits(cursor, 2, minutes)) offsetMinutes = sign * (hours * 60 + minutes); + return true; + } + const char letter = toUpper(*cursor); + if (letter < 'A' || letter > 'Z') return false; + if (cursor[0] == 'U' || cursor[1] == 'T' || parseZoneName(cursor, offsetMinutes)) return true; + if (letter == 'J') return false; + if (letter <= 'M') offsetMinutes = (letter - 'A') * 60; + else if (letter < 'Z') offsetMinutes = ('N' - letter) * 60; return true; } bool parseRfc2822DateTime(const char* cursor, RtcDateTime& time, std::int64_t& offsetMinutes) { int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; - if (!parseWeekday(cursor) || *cursor++ != ',' || *cursor++ != ' ' || !parseDigits(cursor, 2, day) || *cursor++ != ' ') return false; - if (!parseMonth(cursor, month) || *cursor++ != ' ' || !parseDigits(cursor, 4, year) || *cursor++ != ' ' || !parseClock(cursor, hour, minute, second)) return false; + if (readDigits(cursor, 2, day) == 0 || (*cursor != ' ' && *cursor != '-')) return false; + ++cursor; + if (!parseMonth(cursor, month) || (*cursor != ' ' && *cursor != '-')) return false; + ++cursor; + const int yearDigits = readDigits(cursor, 4, year); + if ((yearDigits != 2 && yearDigits != 4) || *cursor++ != ' ') return false; + if (yearDigits == 2) year += year < 50 ? 2000 : 1900; + if (readDigits(cursor, 2, hour) == 0 || hour > 25 || *cursor++ != ':' || readDigits(cursor, 2, minute) == 0) return false; offsetMinutes = 0; - if (*cursor == ' ') { + if (*cursor == ':') { ++cursor; - if (std::strcmp(cursor, "GMT") == 0) { - cursor += 3; - } else if (*cursor == '+' || *cursor == '-') { - const int sign = *cursor++ == '-' ? -1 : 1; - int offsetHours = 0, offsetMinute = 0; - if (!parseDigits(cursor, 2, offsetHours) || !parseDigits(cursor, 2, offsetMinute)) return false; - if (offsetHours > 23 || offsetMinute > 59) return false; - offsetMinutes = sign * (offsetHours * 60 + offsetMinute); - } else { - return false; - } + if (readDigits(cursor, 2, second) == 0) return false; + if (*cursor == ' ' && !parseZone(cursor + 1, offsetMinutes)) return false; + } else if (*cursor != 0 && *cursor != ' ') { + return false; } - if (*cursor != 0) return false; time = makeDateTime(year, month, day, hour, minute, second, 0); return true; } -bool parseAsctimeDateTime(const char* cursor, RtcDateTime& time, std::int64_t& offsetMinutes) { +bool parseAsctimeDateTime(const char* cursor, RtcDateTime& time) { int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; - if (!parseWeekday(cursor) || *cursor++ != ' ' || !parseMonth(cursor, month) || *cursor++ != ' ') return false; + if (!parseMonth(cursor, month) || *cursor++ != ' ') return false; if (*cursor == ' ') { ++cursor; - if (!parseDigits(cursor, 1, day)) return false; - } else if (!parseDigits(cursor, 2, day)) { + if (readDigits(cursor, 1, day) == 0) return false; + } else if (readDigits(cursor, 2, day) == 0) { return false; } - if (*cursor++ != ' ' || !parseClock(cursor, hour, minute, second) || *cursor++ != ' ' || !parseDigits(cursor, 4, year) || *cursor != 0) return false; - offsetMinutes = 0; + if (*cursor++ != ' ' || readDigits(cursor, 2, hour) == 0 || *cursor++ != ':' || readDigits(cursor, 2, minute) == 0 || *cursor++ != ':') return false; + if (readDigits(cursor, 2, second) == 0 || *cursor++ != ' ' || readDigits(cursor, 4, year) != 4) return false; time = makeDateTime(year, month, day, hour, minute, second, 0); return true; } @@ -373,7 +421,7 @@ int APS5_VABI sceRtcGetDosTime(const RtcDateTime* time, uint32_t* dos_time) { if (!dos_time) return SCE_RTC_ERROR_INVALID_POINTER; if (const int result = validate(time); result != 0) return result; if (time->year < 1980 || time->year > 2107) { - NotImplemented_nid_no_patch(__func__); + *dos_time = time->year < 1980 ? 0 : 0xff9fbf7du; return SCE_RTC_ERROR_INVALID_YEAR; } *dos_time = static_cast(time->year - 1980) << 25 | static_cast(time->month) << 21 @@ -434,41 +482,21 @@ int APS5_VABI sceRtcFormatRFC2822LocalTime(char* date_time, const RtcTick* utc) int APS5_VABI sceRtcParseRFC3339(RtcTick* utc, const char* date_time) { if (!utc || !date_time) return SCE_RTC_ERROR_INVALID_POINTER; - const char* cursor = date_time; - int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; - if (!parseDigits(cursor, 4, year) || *cursor++ != '-' || !parseDigits(cursor, 2, month) || *cursor++ != '-' || !parseDigits(cursor, 2, day)) return SCE_RTC_ERROR_BAD_PARSE; - if (*cursor != 'T' && *cursor != 't' && *cursor != ' ') return SCE_RTC_ERROR_BAD_PARSE; - ++cursor; - if (!parseDigits(cursor, 2, hour) || *cursor++ != ':' || !parseDigits(cursor, 2, minute) || *cursor++ != ':' || !parseDigits(cursor, 2, second)) return SCE_RTC_ERROR_BAD_PARSE; - std::uint32_t microsecond = 0; - if (*cursor == '.') { - ++cursor; - std::uint32_t scale = 100000; - if (*cursor < '0' || *cursor > '9') return SCE_RTC_ERROR_BAD_PARSE; - for (; *cursor >= '0' && *cursor <= '9'; ++cursor, scale /= 10) microsecond += static_cast(*cursor - '0') * scale; - } - std::int64_t offsetMinutes = 0; - if (!parseZoneSuffix(cursor, offsetMinutes)) return SCE_RTC_ERROR_BAD_PARSE; - if (*cursor != 0) return SCE_RTC_ERROR_BAD_PARSE; - const RtcDateTime time{static_cast(year), static_cast(month), static_cast(day), - static_cast(hour), static_cast(minute), static_cast(second), microsecond}; - if (const int result = validate(&time); result != 0) return result; - const RtcTick local{toTick(time)}; - return addTicks(utc, &local, -offsetMinutes, TICKS_PER_MINUTE); + return parseRfc3339(utc, date_time); } int APS5_VABI sceRtcParseDateTime(RtcTick* utc, const char* date_time) { if (!utc || !date_time) return SCE_RTC_ERROR_INVALID_POINTER; + const char* cursor = date_time; + while (*cursor == ' ' || *cursor == '\t') ++cursor; + if (isDigit(*cursor)) return parseRfc3339(utc, cursor); + if (!parseWeekday(cursor)) return SCE_RTC_ERROR_BAD_PARSE; + if (*cursor == ',') ++cursor; + while (*cursor == ' ' || *cursor == '\t') ++cursor; RtcDateTime time{}; std::int64_t offsetMinutes = 0; - if (!parseIsoDateTime(date_time, time, offsetMinutes) && !parseRfc2822DateTime(date_time, time, offsetMinutes) - && !parseAsctimeDateTime(date_time, time, offsetMinutes)) { - NotImplemented_nid_no_patch(__func__); - return SCE_RTC_ERROR_BAD_PARSE; - } - if (const int result = validate(&time); result != 0) return result; - const RtcTick local{toTick(time)}; - return addTicks(utc, &local, -offsetMinutes, TICKS_PER_MINUTE); + if (isDigit(*cursor) ? !parseRfc2822DateTime(cursor, time, offsetMinutes) : !parseAsctimeDateTime(cursor, time)) return SCE_RTC_ERROR_BAD_PARSE; + return storeParsedTick(utc, time, offsetMinutes); } int APS5_VABI sceRtcTickAddTicks(RtcTick* dst, const RtcTick* src, int64_t ticks) { diff --git a/core/libs/prx/libSceRudp/Export.cpp b/core/libs/prx/libSceRudp/Export.cpp index bb5821fd9..7b8eb5082 100644 --- a/core/libs/prx/libSceRudp/Export.cpp +++ b/core/libs/prx/libSceRudp/Export.cpp @@ -46,6 +46,18 @@ int APS5_VABI sceRudpInit_nid_postfix(void* mem_pool, int mem_pool_size) { return 0; } +int APS5_VABI sceRudpActivate() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceRudpGetStatus(void* status, std::size_t size) { + (void)status; + (void)size; + NotImplemented_nid_no_patch(__func__); + return 0; +} + int APS5_VABI sceRudpEnd(int ctx_id) { std::lock_guard lk(g_mutex); if (!g_inited) { diff --git a/core/libs/prx/libSceSaveData.native/CMakeLists.txt b/core/libs/prx/libSceSaveData.native/CMakeLists.txt index 02d2f6b71..496c0b9bc 100644 --- a/core/libs/prx/libSceSaveData.native/CMakeLists.txt +++ b/core/libs/prx/libSceSaveData.native/CMakeLists.txt @@ -3,6 +3,16 @@ target_include_directories(libSceSaveData.native PRIVATE ${LIBS_INCLUDE_DIR}) configure_windows_unwind(libSceSaveData.native) if(BUILD_TESTING) + add_test_executable(guest_savedata_search_tests ${LIBS_INCLUDE_DIR}/tests/GuestSaveDataSearch.cpp) + target_include_directories(guest_savedata_search_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_savedata_search_tests PRIVATE libSceSaveData.native libc) + configure_windows_unwind(guest_savedata_search_tests) + add_test(NAME guest_savedata_search COMMAND guest_savedata_search_tests) + if(WIN32) + set_tests_properties(guest_savedata_search PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(guest_savedata_delete_tests ${LIBS_INCLUDE_DIR}/tests/GuestSaveDataDelete.cpp) target_include_directories(guest_savedata_delete_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_savedata_delete_tests PRIVATE libSceSaveData.native libc) @@ -12,6 +22,15 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() + add_test_executable(guest_savedata_mount_status_tests ${LIBS_INCLUDE_DIR}/tests/GuestSaveDataMountStatus.cpp) + target_include_directories(guest_savedata_mount_status_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_savedata_mount_status_tests PRIVATE libSceSaveData.native libc) + add_test(NAME guest_savedata_mount_status COMMAND guest_savedata_mount_status_tests) + if(WIN32) + set_tests_properties(guest_savedata_mount_status PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(guest_savedata_ps4_tests ${LIBS_INCLUDE_DIR}/tests/GuestSaveDataPs4.cpp) target_include_directories(guest_savedata_ps4_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_savedata_ps4_tests PRIVATE libSceSaveData.native libc) diff --git a/core/libs/prx/libSceSaveData.native/Export.cpp b/core/libs/prx/libSceSaveData.native/Export.cpp index f58ae05f0..f6270b5dd 100644 --- a/core/libs/prx/libSceSaveData.native/Export.cpp +++ b/core/libs/prx/libSceSaveData.native/Export.cpp @@ -482,7 +482,7 @@ static int mount3(const SaveDataMount3* mount, SaveDataMountResult* mount_result g_slots[slot].real_path = real_path; std::memcpy(mount_result->mount_point.data, mountPoint.c_str(), mountPoint.size() + 1); mount_result->required_blocks = 0; - mount_result->mount_status = (create || create2) ? 1u : 0u; + mount_result->mount_status = (create || (create2 && !exists)) ? 1u : 0u; return SAVE_DATA_OK; } @@ -577,8 +577,8 @@ static int setSaveDataMemory2(const SaveDataMemorySet2* set_param) { } } if (n != 0) { - std::fstream f(path, std::ios::binary | std::ios::in | std::ios::out); - if (!f) { + std::vector memory; + if (!read_file_all(path, memory) || memory.size() != size) { return SAVE_DATA_ERROR_INTERNAL; } for (std::uint32_t i = 0; i < n; i++) { @@ -586,11 +586,9 @@ static int setSaveDataMemory2(const SaveDataMemorySet2* set_param) { if (d.buf_size == 0) { continue; } - f.seekp(static_cast(d.offset)); - f.write(static_cast(d.buf), static_cast(d.buf_size)); + std::memcpy(memory.data() + d.offset, d.buf, d.buf_size); } - f.flush(); - if (!f) { + if (!write_file_replace(path, memory)) { return SAVE_DATA_ERROR_INTERNAL; } } @@ -741,4 +739,14 @@ int APS5_VABI sceSaveDataDirNameSearchPs4(const SaveDataDirNameSearchCond* cond, SAVEDATA_TRACE("dirNameSearchPs4 user=%d -> 0x%08x", cond->user_id, static_cast(rc)); return rc; } + +int APS5_VABI sceSaveDataConvert() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceSaveDataGetConvertProgress() { + NotImplemented_nid_no_patch(__func__); + return 0; +} } diff --git a/core/libs/prx/libSceSaveData/SaveDataCore.hpp b/core/libs/prx/libSceSaveData/SaveDataCore.hpp index 1a810cb07..200b60771 100644 --- a/core/libs/prx/libSceSaveData/SaveDataCore.hpp +++ b/core/libs/prx/libSceSaveData/SaveDataCore.hpp @@ -203,7 +203,7 @@ const std::filesystem::path& save_root() { } bool ensure_save_root(std::error_code& error) { - std::filesystem::create_directories(save_root(), error); + if (std::filesystem::create_directories(save_root(), error)) RecordWrittenPath_nid_no_patch(save_root()); if (error) { error.clear(); } @@ -233,6 +233,7 @@ void store_aliases_locked(State& st) { for (const auto& [alias, dir_name] : st.aliases) { file << alias << '\t' << dir_name << '\n'; } + RecordWrittenPath_nid_no_patch(save_root() / "_aliases"); } std::string save_alias_locked(State& st, const std::string& dir_name) { @@ -289,7 +290,9 @@ std::filesystem::path save_blocks_path(const std::filesystem::path& directory) { } bool write_blob(const std::filesystem::path& path, const void* data, std::size_t size) { - return replace_file(path, data, size); + if (!replace_file(path, data, size)) return false; + RecordWrittenPath_nid_no_patch(path); + return true; } bool read_blob(const std::filesystem::path& path, void* data, std::size_t size) { @@ -499,6 +502,7 @@ int mount_internal(const SceSaveDataDirName* dir_name, std::uint32_t mount_mode, if (!std::filesystem::create_directories(directory, error) || error) { return SD_ERROR_INTERNAL; } + RecordWrittenPath_nid_no_patch(directory); created = true; if (write_u64(save_blocks_path(directory), blocks) != SD_OK) { std::error_code cleanup; @@ -579,6 +583,7 @@ int delete_internal(std::int32_t user_id, const SceSaveDataDirName* dir_name, return SD_ERROR_NOT_FOUND; } std::filesystem::remove_all(directory, error); + RecordWrittenPath_nid_no_patch(directory); if (error) { return SD_ERROR_INTERNAL; } @@ -1286,7 +1291,8 @@ int set_local_storage_app_status_internal(const SceSaveDataDirName* dir_name, return SD_ERROR_PARAMETER; } std::error_code error; - std::filesystem::create_directories(app_status_path(dir_name->data).parent_path(), error); + if (std::filesystem::create_directories(app_status_path(dir_name->data).parent_path(), error)) + RecordWrittenPath_nid_no_patch(app_status_path(dir_name->data).parent_path()); if (error) { return SD_ERROR_INTERNAL; } @@ -1301,7 +1307,8 @@ int delete_local_storage_app_status_internal(const SceSaveDataDirName* dir_name, return SD_ERROR_PARAMETER; } std::error_code error; - std::filesystem::remove(app_status_path(dir_name->data), error); + if (std::filesystem::remove(app_status_path(dir_name->data), error)) + RecordWrittenPath_nid_no_patch(app_status_path(dir_name->data)); return error ? SD_ERROR_INTERNAL : SD_OK; } diff --git a/core/libs/prx/libSceSaveDataDialog.native/CMakeLists.txt b/core/libs/prx/libSceSaveDataDialog.native/CMakeLists.txt new file mode 100644 index 000000000..ddbc456db --- /dev/null +++ b/core/libs/prx/libSceSaveDataDialog.native/CMakeLists.txt @@ -0,0 +1,15 @@ +add_library(libSceSaveDataDialog.native SHARED EXCLUDE_FROM_ALL Export.cpp) +target_include_directories(libSceSaveDataDialog.native PRIVATE ${LIBS_INCLUDE_DIR}) +configure_windows_unwind(libSceSaveDataDialog.native) + +if(BUILD_TESTING) + add_test_executable(guest_savedata_dialog_tests ${LIBS_INCLUDE_DIR}/tests/GuestSaveDataDialog.cpp) + target_include_directories(guest_savedata_dialog_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_savedata_dialog_tests PRIVATE libSceSaveDataDialog.native) + configure_windows_unwind(guest_savedata_dialog_tests) + add_test(NAME guest_savedata_dialog COMMAND guest_savedata_dialog_tests) + if(WIN32) + set_tests_properties(guest_savedata_dialog PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceSaveDataDialog.native/Export.cpp b/core/libs/prx/libSceSaveDataDialog.native/Export.cpp index 0835cdff4..3c834f7d9 100644 --- a/core/libs/prx/libSceSaveDataDialog.native/Export.cpp +++ b/core/libs/prx/libSceSaveDataDialog.native/Export.cpp @@ -69,7 +69,8 @@ int APS5_VABI sceSaveDataDialogOpen(const void* param) { } } } - g_status = SAVE_DATA_DIALOG_STATUS_FINISHED; + g_status = g_mode == SAVE_DATA_DIALOG_MODE_PROGRESS_BAR + ? SAVE_DATA_DIALOG_STATUS_RUNNING : SAVE_DATA_DIALOG_STATUS_FINISHED; return SAVE_DATA_DIALOG_OK; } diff --git a/core/libs/prx/libSceSaveDataDialog.native/SaveDataDialog.hpp b/core/libs/prx/libSceSaveDataDialog.native/SaveDataDialog.hpp index 907db674a..3259b8d48 100644 --- a/core/libs/prx/libSceSaveDataDialog.native/SaveDataDialog.hpp +++ b/core/libs/prx/libSceSaveDataDialog.native/SaveDataDialog.hpp @@ -12,8 +12,11 @@ constexpr int SAVE_DATA_DIALOG_ERROR_ARG_NULL = static_cast(0x80B8000D); constexpr int SAVE_DATA_DIALOG_STATUS_NONE = 0; constexpr int SAVE_DATA_DIALOG_STATUS_INITIALIZED = 1; +constexpr int SAVE_DATA_DIALOG_STATUS_RUNNING = 2; constexpr int SAVE_DATA_DIALOG_STATUS_FINISHED = 3; +constexpr int SAVE_DATA_DIALOG_MODE_PROGRESS_BAR = 5; + constexpr int SAVE_DATA_DIALOG_RESULT_OK = 0; constexpr int SAVE_DATA_DIALOG_BUTTON_ID_OK = 1; diff --git a/core/libs/prx/libSceSsl/Export.cpp b/core/libs/prx/libSceSsl/Export.cpp index 5ef04eacb..cd9b00548 100644 --- a/core/libs/prx/libSceSsl/Export.cpp +++ b/core/libs/prx/libSceSsl/Export.cpp @@ -3,11 +3,16 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" #include +#include +#include +#include // No network is emulated: contexts, templates and requests can be created, but any request // that would touch the network fails with the library's network error. static constexpr int ERROR_NETWORK = static_cast(0x80435001); static std::atomic g_nextHandle{1}; +static std::mutex g_poolsMutex; +static std::map g_pools; namespace { @@ -19,6 +24,13 @@ struct SslData { size_t size; }; +struct SslMemoryPoolStats { + size_t pool_size; + size_t max_inuse_size; + size_t current_inuse_size; + int32_t reserved; +}; + struct SslCaCerts { SslData* certs; size_t num; @@ -44,12 +56,15 @@ int APS5_VABI sceSslGetCaCerts(int ssl_ctx_id, void* ca_certs) { } int APS5_VABI sceSslInit_nid_postfix(uint64_t pool_size) { - (void)pool_size; - return g_nextHandle.fetch_add(1, std::memory_order_relaxed); + const int id = g_nextHandle.fetch_add(1, std::memory_order_relaxed); + std::lock_guard lock(g_poolsMutex); + g_pools[id] = pool_size; + return id; } int APS5_VABI sceSslTerm_nid_postfix(int ssl_ctx_id) { - (void)ssl_ctx_id; + std::lock_guard lock(g_poolsMutex); + g_pools.erase(ssl_ctx_id); return 0; } @@ -68,8 +83,12 @@ int APS5_VABI sceSslLoadCert() { return 0; } -int APS5_VABI sceSslGetMemoryPoolStats() { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceSslGetMemoryPoolStats(int ssl_ctx_id, SslMemoryPoolStats* stats) { + if (stats == nullptr) APS5_INVALID_ARG_EX; + std::lock_guard lock(g_poolsMutex); + const auto pool = g_pools.find(ssl_ctx_id); + if (pool == g_pools.end()) throw std::invalid_argument("sceSslGetMemoryPoolStats: unknown context"); + *stats = {static_cast(pool->second), 0, 0, 0}; return 0; } diff --git a/core/libs/prx/libSceSystemService/CMakeLists.txt b/core/libs/prx/libSceSystemService/CMakeLists.txt index b1676e0f5..e90856e4e 100644 --- a/core/libs/prx/libSceSystemService/CMakeLists.txt +++ b/core/libs/prx/libSceSystemService/CMakeLists.txt @@ -29,4 +29,17 @@ if(BUILD_TESTING) set_tests_properties(guest_system_service PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_system_kill_app_tests ${LIBS_INCLUDE_DIR}/tests/GuestSystemKillApp.cpp) + target_include_directories(guest_system_kill_app_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_system_kill_app_tests PRIVATE libSceSystemService libc) + configure_windows_unwind(guest_system_kill_app_tests) + add_test(NAME guest_system_kill_app_parameters COMMAND guest_system_kill_app_tests) + add_test(NAME guest_system_kill_app COMMAND guest_system_kill_app_tests --kill) + add_test(NAME guest_system_kill_app_status COMMAND guest_system_kill_app_tests --kill) + set_tests_properties(guest_system_kill_app PROPERTIES + PASS_REGULAR_EXPRESSION "Guest shutdown and atexit completed") + if(WIN32) + set_tests_properties(guest_system_kill_app guest_system_kill_app_status guest_system_kill_app_parameters PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() diff --git a/core/libs/prx/libSceSystemService/Export.cpp b/core/libs/prx/libSceSystemService/Export.cpp index 0de0c37f4..c3c3c4709 100644 --- a/core/libs/prx/libSceSystemService/Export.cpp +++ b/core/libs/prx/libSceSystemService/Export.cpp @@ -121,13 +121,11 @@ int APS5_VABI sceSystemServiceInitializePlayerDialogParam(void* param) { } int APS5_VABI sceSystemServiceDisableMediaPlay() { - NotImplemented_nid_no_patch(__func__); - return 0; + return SYSTEM_SERVICE_OK; } int APS5_VABI sceSystemServiceReenableMediaPlay() { - NotImplemented_nid_no_patch(__func__); - return 0; + return SYSTEM_SERVICE_OK; } int APS5_VABI sceSystemServiceLaunchPlayerDialog(const void* param) { @@ -158,4 +156,18 @@ int APS5_VABI sceSystemServiceShowControllerSettings(void) { return 0; } +int APS5_VABI sceSystemServiceGetAppIdOfRunningBigApp(void) { + return SYSTEM_SERVICE_RUNNING_APP_ID; +} + +int APS5_VABI sceSystemServiceKillApp(int appId, int how, int reason, int coreDump) { + if (appId != SYSTEM_SERVICE_RUNNING_APP_ID) { + NotImplemented_nid_no_patch("sceSystemServiceKillApp: application other than the running title"); + } + if (how != -1 || reason != 0 || coreDump != 0) { + NotImplemented_nid_no_patch("sceSystemServiceKillApp: arguments other than -1, 0 and 0"); + } + LibcExit_nid_no_patch(0); +} + } diff --git a/core/libs/prx/libSceSystemService/SystemService.hpp b/core/libs/prx/libSceSystemService/SystemService.hpp index e86820da4..109d1c269 100644 --- a/core/libs/prx/libSceSystemService/SystemService.hpp +++ b/core/libs/prx/libSceSystemService/SystemService.hpp @@ -8,6 +8,8 @@ constexpr int SYSTEM_SERVICE_OK = 0; constexpr int SYSTEM_SERVICE_ERROR_PARAMETER = -2136932349; constexpr int SYSTEM_SERVICE_ERROR_NO_EVENT = -2136932348; +constexpr int SYSTEM_SERVICE_RUNNING_APP_ID = 1; + constexpr int SYSTEM_SERVICE_PARAM_ID_LANG = 1; constexpr int SYSTEM_SERVICE_PARAM_ID_DATE_FORMAT = 2; constexpr int SYSTEM_SERVICE_PARAM_ID_TIME_FORMAT = 3; diff --git a/core/libs/prx/libSceTextToSpeech2/CMakeLists.txt b/core/libs/prx/libSceTextToSpeech2/CMakeLists.txt new file mode 100644 index 000000000..cb54c1384 --- /dev/null +++ b/core/libs/prx/libSceTextToSpeech2/CMakeLists.txt @@ -0,0 +1,15 @@ +add_library(libSceTextToSpeech2 SHARED EXCLUDE_FROM_ALL Export.cpp) +target_include_directories(libSceTextToSpeech2 PRIVATE ${LIBS_INCLUDE_DIR}) +configure_windows_unwind(libSceTextToSpeech2) + +if(BUILD_TESTING) + add_test_executable(guest_text_to_speech2_tests ${LIBS_INCLUDE_DIR}/tests/GuestTextToSpeech2.cpp) + target_include_directories(guest_text_to_speech2_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_text_to_speech2_tests PRIVATE libSceTextToSpeech2 libc) + configure_windows_unwind(guest_text_to_speech2_tests) + add_test(NAME guest_text_to_speech2 COMMAND guest_text_to_speech2_tests) + if(WIN32) + set_tests_properties(guest_text_to_speech2 PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceTextToSpeech2/Export.cpp b/core/libs/prx/libSceTextToSpeech2/Export.cpp index 2b94ca2f8..8e344ce25 100644 --- a/core/libs/prx/libSceTextToSpeech2/Export.cpp +++ b/core/libs/prx/libSceTextToSpeech2/Export.cpp @@ -3,6 +3,12 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +namespace { + +constexpr std::int32_t SCE_KERNEL_ERROR_EOPNOTSUPP = static_cast(0x8002002D); + +} + extern "C" { int APS5_VABI sceTextToSpeech2GetSystemStatus() { @@ -10,4 +16,39 @@ int APS5_VABI sceTextToSpeech2GetSystemStatus() { return 0; } +std::int32_t APS5_VABI sceTextToSpeech2Initialize(const void* param) { + (void)param; + return SCE_KERNEL_ERROR_EOPNOTSUPP; +} + +int APS5_VABI sceTextToSpeech2Cancel() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceTextToSpeech2Close() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceTextToSpeech2GetSpeechStatus() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceTextToSpeech2Open() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceTextToSpeech2Speak() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceTextToSpeech2Terminate() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + } diff --git a/core/libs/prx/libSceUlt/CMakeLists.txt b/core/libs/prx/libSceUlt/CMakeLists.txt index 56c1216ca..e9ce2a507 100644 --- a/core/libs/prx/libSceUlt/CMakeLists.txt +++ b/core/libs/prx/libSceUlt/CMakeLists.txt @@ -15,4 +15,17 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endforeach() + + add_test_executable(guest_ult_thread_tests ${LIBS_INCLUDE_DIR}/tests/GuestUltThread.cpp) + target_include_directories(guest_ult_thread_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_ult_thread_tests PRIVATE libSceUlt libc) + configure_windows_unwind(guest_ult_thread_tests) + foreach(testCase IN ITEMS outstanding validation) + add_test(NAME guest_ult_thread_${testCase} COMMAND guest_ult_thread_tests ${testCase}) + set_tests_properties(guest_ult_thread_${testCase} PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_ult_thread_${testCase} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() endif() diff --git a/core/libs/prx/libSceUlt/Export.cpp b/core/libs/prx/libSceUlt/Export.cpp index a224d3c35..979033e5a 100644 --- a/core/libs/prx/libSceUlt/Export.cpp +++ b/core/libs/prx/libSceUlt/Export.cpp @@ -123,6 +123,9 @@ int APS5_VABI sceUltFinalize() { std::vector> semaphores; { std::lock_guard lock(gMutex); + if (!gUlthreads.empty()) { + return ULT_ERROR_BUSY; + } semaphores.reserve(gSemaphores.size()); for (auto& entry : gSemaphores) { auto& state = entry.second; @@ -143,7 +146,6 @@ int APS5_VABI sceUltFinalize() { gQueues.clear(); gQueueDataPools.clear(); gRuntimes.clear(); - gUlthreads.clear(); } for (const auto& state : semaphores) { state->_available.notify_all(); diff --git a/core/libs/prx/libSceUsbd/CMakeLists.txt b/core/libs/prx/libSceUsbd/CMakeLists.txt new file mode 100644 index 000000000..d7acf60da --- /dev/null +++ b/core/libs/prx/libSceUsbd/CMakeLists.txt @@ -0,0 +1,15 @@ +add_library(libSceUsbd SHARED EXCLUDE_FROM_ALL Export.cpp) +target_include_directories(libSceUsbd PRIVATE ${LIBS_INCLUDE_DIR}) +configure_windows_unwind(libSceUsbd) + +if(BUILD_TESTING) + add_test_executable(guest_usbd_tests ${LIBS_INCLUDE_DIR}/tests/GuestUsbd.cpp) + target_include_directories(guest_usbd_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_usbd_tests PRIVATE libSceUsbd libc) + configure_windows_unwind(guest_usbd_tests) + add_test(NAME guest_usbd COMMAND guest_usbd_tests) + if(WIN32) + set_tests_properties(guest_usbd PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceUsbd/Export.cpp b/core/libs/prx/libSceUsbd/Export.cpp index fb34206c2..c5c238411 100644 --- a/core/libs/prx/libSceUsbd/Export.cpp +++ b/core/libs/prx/libSceUsbd/Export.cpp @@ -3,8 +3,172 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" -extern "C" { +#include +#include +#include -APS5_DUMMY_FUN +namespace { + +constexpr std::int32_t SCE_USBD_ERROR_INVALID_ARG = static_cast(0x80240002); + +struct UsbdTimeval { + std::int64_t seconds; + std::int64_t microseconds; +}; + +void* g_emptyDeviceList[1] = {nullptr}; + +} + +extern "C" { + +std::int32_t APS5_VABI sceUsbdInit() { + return 0; +} + +void APS5_VABI sceUsbdExit() { +} + +std::int64_t APS5_VABI sceUsbdGetDeviceList(void*** list) { + if (list == nullptr) return SCE_USBD_ERROR_INVALID_ARG; + *list = g_emptyDeviceList; + return 0; +} + +void APS5_VABI sceUsbdFreeDeviceList(void** list, std::int32_t unrefDevices) { + (void)unrefDevices; + if (list != nullptr && list != g_emptyDeviceList) throw std::runtime_error(std::string(__func__) + ": list was not returned by sceUsbdGetDeviceList"); +} + +std::int32_t APS5_VABI sceUsbdHandleEventsTimeout(const UsbdTimeval* timeout) { + if (timeout == nullptr || timeout->seconds < 0 || timeout->microseconds < 0 || timeout->microseconds >= 1000000) return SCE_USBD_ERROR_INVALID_ARG; + std::this_thread::sleep_for(std::chrono::seconds(timeout->seconds) + std::chrono::microseconds(timeout->microseconds)); + return 0; +} + +int APS5_VABI sceUsbdAllocTransfer() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdAttachKernelDriver() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdCancelTransfer() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdCheckConnected() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdClaimInterface() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdClose() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdControlTransfer() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdEventHandlingOk() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdFillInterruptTransfer() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdFreeConfigDescriptor() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdFreeTransfer() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdGetActiveConfigDescriptor() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdGetBusNumber() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdGetConfigDescriptor() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdGetDeviceAddress() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdGetDeviceDescriptor() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdGetStringDescriptor() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdKernelDriverActive() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdOpen() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdRefDevice() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdReleaseInterface() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdResetDevice() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdSetConfiguration() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdSubmitTransfer() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceUsbdUnrefDevice() { + NotImplemented_nid_no_patch(__func__); + return 0; +} } diff --git a/core/libs/prx/libSceVideoOut/CMakeLists.txt b/core/libs/prx/libSceVideoOut/CMakeLists.txt index 075b2aab4..2a15963e5 100644 --- a/core/libs/prx/libSceVideoOut/CMakeLists.txt +++ b/core/libs/prx/libSceVideoOut/CMakeLists.txt @@ -27,7 +27,7 @@ foreach(runtimeLinkFlag IN ITEMS -static-libgcc -static-libstdc++ -static) string(REPLACE "${runtimeLinkFlag}" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") endforeach() add_test_executable(video_out_flip_tests EXCLUDE_FROM_ALL tests/Flip.cpp) -target_include_directories(video_out_flip_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/SDL2/include) +target_include_directories(video_out_flip_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/SDL2/include ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) target_link_libraries(video_out_flip_tests PRIVATE libSceVideoOut libSceAgcDriver libkernel libc) set_target_properties(video_out_flip_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(video_out_flip_tests) @@ -44,10 +44,14 @@ if(BUILD_TESTING) target_link_libraries(guest_video_out_wqhd_detection_tests PRIVATE libSceVideoOut libSceAgcDriver libkernel libc) configure_windows_unwind(guest_video_out_wqhd_detection_tests) add_test(NAME guest_video_out_wqhd_detection COMMAND guest_video_out_wqhd_detection_tests) - set_tests_properties(guest_video_out_wqhd_detection PROPERTIES TIMEOUT 20) + set_tests_properties(guest_video_out_wqhd_detection PROPERTIES TIMEOUT 20 SKIP_RETURN_CODE 77) if(WIN32) set_tests_properties(guest_video_out_wqhd_detection PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + add_test(NAME guest_video_out_wqhd_detection_no_vulkan COMMAND guest_video_out_wqhd_detection_tests) + set_tests_properties(guest_video_out_wqhd_detection_no_vulkan PROPERTIES TIMEOUT 20 SKIP_RETURN_CODE 77 + ENVIRONMENT "VK_DRIVER_FILES=${CMAKE_CURRENT_BINARY_DIR}/missing_icd.json;VK_ICD_FILENAMES=${CMAKE_CURRENT_BINARY_DIR}/missing_icd.json" + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceVideoOut/include/FrameTimingLog.hpp b/core/libs/prx/libSceVideoOut/include/FrameTimingLog.hpp new file mode 100644 index 000000000..56f96a6d4 --- /dev/null +++ b/core/libs/prx/libSceVideoOut/include/FrameTimingLog.hpp @@ -0,0 +1,94 @@ +#ifndef CORE_LIBS_PRX_LIBSCEVIDEOOUT_INCLUDE_FRAMETIMINGLOG_HPP +#define CORE_LIBS_PRX_LIBSCEVIDEOOUT_INCLUDE_FRAMETIMINGLOG_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class FrameTimingLog { +public: + using Report = std::function; + + FrameTimingLog() { + pending.reserve(maxReports); + file.rdbuf()->pubsetbuf(buffer.data(), static_cast(buffer.size())); + file.exceptions(std::ios::badbit | std::ios::failbit); + file.imbue(std::locale::classic()); + file.open("frame-timing.log", std::ios::out | std::ios::binary | std::ios::trunc); + worker = std::jthread([this](std::stop_token token) { run(token); }); + } + + ~FrameTimingLog() { + worker.request_stop(); + changed.notify_one(); + } + + void Enqueue(Report report) { + { + std::lock_guard lock(mutex); + if (failure != nullptr) std::rethrow_exception(failure); + if (worker.get_stop_token().stop_requested()) throw std::runtime_error("frame timing log is closed"); + if (pending.size() == maxReports) throw std::runtime_error("frame timing log queue is full"); + pending.push_back(std::move(report)); + } + changed.notify_one(); + } + + void Finish() { + worker.request_stop(); + changed.notify_one(); + if (worker.joinable()) worker.join(); + if (failure != nullptr) std::rethrow_exception(failure); + } + +private: + void run(std::stop_token token) { + try { + std::vector reports; + reports.reserve(maxReports); + auto flushAt = std::chrono::steady_clock::now() + std::chrono::seconds(1); + for (;;) { + bool stopping = false; + { + std::unique_lock lock(mutex); + changed.wait_until(lock, flushAt, [&] { return token.stop_requested() || !pending.empty(); }); + reports.swap(pending); + stopping = token.stop_requested(); + } + for (const auto& report : reports) report(file); + reports.clear(); + if (stopping) break; + const auto now = std::chrono::steady_clock::now(); + if (now >= flushAt) { + file.flush(); + flushAt = now + std::chrono::seconds(1); + } + } + file.close(); + } catch (...) { + std::lock_guard lock(mutex); + failure = std::current_exception(); + } + } + + static constexpr std::size_t maxReports = 256; + std::array buffer{}; + std::ofstream file; + std::mutex mutex; + std::condition_variable changed; + std::vector pending; + std::exception_ptr failure; + std::jthread worker; +}; + +#endif diff --git a/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp b/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp index 97c1ef47f..02adf6ea0 100644 --- a/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp +++ b/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp" @@ -41,6 +42,9 @@ static constexpr int VIDEO_OUT_BUS_TYPE_MAIN = 0; static constexpr int VIDEO_OUT_BUS_TYPE_OVERLAY = 1; static constexpr int VIDEO_OUT_BUS_TYPE_SUB = 2; +static constexpr uint32_t VIDEO_OUT_DEFAULT_WIDTH = 1920; +static constexpr uint32_t VIDEO_OUT_DEFAULT_HEIGHT = 1080; + static constexpr std::uint32_t VIDEO_OUT_OPEN_PARAM_FIRST_WORD = 16; static constexpr std::int32_t VIDEO_OUT_SERVICE_THREAD_PRIORITY_HIGHEST = 256; static constexpr std::int32_t VIDEO_OUT_SERVICE_THREAD_PRIORITY_LOWEST = 767; @@ -113,8 +117,8 @@ struct VideoOutConfig { std::vector preVblankEvents; std::vector outputModeEvents; - uint32_t width = 1920; - uint32_t height = 1080; + uint32_t width = VIDEO_OUT_DEFAULT_WIDTH; + uint32_t height = VIDEO_OUT_DEFAULT_HEIGHT; uint64_t generation = 0; bool opened = false; bool closing = false; @@ -201,7 +205,7 @@ public: private: bool close(int handle); - void presentLoop(std::stop_token token); + void presentLoop(std::stop_token token, std::promise& started); void vblankLoop(std::stop_token token); void vblankEnd(); void processFlip(FlipRequest& req); diff --git a/core/libs/prx/libSceVideoOut/src/Buffer.cpp b/core/libs/prx/libSceVideoOut/src/Buffer.cpp index 5c8c2870e..481c92c2f 100644 --- a/core/libs/prx/libSceVideoOut/src/Buffer.cpp +++ b/core/libs/prx/libSceVideoOut/src/Buffer.cpp @@ -88,7 +88,6 @@ int APS5_VABI sceVideoOutRegisterBuffers2(int handle, int setIndex, int bufferIn group.occupied = true; std::array registered{}; for (int i = 0; i < bufferNum; ++i) { - if (buffers[i].reserved[0] != nullptr || buffers[i].reserved[1] != nullptr) throw std::runtime_error("VideoOut: reserved buffer pointers are set"); registered[i] = {setIndex, reinterpret_cast(buffers[i].data), reinterpret_cast(buffers[i].metadata)}; const auto display = DescribeVideoOutBuffer(registered[i], group); AgcDriverCheckGuestMemory_nid_postfix(buffers[i].data, AgcDriverDisplayBufferSize_nid_postfix(display), 65536); diff --git a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp index b61d35d41..8f185445a 100644 --- a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp +++ b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp @@ -16,10 +16,14 @@ #include "prx/libkernel/Equeue/Equeue.hpp" #include "prx/libkernel/Time/include/Time.hpp" #include "prx/libSceVideoOut/include/VideoOutDriver.hpp" +#if APS5_ENABLE_TIMING_LOG +#include "prx/libSceVideoOut/include/FrameTimingLog.hpp" +#endif #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Presentation.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" +#include "prx/libc/include/PreciseWait.hpp" #include "prx/libc/include/Shutdown.hpp" namespace { @@ -28,6 +32,34 @@ void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(std::string("VideoOut: ") + reason); } +std::vector windowExtensions(SDL_Window* window) { + unsigned extensionCount = 0; + if (!SDL_Vulkan_GetInstanceExtensions(window, &extensionCount, nullptr)) throw std::runtime_error(std::string("SDL_Vulkan_GetInstanceExtensions failed: ") + SDL_GetError()); + std::vector extensions(extensionCount); + if (!SDL_Vulkan_GetInstanceExtensions(window, &extensionCount, extensions.data())) throw std::runtime_error(std::string("SDL_Vulkan_GetInstanceExtensions failed: ") + SDL_GetError()); + extensions.resize(extensionCount); + return extensions; +} + +VkSurfaceKHR createWindowSurface(void* context, VkInstance instance) { + VkSurfaceKHR surface = VK_NULL_HANDLE; + if (!SDL_Vulkan_CreateSurface(static_cast(context), instance, &surface)) throw std::runtime_error(std::string("SDL_Vulkan_CreateSurface failed: ") + SDL_GetError()); + return surface; +} + +void windowDrawableSize(void* context, std::uint32_t* width, std::uint32_t* height) { + if ((SDL_GetWindowFlags(static_cast(context)) & SDL_WINDOW_MINIMIZED) != 0) { + *width = 0; + *height = 0; + return; + } + int drawableWidth = 0; + int drawableHeight = 0; + SDL_Vulkan_GetDrawableSize(static_cast(context), &drawableWidth, &drawableHeight); + *width = drawableWidth > 0 ? static_cast(drawableWidth) : 0; + *height = drawableHeight > 0 ? static_cast(drawableHeight) : 0; +} + void checkConfig(const VideoOutConfig& cfg) { cfg.Check(); } @@ -205,7 +237,10 @@ VideoOutDriver::VideoOutDriver() { } try { AgcDriverWaitIdle_nid_postfix(); - presentThread = std::jthread([this](std::stop_token token) { presentLoop(token); }); + std::promise started; + auto attached = started.get_future(); + presentThread = std::jthread([this, &started](std::stop_token token) { presentLoop(token, started); }); + attached.get(); vblankThread = std::jthread([this](std::stop_token token) { vblankLoop(token); }); LibcRegisterShutdown_nid_postfix([] { VideoOutDriver::Get().Shutdown(); }); } catch (...) { @@ -398,27 +433,8 @@ void VideoOutDriver::processFlip(FlipRequest& req) { } require(req.width != 0 && req.height != 0 && req.width <= static_cast(std::numeric_limits::max()) && req.height <= static_cast(std::numeric_limits::max()), "invalid window dimensions"); window.Ensure(req.width, req.height); - unsigned extensionCount = 0; - if (!SDL_Vulkan_GetInstanceExtensions(window.Handle(), &extensionCount, nullptr)) throw std::runtime_error(std::string("SDL_Vulkan_GetInstanceExtensions failed: ") + SDL_GetError()); - std::vector extensions(extensionCount); - if (!SDL_Vulkan_GetInstanceExtensions(window.Handle(), &extensionCount, extensions.data())) throw std::runtime_error(std::string("SDL_Vulkan_GetInstanceExtensions failed: ") + SDL_GetError()); - extensions.resize(extensionCount); - const AgcDriver::PresentationWindow target{window.Handle(), extensions, [](void* context, VkInstance instance) { - VkSurfaceKHR surface = VK_NULL_HANDLE; - if (!SDL_Vulkan_CreateSurface(static_cast(context), instance, &surface)) throw std::runtime_error(std::string("SDL_Vulkan_CreateSurface failed: ") + SDL_GetError()); - return surface; - }, [](void* context, std::uint32_t* width, std::uint32_t* height) { - if ((SDL_GetWindowFlags(static_cast(context)) & SDL_WINDOW_MINIMIZED) != 0) { - *width = 0; - *height = 0; - return; - } - int drawableWidth = 0; - int drawableHeight = 0; - SDL_Vulkan_GetDrawableSize(static_cast(context), &drawableWidth, &drawableHeight); - *width = drawableWidth > 0 ? static_cast(drawableWidth) : 0; - *height = drawableHeight > 0 ? static_cast(drawableHeight) : 0; - }, req.width, req.height, req.timing}; + const auto extensions = windowExtensions(window.Handle()); + const AgcDriver::PresentationWindow target{window.Handle(), extensions, &createWindowSurface, &windowDrawableSize, req.width, req.height, req.timing}; timing.Mark("window_prepare"); const auto gpuReady = [](void* context) { auto& request = *static_cast(context); @@ -462,9 +478,25 @@ void VideoOutDriver::processFlip(FlipRequest& req) { timing.Mark("notify_game"); } -void VideoOutDriver::presentLoop(std::stop_token token) { - std::shared_ptr current; +void VideoOutDriver::presentLoop(std::stop_token token, std::promise& started) { try { + window.Ensure(VIDEO_OUT_DEFAULT_WIDTH, VIDEO_OUT_DEFAULT_HEIGHT); + const auto extensions = windowExtensions(window.Handle()); + AgcDriverAttachWindow_nid_postfix({window.Handle(), extensions, &createWindowSurface, &windowDrawableSize, VIDEO_OUT_DEFAULT_WIDTH, VIDEO_OUT_DEFAULT_HEIGHT, nullptr}); + } catch (...) { + window.Destroy(); + started.set_exception(std::current_exception()); + return; + } + started.set_value(); + std::shared_ptr current; +#if APS5_ENABLE_TIMING_LOG + std::unique_ptr timingLog; +#endif + try { +#if APS5_ENABLE_TIMING_LOG + timingLog = std::make_unique(); +#endif PadInput padInput; MouseInput mouseInput; KeyboardInput keyboardInput; @@ -506,7 +538,9 @@ void VideoOutDriver::presentLoop(std::stop_token token) { if (previous != AgcDriver::FrameTiming::Clock::time_point{}) interval = finished - previous; current->cfg->lastTimingFlip = finished; } - current->timing->Print(current->outputHandle, current->index, current->flipArg, finished, interval); +#if APS5_ENABLE_TIMING_LOG + timingLog->Enqueue(current->timing->Capture(current->outputHandle, current->index, current->flipArg, finished, interval)); +#endif } if (current) { current.reset(); @@ -523,6 +557,15 @@ void VideoOutDriver::presentLoop(std::stop_token token) { std::fflush(stderr); std::terminate(); } +#if APS5_ENABLE_TIMING_LOG + try { + timingLog->Finish(); + } catch (const std::exception& error) { + std::fprintf(stderr, "[videoout] frame timing log failed: %s\n", error.what()); + std::fflush(stderr); + std::terminate(); + } +#endif current.reset(); { std::list> cancelled; @@ -541,9 +584,10 @@ void VideoOutDriver::vblankLoop(std::stop_token token) { try { for (int64_t frame = 1; !token.stop_requested(); ++frame) { const auto next = start + std::chrono::duration_cast(Frame(frame)); + const auto remaining = std::chrono::duration_cast(next - std::chrono::steady_clock::now()).count(); + if (remaining > 0) PreciseSleepUs(static_cast(remaining)); { - std::unique_lock lock(flipQueue->mutex); - flipQueue->changed.wait_until(lock, next, [&] { return token.stop_requested() || flipQueue->failure; }); + std::lock_guard lock(flipQueue->mutex); if (token.stop_requested() || flipQueue->failure) return; } vblankEnd(); diff --git a/core/libs/prx/libSceVideoOut/tests/Flip.cpp b/core/libs/prx/libSceVideoOut/tests/Flip.cpp index 048a7e892..ed59a5177 100644 --- a/core/libs/prx/libSceVideoOut/tests/Flip.cpp +++ b/core/libs/prx/libSceVideoOut/tests/Flip.cpp @@ -4,6 +4,7 @@ #include "prx/libSceVideoOut/include/Event.hpp" #include "prx/libkernel/Equeue/Equeue.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "prx/libc/include/Shutdown.hpp" #include @@ -257,13 +258,22 @@ void testControls() { } check(sceVideoOutWaitVblank(handle) == 0, "vblank wait failed"); check(owner->GetTriggeredEvents(&event, 1) == 1 && event.udata == &settings && sceVideoOutGetEventId(&event) == VIDEO_OUT_EVENT_VBLANK, "vblank event or updated user data missing"); + std::vector allocation(65536 + 65535); + const auto storage = alignedBuffer(allocation); + VideoOutBuffers buffer{storage.data(), nullptr, {allocation.data(), storage.data()}}; + VideoOutBufferAttribute2 attribute{}; + attribute.width = 64; + attribute.height = 64; + attribute.pixel_format = 0x8000000000000000ull; + check(sceVideoOutRegisterBuffers2(handle, 0, 0, &buffer, 1, &attribute, 0, nullptr) == 0, "buffers with set reserved pointers were rejected"); + check(sceVideoOutUnregisterBuffers(handle, 0) == 0, "buffer unregistration failed"); sceVideoOutClose(handle); check(owner->GetTriggeredEvents(&event, 1) == 0, "closed port retained pending events"); check(sceKernelDeleteEqueue(queue) == 0, "event queue deletion failed"); LibcRunShutdown_nid_postfix(); } -void testPresentation(bool expectUnavailable) { +void testPresentation() { const int handle = sceVideoOutOpen(255, 0, 0, nullptr); auto cfg = VideoOutDriver::Get().GetConfig(handle); std::vector allocation(6 * 65536 + 65535); @@ -289,16 +299,6 @@ void testPresentation(bool expectUnavailable) { sceVideoOutSubmitFlip(handle, index, 1, -123456789); std::unique_lock lock(cfg->mutex); check(cfg->vblankCond.wait_for(lock, std::chrono::seconds(15), [&] { return (cfg->failure && cfg->flipStatus.flipPendingNum == 0) || cfg->flipStatus.count == target; }), "presentation did not complete"); - if (expectUnavailable) { - check(cfg->failure != nullptr && cfg->flipStatus.count == 0 && cfg->bufferPending[0] == 0, "failed presentation was marked complete or retained its buffer"); - const auto error = cfg->failure; - lock.unlock(); - const auto message = expectFailure([&] { std::rethrow_exception(error); }); - check(message.find("required instance extension missing") != std::string::npos, "unexpected presentation failure"); - check(expectFailure([] { AgcDriverWaitIdle_nid_postfix(); }) == message, "AGC lost presentation failure"); - check(expectFailure([] { LibcRunShutdown_nid_postfix(); }) == message, "shutdown lost presentation failure"); - return; - } if (cfg->failure) std::rethrow_exception(cfg->failure); check(cfg->flipStatus.flipArg == -123456789 && cfg->flipStatus.currentBuffer == index && cfg->flipStatus.flipPendingNum == 0, "presentation status is wrong"); check(cfg->lastFlipVblank >= previousVblank + 3, "flip rate did not wait for its interval"); @@ -312,13 +312,20 @@ void testPresentation(bool expectUnavailable) { LibcRunShutdown_nid_postfix(); } +void testUnavailable() { + const auto message = expectFailure([] { sceVideoOutOpen(255, 0, 0, nullptr); }); + check(expectFailure([] { sceVideoOutOpen(255, 0, 0, nullptr); }) == message, "a second open lost the presentation failure"); + std::printf("presentation unavailable: %s\n", message.c_str()); + LibcRunShutdown_nid_postfix(); +} + void testCompressedPresentation() { const int handle = sceVideoOutOpen(255, 0, 0, nullptr); auto cfg = VideoOutDriver::Get().GetConfig(handle); std::vector allocation(6 * 65536 + 65535); const auto storage = alignedBuffer(allocation); fillBuffer(storage, 259, 137); - std::vector keys(storage.size() / 256); + std::vector keys(4096); VideoOutBuffers buffer{storage.data(), keys.data(), {nullptr, nullptr}}; VideoOutBufferAttribute2 attribute{}; sceVideoOutSetBufferAttribute2(&attribute, 0x8000000000000000ull, 0, 259, 137, 0, 0x208, 0xff102030); @@ -370,6 +377,49 @@ void testBackToBack() { LibcRunShutdown_nid_postfix(); } +std::uint32_t busyLoopResult(std::uint32_t iterations) { + std::uint32_t value = 1; + for (std::uint32_t i = 0; i < iterations; ++i) value = value * value + 1u; + return value; +} + +struct BusyLoop { + volatile std::uint32_t* result; + volatile std::uint32_t* label; + std::vector commands; +}; + +BusyLoop prepareBusyLoop(std::uint32_t iterations) { + alignas(256) static const std::array code{ + 0xbe850380, 0x7e040281, 0xd5690002, 0x00020502, 0x4a040481, 0x80058105, + 0xbf070405, 0xbf85fffa, 0x7e020280, 0xe0701000, 0x80000201, 0xbf810000}; + static Shader shader{}; + shader.file_header = 0x34333231; + shader.version = 0x18; + shader.header_size = sizeof(Shader); + shader.shader_size = sizeof(code); + shader.code = code.data(); + AgcDriverRegisterShader_nid_postfix(&shader); + std::int64_t physical = 0; + void* mapped = nullptr; + check(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, 65536, 65536, 0, &physical) == 0, "direct memory allocation failed"); + check(sceKernelMapDirectMemory(&mapped, 65536, 0x33, 0, physical, 65536) == 0, "direct memory mapping failed"); + auto* result = &static_cast(mapped)[0]; + auto* label = &static_cast(mapped)[64]; + *result = 0; + *label = 0; + const auto program = reinterpret_cast(code.data()); + const auto output = reinterpret_cast(result); + const auto target = reinterpret_cast(label); + return {result, label, { + 0xc0027600, 0x20c, static_cast(program >> 8u), static_cast(program >> 40u), + 0xc0017600, 0x213, 5u << 1u, + 0xc0037600, 0x207, 32, 1, 1, + 0xc0057600, 0x240, static_cast(output), static_cast(output >> 32u) & 0xffffu, 4, 0x31016fac, iterations, + 0xc0031500, 1, 1, 1, 0x8041, + 0xc0064900, 0x514, 1u << 29u, static_cast(target), static_cast(target >> 32u), 1, 0, 0}}; +} + void testReleaseVblank() { const int handle = sceVideoOutOpen(255, 0, 0, nullptr); auto cfg = VideoOutDriver::Get().GetConfig(handle); @@ -381,26 +431,22 @@ void testReleaseVblank() { sceVideoOutSetBufferAttribute2(&attribute, 0x8000000000000000ull, 0, 259, 137, 0, 0, 0); sceVideoOutRegisterBuffers2(handle, 0, 0, &buffer, 1, &attribute, 0, nullptr); check(sceVideoOutSetFlipRate(handle, 2) == 0, "flip rate was not applied"); + auto busy = prepareBusyLoop(100000000u); + Packet packet{busy.commands.data(), static_cast(busy.commands.size()), 0, {}}; + check(sceAgcDriverSubmitDcb(&packet) == 0, "the GPU work was not accepted"); sceVideoOutSubmitFlip(handle, 0, 1, 1); { std::unique_lock lock(cfg->mutex); const bool done = cfg->vblankCond.wait_for(lock, std::chrono::seconds(15), [&] { return cfg->failure || cfg->flipStatus.count == 1; }); if (cfg->failure) std::rethrow_exception(cfg->failure); check(done, "the flip did not complete"); - check(cfg->vblankStatus.count > 3, "the first presentation did not outlast the vblank it was released at"); - check(cfg->lastFlipVblank == 3, "the next flip is not paced from the vblank this flip was released at"); + check(cfg->lastFlipVblank >= 3 && cfg->vblankStatus.count >= cfg->lastFlipVblank + 3, "the next flip is not paced from the vblank this flip was released at"); } sceVideoOutUnregisterBuffers(handle, 0); sceVideoOutClose(handle); LibcRunShutdown_nid_postfix(); } -std::uint32_t busyLoopResult(std::uint32_t iterations) { - std::uint32_t value = 1; - for (std::uint32_t i = 0; i < iterations; ++i) value = value * value + 1u; - return value; -} - void testFlipAfterGpuWork() { const int handle = sceVideoOutOpen(255, 0, 0, nullptr); auto cfg = VideoOutDriver::Get().GetConfig(handle); @@ -411,36 +457,11 @@ void testFlipAfterGpuWork() { VideoOutBufferAttribute2 attribute{}; sceVideoOutSetBufferAttribute2(&attribute, 0x8000000000000000ull, 0, 259, 137, 0, 0, 0); sceVideoOutRegisterBuffers2(handle, 0, 0, &buffer, 1, &attribute, 0, nullptr); - alignas(256) static const std::array code{ - 0xbe850380, 0x7e040281, 0xd5690002, 0x00020502, 0x4a040481, 0x80058105, - 0xbf070405, 0xbf85fffa, 0x7e020280, 0xe0701000, 0x80000201, 0xbf810000}; - Shader shader{}; - shader.file_header = 0x34333231; - shader.version = 0x18; - shader.header_size = sizeof(Shader); - shader.shader_size = sizeof(code); - shader.code = code.data(); - AgcDriverRegisterShader_nid_postfix(&shader); - std::int64_t physical = 0; - void* mapped = nullptr; - check(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, 65536, 65536, 0, &physical) == 0, "direct memory allocation failed"); - check(sceKernelMapDirectMemory(&mapped, 65536, 0x33, 0, physical, 65536) == 0, "direct memory mapping failed"); - auto& result = static_cast(mapped)[0]; - auto& label = static_cast(mapped)[64]; - result = 0; - label = 0; constexpr std::uint32_t iterations = 100000000u; - const auto program = reinterpret_cast(code.data()); - const auto output = reinterpret_cast(&result); - const auto target = reinterpret_cast(&label); - std::vector commands{ - 0xc0027600, 0x20c, static_cast(program >> 8u), static_cast(program >> 40u), - 0xc0017600, 0x213, 5u << 1u, - 0xc0037600, 0x207, 32, 1, 1, - 0xc0057600, 0x240, static_cast(output), static_cast(output >> 32u) & 0xffffu, 4, 0x31016fac, iterations, - 0xc0031500, 1, 1, 1, 0x8041, - 0xc0064900, 0x514, 1u << 29u, static_cast(target), static_cast(target >> 32u), 1, 0, 0}; - Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + auto busy = prepareBusyLoop(iterations); + auto& result = *busy.result; + auto& label = *busy.label; + Packet packet{busy.commands.data(), static_cast(busy.commands.size()), 0, {}}; check(sceVideoOutSubmitFlip(handle, 0, 1, 4) == 0, "the first flip was not accepted"); { std::unique_lock lock(cfg->mutex); @@ -476,18 +497,50 @@ void testFlipAfterGpuWork() { LibcRunShutdown_nid_postfix(); } +void testOneDevice() { + const int handle = sceVideoOutOpen(255, 0, 0, nullptr); + auto cfg = VideoOutDriver::Get().GetConfig(handle); + std::vector allocation(6 * 65536 + 65535); + const auto storage = alignedBuffer(allocation); + fillBuffer(storage, 259, 137); + VideoOutBuffers buffer{storage.data(), nullptr, {nullptr, nullptr}}; + VideoOutBufferAttribute2 attribute{}; + sceVideoOutSetBufferAttribute2(&attribute, 0x8000000000000000ull, 0, 259, 137, 0, 0, 0); + sceVideoOutRegisterBuffers2(handle, 0, 0, &buffer, 1, &attribute, 0, nullptr); + constexpr std::uint32_t iterations = 1000u; + auto busy = prepareBusyLoop(iterations); + Packet packet{busy.commands.data(), static_cast(busy.commands.size()), 0, {}}; + check(sceAgcDriverSubmitDcb(&packet) == 0, "the GPU work was not accepted"); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (*busy.label != 1 && std::chrono::steady_clock::now() < deadline) std::this_thread::yield(); + check(*busy.label == 1 && *busy.result == busyLoopResult(iterations), "the GPU work did not run"); + check(sceVideoOutSubmitFlip(handle, 0, 1, 1) == 0, "the flip was not accepted"); + { + std::unique_lock lock(cfg->mutex); + const bool done = cfg->vblankCond.wait_for(lock, std::chrono::seconds(30), [&] { return cfg->failure || cfg->flipStatus.count == 1; }); + if (cfg->failure) std::rethrow_exception(cfg->failure); + check(done, "the flip did not complete"); + } + sceVideoOutUnregisterBuffers(handle, 0); + sceVideoOutClose(handle); + LibcRunShutdown_nid_postfix(); + const AgcDriver::VulkanDevice next; + check(next.Serial() == 2, "the GPU work and the flip used more than one device"); +} + } int main(int argc, char** argv) { try { if (argc == 2 && std::string(argv[1]) == "decode") testDecode(); else if (argc == 2 && std::string(argv[1]) == "controls") testControls(); - else if (argc == 2 && std::string(argv[1]) == "present") testPresentation(false); + else if (argc == 2 && std::string(argv[1]) == "present") testPresentation(); else if (argc == 2 && std::string(argv[1]) == "backtoback") testBackToBack(); else if (argc == 2 && std::string(argv[1]) == "dcc") testCompressedPresentation(); else if (argc == 2 && std::string(argv[1]) == "pacing") testReleaseVblank(); else if (argc == 2 && std::string(argv[1]) == "aftergpu") testFlipAfterGpuWork(); - else if (argc == 2 && std::string(argv[1]) == "unavailable") testPresentation(true); + else if (argc == 2 && std::string(argv[1]) == "unavailable") testUnavailable(); + else if (argc == 2 && std::string(argv[1]) == "onedevice") testOneDevice(); else testLifetime(argc == 2 && std::string(argv[1]) == "reopen"); std::puts("VideoOut flip tests passed"); return 0; diff --git a/core/libs/prx/libSceVideodec2/Export.cpp b/core/libs/prx/libSceVideodec2/Export.cpp index 28bea7762..a4d91f714 100644 --- a/core/libs/prx/libSceVideodec2/Export.cpp +++ b/core/libs/prx/libSceVideodec2/Export.cpp @@ -16,6 +16,7 @@ extern "C" { #include #include +#include #include } @@ -24,6 +25,9 @@ namespace { constexpr std::uint64_t WorkMemoryBytes = 1u << 20u; constexpr std::uint32_t PitchAlignment = 256; constexpr std::uint32_t HeightAlignment = 32; +constexpr std::uint32_t CodecAvc = 1; +constexpr std::uint32_t CodecHevc = 974921; +constexpr std::uint32_t CodecVp9 = 2382845; struct ComputeMemoryInfo { std::uint64_t thisSize; @@ -174,9 +178,18 @@ std::shared_ptr FindDecoder(std::uint64_t handle) { return found != decoders.end() ? found->second : nullptr; } +AVCodecID CodecId(std::uint32_t codec) { + switch (codec) { + case CodecAvc: return AV_CODEC_ID_H264; + case CodecHevc: return AV_CODEC_ID_HEVC; + case CodecVp9: return AV_CODEC_ID_VP9; + default: throw std::runtime_error("Videodec2: unsupported codec"); + } +} + void OpenCodec(Decoder& decoder) { - const AVCodec* codec = avcodec_find_decoder(AV_CODEC_ID_H264); - if (codec == nullptr) throw std::runtime_error("Videodec2: H.264 decoder is unavailable"); + const AVCodec* codec = avcodec_find_decoder(CodecId(decoder.codec)); + if (codec == nullptr) throw std::runtime_error("Videodec2: decoder is unavailable"); decoder.context = avcodec_alloc_context3(codec); if (decoder.context == nullptr) throw std::runtime_error("Videodec2: cannot allocate codec context"); decoder.context->err_recognition = AV_EF_EXPLODE; @@ -184,7 +197,7 @@ void OpenCodec(Decoder& decoder) { decoder.context->thread_type = FF_THREAD_SLICE; if (avcodec_open2(decoder.context, codec, nullptr) < 0) { avcodec_free_context(&decoder.context); - throw std::runtime_error("Videodec2: cannot open H.264 decoder"); + throw std::runtime_error("Videodec2: cannot open decoder"); } } @@ -233,6 +246,8 @@ bool WriteNv12(Decoder& decoder, const AVFrame& frame, std::uint8_t* target, std std::uint8_t* planes[4] = {target, target + luma, nullptr, nullptr}; const int strides[4] = {static_cast(pitch), static_cast(pitch), 0, 0}; const auto format = static_cast(frame.format); + const AVPixFmtDescriptor* descriptor = av_pix_fmt_desc_get(format); + if (descriptor == nullptr || descriptor->comp[0].depth > 8) throw std::runtime_error("Videodec2: output deeper than 8 bits is not implemented"); if (format == AV_PIX_FMT_YUV420P || format == AV_PIX_FMT_YUVJ420P) { for (std::uint32_t row = 0; row < height; ++row) std::memcpy(planes[0] + static_cast(row) * pitch, frame.data[0] + static_cast(row) * frame.linesize[0], width); for (std::uint32_t row = 0; row < height / 2; ++row) { @@ -329,14 +344,14 @@ int APS5_VABI sceVideodec2QueryDecoderMemoryInfo_nid_postfix(const DecoderConfig int APS5_VABI sceVideodec2CreateDecoder_nid_postfix(const DecoderConfigInfo* config, const DecoderMemoryInfo* memory, std::uint64_t* handle) { if (!config || !memory || !handle) throw std::runtime_error("Videodec2: invalid argument pointer"); - if (config->codecType != 1) throw std::runtime_error("Videodec2: unsupported codec"); + CodecId(config->codecType); auto decoder = std::make_shared(); decoder->codec = config->codecType; OpenCodec(*decoder); std::lock_guard guard(lock); *handle = nextDecoder++; decoders.emplace(*handle, decoder); - std::fprintf(stderr, "[videodec2] decoder %llu: codec=%u max %dx%d (%s)\n", static_cast(*handle), config->codecType, config->maxFrameWidth, config->maxFrameHeight, "H.264 through FFmpeg"); + std::fprintf(stderr, "[videodec2] decoder %llu: codec=%u max %dx%d (%s)\n", static_cast(*handle), config->codecType, config->maxFrameWidth, config->maxFrameHeight, decoder->context->codec->long_name); return 0; } @@ -357,7 +372,8 @@ int APS5_VABI sceVideodec2Decode_nid_postfix(std::uint64_t handle, const InputDa const Picture picture{input->ptsData, input->dtsData, input->attachedData}; if (decoder.flushing) throw std::runtime_error("Videodec2: reset required after flushing"); if (!input->auData || input->auSize == 0 || input->auSize > static_cast(std::numeric_limits::max())) throw std::runtime_error("Videodec2: invalid access unit"); - const auto* unit = ToAnnexB(decoder, input->auData, static_cast(input->auSize)); + if (decoder.codec == CodecVp9) decoder.annexB.assign(input->auData, input->auData + input->auSize); + const auto* unit = decoder.codec == CodecVp9 ? &decoder.annexB : ToAnnexB(decoder, input->auData, static_cast(input->auSize)); AVPacket* packet = av_packet_alloc(); if (!packet) throw std::runtime_error("Videodec2: cannot allocate packet"); const auto freePacket = [](AVPacket* value) { av_packet_free(&value); }; @@ -410,6 +426,7 @@ int APS5_VABI sceVideodec2GetPictureInfo_nid_postfix(const OutputInfo* output, A std::lock_guard guard(decoder->mutex); const auto found = decoder->pictures.find(output->frameBuffer); if (found == decoder->pictures.end()) continue; + if (decoder->codec != CodecAvc) throw std::runtime_error("Videodec2: picture info is only implemented for H.264"); const auto& picture = found->second; first->isValid = true; first->ptsData = picture.pts; diff --git a/core/libs/prx/libSceVoiceChat/Export.cpp b/core/libs/prx/libSceVoiceChat/Export.cpp index 4fbb609c9..03569b489 100644 --- a/core/libs/prx/libSceVoiceChat/Export.cpp +++ b/core/libs/prx/libSceVoiceChat/Export.cpp @@ -61,4 +61,8 @@ VOICECHAT_OFFLINE_REQUEST(sceVoiceChatRequestCreatePlayerSessionVoiceChatChannel VOICECHAT_OFFLINE_REQUEST(sceVoiceChatRequestDeletePlayerSessionVoiceChatChannel) VOICECHAT_OFFLINE_REQUEST(sceVoiceChatRequestJoinPlayerSessionVoiceChatChannel) VOICECHAT_OFFLINE_REQUEST(sceVoiceChatRequestLeavePlayerSessionVoiceChatChannel) +VOICECHAT_OFFLINE_REQUEST(sceVoiceChatRequestCreateVoiceChatGroup) +VOICECHAT_OFFLINE_REQUEST(sceVoiceChatRequestJoinVoiceChatGroup) +VOICECHAT_OFFLINE_REQUEST(sceVoiceChatRequestLeaveVoiceChatGroup) +VOICECHAT_OFFLINE_REQUEST(sceVoiceChatRequestDeleteVoiceChatGroup) } diff --git a/core/libs/prx/libSceVoiceQoS/Export.cpp b/core/libs/prx/libSceVoiceQoS/Export.cpp index 0aa4ef2cb..4ab9fb411 100644 --- a/core/libs/prx/libSceVoiceQoS/Export.cpp +++ b/core/libs/prx/libSceVoiceQoS/Export.cpp @@ -1,8 +1,6 @@ #include #include #include -#include -#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" @@ -10,6 +8,9 @@ namespace { constexpr std::int32_t VOICE_QOS_APP_TYPE_GAME = 0x20000000; constexpr std::int32_t VOICE_QOS_APP_TYPE_10000000 = 0x10000000; +constexpr std::uint32_t VOICE_QOS_MEMORY_SIZE = 0x40000; +constexpr int SCE_VOICE_ERROR_LIBVOICEQOS_ARGUMENT_INVALID = static_cast(0x804E0902); +constexpr int SCE_VOICE_ERROR_LIBVOICEQOS_INITIALIZED = static_cast(0x804E0905); std::mutex g_mutex; bool g_initialized = false; @@ -19,10 +20,9 @@ bool g_initialized = false; extern "C" { int APS5_VABI sceVoiceQoSInit(void* mem_block, uint32_t mem_size, int32_t app_type) { - if (!mem_block || mem_size == 0) throw std::invalid_argument("sceVoiceQoSInit: null or empty memory block"); - if (app_type != VOICE_QOS_APP_TYPE_GAME && app_type != VOICE_QOS_APP_TYPE_10000000) throw std::invalid_argument("sceVoiceQoSInit: unsupported app type " + std::to_string(app_type)); std::lock_guard lock(g_mutex); - if (g_initialized) throw std::runtime_error("sceVoiceQoSInit: already initialized"); + if (g_initialized) return SCE_VOICE_ERROR_LIBVOICEQOS_INITIALIZED; + if (!mem_block || mem_size != VOICE_QOS_MEMORY_SIZE || (app_type != VOICE_QOS_APP_TYPE_GAME && app_type != VOICE_QOS_APP_TYPE_10000000)) return SCE_VOICE_ERROR_LIBVOICEQOS_ARGUMENT_INVALID; g_initialized = true; return 0; } diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index c59665ab8..30aaf71ba 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -4,6 +4,7 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/specifics/gcc/StackGuard.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/specifics/windows/CrashReport.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/specifics/linux/DlIteratePhdr.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/specifics/macos/GuestImages.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Strings.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/AllocatingStrings.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/WideConversion.cpp @@ -45,6 +46,16 @@ if(WIN32) endif() if(BUILD_TESTING) + add_test_executable(guest_time_conversion_tests ${LIBS_INCLUDE_DIR}/tests/GuestTimeConversion.cpp) + target_include_directories(guest_time_conversion_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_time_conversion_tests PRIVATE libc) + configure_windows_unwind(guest_time_conversion_tests) + add_test(NAME guest_time_conversion COMMAND guest_time_conversion_tests) + set_tests_properties(guest_time_conversion PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_time_conversion PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_scan_count_tests ${LIBS_INCLUDE_DIR}/tests/GuestScanCount.cpp) target_include_directories(guest_scan_count_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_scan_count_tests PRIVATE libc) @@ -82,6 +93,16 @@ if(BUILD_TESTING) set_tests_properties(guest_wide_formatting PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + + add_test_executable(guest_wide_string_tests ${LIBS_INCLUDE_DIR}/tests/GuestWideStrings.cpp) + target_include_directories(guest_wide_string_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_wide_string_tests PRIVATE libc) + configure_windows_unwind(guest_wide_string_tests) + add_test(NAME guest_wide_strings COMMAND guest_wide_string_tests) + if(WIN32) + set_tests_properties(guest_wide_strings PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_character_tests ${LIBS_INCLUDE_DIR}/tests/GuestCharacter.cpp) target_include_directories(guest_character_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_character_tests PRIVATE libc) @@ -139,6 +160,15 @@ if(BUILD_TESTING) set_tests_properties(guest_working_directory PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_path_case_tests ${LIBS_INCLUDE_DIR}/tests/GuestPathCase.cpp) + target_include_directories(guest_path_case_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_path_case_tests PRIVATE libc libkernel) + configure_windows_unwind(guest_path_case_tests) + add_test(NAME guest_path_case COMMAND guest_path_case_tests) + if(WIN32) + set_tests_properties(guest_path_case PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_directory_tests ${LIBS_INCLUDE_DIR}/tests/GuestDirectory.cpp) target_include_directories(guest_directory_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_directory_tests PRIVATE libc) @@ -166,6 +196,16 @@ if(BUILD_TESTING) set_tests_properties(guest_lifecycle PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_thread_callback_abi_tests ${LIBS_INCLUDE_DIR}/tests/GuestThreadCallbackAbi.cpp) + target_include_directories(guest_thread_callback_abi_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_thread_callback_abi_tests PRIVATE libc) + configure_windows_unwind(guest_thread_callback_abi_tests) + add_test(NAME guest_thread_callback_abi COMMAND guest_thread_callback_abi_tests) + set_tests_properties(guest_thread_callback_abi PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_thread_callback_abi PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_reallocalign_tests ${LIBS_INCLUDE_DIR}/tests/GuestReallocalign.cpp) target_include_directories(guest_reallocalign_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_reallocalign_tests PRIVATE libc) @@ -206,8 +246,8 @@ if(BUILD_TESTING) target_include_directories(guest_formatting_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_formatting_tests PRIVATE libc) configure_windows_unwind(guest_formatting_tests) + add_test(NAME guest_formatting COMMAND guest_formatting_tests) if(WIN32) - add_test(NAME guest_formatting COMMAND guest_formatting_tests) set_tests_properties(guest_formatting PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() @@ -220,6 +260,15 @@ if(BUILD_TESTING) set_tests_properties(guest_search PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_demangle_tests ${LIBS_INCLUDE_DIR}/tests/GuestDemangle.cpp) + target_include_directories(guest_demangle_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_demangle_tests PRIVATE libc) + configure_windows_unwind(guest_demangle_tests) + add_test(NAME guest_demangle COMMAND guest_demangle_tests) + if(WIN32) + set_tests_properties(guest_demangle PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_string_tests ${LIBS_INCLUDE_DIR}/tests/GuestStrings.cpp) target_include_directories(guest_string_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_string_tests PRIVATE libc libScePosixForWebKit) @@ -243,11 +292,21 @@ if(BUILD_TESTING) target_include_directories(exception_runtime_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(exception_runtime_tests PRIVATE libc) add_test(NAME exception_runtime COMMAND exception_runtime_tests) - enable_language(C) - add_test_executable(exception_c_cleanup_tests ${LIBS_INCLUDE_DIR}/tests/ExceptionCCleanup.cpp ${LIBS_INCLUDE_DIR}/tests/ExceptionCCleanup.c) - set_source_files_properties(${LIBS_INCLUDE_DIR}/tests/ExceptionCCleanup.c PROPERTIES COMPILE_OPTIONS -fexceptions) - target_link_libraries(exception_c_cleanup_tests PRIVATE libc) - add_test(NAME exception_c_cleanup COMMAND exception_c_cleanup_tests) + add_test_executable(exception_personality_tests ${LIBS_INCLUDE_DIR}/tests/ExceptionPersonality.cpp) + target_include_directories(exception_personality_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(exception_personality_tests PRIVATE libc) + add_test(NAME exception_personality COMMAND exception_personality_tests) + if(NOT APPLE) + enable_language(C) + add_test_executable(exception_c_cleanup_tests ${LIBS_INCLUDE_DIR}/tests/ExceptionCCleanup.cpp ${LIBS_INCLUDE_DIR}/tests/ExceptionCCleanup.c) + set_source_files_properties(${LIBS_INCLUDE_DIR}/tests/ExceptionCCleanup.c PROPERTIES COMPILE_OPTIONS -fexceptions) + target_link_libraries(exception_c_cleanup_tests PRIVATE libc) + add_test(NAME exception_c_cleanup COMMAND exception_c_cleanup_tests) + add_test_executable(guest_image_anonymous_tests ${LIBS_INCLUDE_DIR}/tests/GuestImageAnonymous.cpp) + target_include_directories(guest_image_anonymous_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_image_anonymous_tests PRIVATE libc) + add_test(NAME guest_image_anonymous COMMAND guest_image_anonymous_tests) + endif() endif() endif() @@ -279,6 +338,25 @@ if(BUILD_TESTING) set_tests_properties(application_heap_replacement application_heap_invalid_metadata application_heap_default application_heap_exit_order PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_test_executable(cpu_topology_tests ${LIBS_INCLUDE_DIR}/tests/CpuTopology.cpp) + target_include_directories(cpu_topology_tests PRIVATE ${LIBS_INCLUDE_DIR}) + add_test(NAME cpu_topology COMMAND cpu_topology_tests) + endif() +endif() + +if(WIN32) + add_test_executable(guest_heap_rounding_tests EXCLUDE_FROM_ALL ${LIBS_INCLUDE_DIR}/tests/GuestHeapRounding.cpp) + target_include_directories(guest_heap_rounding_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_heap_rounding_tests PRIVATE libc) + configure_windows_unwind(guest_heap_rounding_tests) + set_target_properties(guest_heap_rounding_tests PROPERTIES CXX_EXTENSIONS OFF) + if(BUILD_TESTING) + set_target_properties(guest_heap_rounding_tests PROPERTIES EXCLUDE_FROM_ALL FALSE) + add_test(NAME guest_heap_rounding_overflow COMMAND guest_heap_rounding_tests) + set_tests_properties(guest_heap_rounding_overflow PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() add_test_executable(guest_locale_tests EXCLUDE_FROM_ALL ${LIBS_INCLUDE_DIR}/tests/GuestLocale.cpp) @@ -302,4 +380,9 @@ if(MINGW) set_target_properties(windows_exception_tests PROPERTIES CXX_EXTENSIONS OFF) target_link_libraries(windows_exception_tests PRIVATE libc) configure_windows_unwind(windows_exception_tests) + if(BUILD_TESTING) + set_target_properties(windows_exception_tests PROPERTIES EXCLUDE_FROM_ALL FALSE) + add_test(NAME windows_exceptions COMMAND windows_exception_tests) + set_tests_properties(windows_exceptions PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() endif() diff --git a/core/libs/prx/libc/Export.cpp b/core/libs/prx/libc/Export.cpp index ebbb25bea..df3cffffd 100644 --- a/core/libs/prx/libc/Export.cpp +++ b/core/libs/prx/libc/Export.cpp @@ -18,20 +18,9 @@ extern "C" { LibcHeapTraceInfo_nid_no_patch(info); } - int APS5_VABI LibcHeapErrorReportForGame_nid_postfix( - uint64_t msp, uint64_t ptr, uint64_t error, - uint64_t arg3, uint64_t arg4, uint64_t arg5 - ) { - (void)msp; (void)ptr; (void)error; - (void)arg3; (void)arg4; (void)arg5; - NotImplemented_nid_no_patch(__func__); - return 0; - } - // Dead import of Cyberpunk 2077 (PPSA04029): no call sites, but the // Windows loader resolves imports strictly, so it must be present. -APS5_EXPORT("u2tMGOLaqnE", libcUnknown_u2tMGOLaqnE); -int APS5_VABI libcUnknown_u2tMGOLaqnE() { +int APS5_VABI _ZSt14_Atomic_assertPKcS0__nid_postfix() { NotImplemented_nid_no_patch(__func__); return 0; } diff --git a/core/libs/prx/libc/include/CpuTopology.hpp b/core/libs/prx/libc/include/CpuTopology.hpp index 3150ed400..a1061cb3c 100644 --- a/core/libs/prx/libc/include/CpuTopology.hpp +++ b/core/libs/prx/libc/include/CpuTopology.hpp @@ -8,6 +8,12 @@ #ifdef _WIN32 #include +#elif defined(__linux__) +#include +#include +#include +#include +#include #endif // Hybrid CPU layout (Intel P/E cores) from the OS: which logical processors belong to the lowest and @@ -22,6 +28,32 @@ struct Layout { bool hybrid = false; }; +#ifdef __linux__ +inline std::uint64_t MaskFromCpuList(const char* path) { + std::ifstream file(path); + std::string text; + if (!std::getline(file, text)) return 0; + std::uint64_t mask = 0; + const char* cursor = text.c_str(); + while (*cursor != '\0') { + char* end = nullptr; + const unsigned long first = std::strtoul(cursor, &end, 10); + if (end == cursor) return 0; + unsigned long last = first; + cursor = end; + if (*cursor == '-') { + last = std::strtoul(cursor + 1, &end, 10); + if (end == cursor + 1 || last < first) return 0; + cursor = end; + } + for (unsigned long cpu = first; cpu <= last && cpu < 64; ++cpu) mask |= std::uint64_t{1} << cpu; + if (*cursor == ',') ++cursor; + else if (*cursor != '\0') return 0; + } + return mask; +} +#endif + inline const Layout& Get() { static const Layout layout = [] { Layout result{}; @@ -57,6 +89,16 @@ inline const Layout& Get() { result.efficient = byClass[lowest]; result.performant = byClass[highest]; result.hybrid = result.efficient != 0 && result.performant != 0; +#elif defined(__linux__) + cpu_set_t set; + CPU_ZERO(&set); + if (sched_getaffinity(0, sizeof(set), &set) == 0) { + for (unsigned cpu = 0; cpu < 64; ++cpu) + if (CPU_ISSET(cpu, &set)) result.process |= std::uint64_t{1} << cpu; + } + result.performant = MaskFromCpuList("/sys/devices/cpu_core/cpus"); + result.efficient = MaskFromCpuList("/sys/devices/cpu_atom/cpus"); + result.hybrid = result.efficient != 0 && result.performant != 0 && (result.efficient & result.performant) == 0; #endif return result; }(); @@ -78,6 +120,8 @@ inline bool Trace() { inline unsigned long ThreadId(void* handle) { #ifdef _WIN32 return handle != nullptr ? GetThreadId(static_cast(handle)) : GetCurrentThreadId(); +#elif defined(__linux__) + return handle == nullptr ? static_cast(gettid()) : 0; #else (void)handle; return 0; @@ -93,6 +137,15 @@ inline std::uint64_t Pin(void* handle, std::uint64_t mask) { if (mask == 0) return 0; HANDLE thread = handle != nullptr ? static_cast(handle) : GetCurrentThread(); return SetThreadAffinityMask(thread, static_cast(mask)) != 0 ? mask : 0; +#elif defined(__linux__) + const auto& layout = Get(); + if (layout.process != 0) mask &= layout.process; + if (mask == 0 || handle != nullptr) return 0; + cpu_set_t set; + CPU_ZERO(&set); + for (unsigned cpu = 0; cpu < 64; ++cpu) + if ((mask >> cpu) & 1u) CPU_SET(cpu, &set); + return pthread_setaffinity_np(pthread_self(), sizeof(set), &set) == 0 ? mask : 0; #else (void)handle; (void)mask; diff --git a/core/libs/prx/libc/include/General.hpp b/core/libs/prx/libc/include/General.hpp index 29ed9fda5..3aa9dcbc6 100644 --- a/core/libs/prx/libc/include/General.hpp +++ b/core/libs/prx/libc/include/General.hpp @@ -3,6 +3,7 @@ #include #include +#include #include "general/LogMacros.hpp" #include "general/VabiMacros.hpp" @@ -16,6 +17,10 @@ extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path); // directory instead of the run directory; the prefix matches whole path components only. extern "C" void AddPathAlias_nid_no_patch(const char* guestPrefix, const char* hostPath); extern "C" void RemovePathAlias_nid_no_patch(const char* guestPrefix); +extern "C" void RecordWrittenPath_nid_no_patch(const std::filesystem::path& path); +extern "C" std::vector WrittenPaths_nid_no_patch(); +extern "C" void SyncWrittenPaths_nid_no_patch(); +extern "C" void BlockPathAlias_nid_no_patch(const char* guestPrefix); #define APS5_INVALID_ARG_EX throw std::invalid_argument(std::string(__func__) + ": invalid argument") diff --git a/core/libs/prx/libc/include/GuestArena.hpp b/core/libs/prx/libc/include/GuestArena.hpp index 6a54c7980..0cbeac68d 100644 --- a/core/libs/prx/libc/include/GuestArena.hpp +++ b/core/libs/prx/libc/include/GuestArena.hpp @@ -31,6 +31,8 @@ void GuestArenaUnpinWritable_nid_postfix(const void* pointer, std::size_t bytes) bool GuestArenaProtection_nid_postfix(std::uintptr_t address, std::uint32_t* protection); bool GuestArenaCollectWrites_nid_postfix(std::uintptr_t address, std::size_t bytes, void** pages, std::size_t* count, bool clear); bool GuestArenaHostRegionOverlaps_nid_postfix(std::uintptr_t address, std::size_t bytes); +std::uint64_t GuestArenaCommitGeneration_nid_postfix(); +void GuestArenaSetPrivateMappingObserver_nid_postfix(void (*callback)(std::uintptr_t address, std::size_t bytes, std::uint64_t generation)); void GuestArenaCommit_nid_postfix(void* pointer, std::size_t bytes, std::uint32_t protection, std::size_t granule); void GuestArenaReset_nid_postfix(void* pointer, std::size_t bytes); void GuestArenaMap_nid_postfix(void* pointer, std::size_t bytes, void* section, std::uint64_t offset, std::uint32_t protection); diff --git a/core/libs/prx/libc/include/VarArgsAbi.hpp b/core/libs/prx/libc/include/VarArgsAbi.hpp deleted file mode 100644 index dc2d0e87b..000000000 --- a/core/libs/prx/libc/include/VarArgsAbi.hpp +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_EXCEPTIONS_VARARGSABI_HPP -#define CORE_LIBS_PRX_LIBC_INCLUDE_EXCEPTIONS_VARARGSABI_HPP - -#include -#include -#include "SceTypes.hpp" -#include "prx/libc/include/specifics/x86_64/SimdTypes.hpp" - -namespace LibcDetail { - -struct RegSaveArea { - std::uint64_t gp[6]; - X86_64::Xmm fp[8]; -}; - -struct VaListLayout { - unsigned int gpOffset; - unsigned int fpOffset; - void* overflowArgArea; - void* regSaveArea; -}; - -inline void FillRegSaveArea( - RegSaveArea& regs, - std::uint64_t gp0, std::uint64_t gp1, std::uint64_t gp2, - std::uint64_t gp3, std::uint64_t gp4, std::uint64_t gp5, - X86_64::Xmm fp0, X86_64::Xmm fp1, X86_64::Xmm fp2, X86_64::Xmm fp3, - X86_64::Xmm fp4, X86_64::Xmm fp5, X86_64::Xmm fp6, X86_64::Xmm fp7 -) { - regs.gp[0] = gp0; - regs.gp[1] = gp1; - regs.gp[2] = gp2; - regs.gp[3] = gp3; - regs.gp[4] = gp4; - regs.gp[5] = gp5; - regs.fp[0] = fp0; - regs.fp[1] = fp1; - regs.fp[2] = fp2; - regs.fp[3] = fp3; - regs.fp[4] = fp4; - regs.fp[5] = fp5; - regs.fp[6] = fp6; - regs.fp[7] = fp7; -} - -inline std::va_list* BuildVaList( - VaListLayout& layout, RegSaveArea& regs, - unsigned int consumedGpRegisters, void* overflowArgArea -) { - layout.gpOffset = consumedGpRegisters * 8u; - layout.fpOffset = 6u * 8u; - layout.overflowArgArea = overflowArgArea; - layout.regSaveArea = ®s; - return reinterpret_cast(&layout); -} - -} - -#endif diff --git a/core/libs/prx/libc/include/WindowsMappings.hpp b/core/libs/prx/libc/include/WindowsMappings.hpp index 200e8e3f8..3c423d148 100644 --- a/core/libs/prx/libc/include/WindowsMappings.hpp +++ b/core/libs/prx/libc/include/WindowsMappings.hpp @@ -28,7 +28,8 @@ public: return allocate(GetCurrentProcess(), address, bytes, MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, nullptr, 0); } - void Commit(void* address, std::size_t bytes, DWORD protection, std::size_t granule, bool watched) { + std::vector> Commit(void* address, std::size_t bytes, DWORD protection, std::size_t granule, bool watched) { + std::vector> created; std::lock_guard lock(mutex); const auto end = reinterpret_cast(address) + bytes; for (auto cursor = reinterpret_cast(address); cursor < end;) { @@ -46,6 +47,8 @@ public: reset(cursor, size); const DWORD flags = MEM_RESERVE | MEM_COMMIT | MEM_REPLACE_PLACEHOLDER | (watched ? MEM_WRITE_WATCH : 0); if (!allocate(GetCurrentProcess(), reinterpret_cast(cursor), size, flags, protection, nullptr, 0)) fail("replace guest placeholder with private memory"); + if (!created.empty() && created.back().first + created.back().second == cursor) created.back().second += size; + else created.emplace_back(cursor, size); cursor += size; } else { if (memory.State != MEM_COMMIT) throw std::runtime_error("guest memory is not committed"); @@ -60,6 +63,7 @@ public: cursor = stop; } } + return created; } void Reset(void* address, std::size_t bytes) { diff --git a/core/libs/prx/libc/include/exceptions/Unwind.hpp b/core/libs/prx/libc/include/exceptions/Unwind.hpp index 217d9a4ee..07d03fd3f 100644 --- a/core/libs/prx/libc/include/exceptions/Unwind.hpp +++ b/core/libs/prx/libc/include/exceptions/Unwind.hpp @@ -99,6 +99,7 @@ inline unsigned EncodingSize(Byte encoding) { extern "C" { _Unwind_Reason_Code APS5_VABI __gxx_personality_v0_nid_postfix(int, _Unwind_Action, std::uint64_t, _Unwind_Exception*, _Unwind_Context*); + _Unwind_Reason_Code APS5_VABI __gcc_personality_v0_nid_postfix(int, _Unwind_Action, std::uint64_t, _Unwind_Exception*, _Unwind_Context*); _Unwind_Reason_Code APS5_VABI _Unwind_RaiseException_nid_postfix(_Unwind_Exception*); [[noreturn]] void APS5_VABI _Unwind_Resume_nid_postfix(_Unwind_Exception*); _Unwind_Reason_Code APS5_VABI _Unwind_Resume_or_Rethrow_nid_postfix(_Unwind_Exception*); diff --git a/core/libs/prx/libc/include/specifics/itanium/CxxAbi.hpp b/core/libs/prx/libc/include/specifics/itanium/CxxAbi.hpp index 7c68c9e16..53d45d895 100644 --- a/core/libs/prx/libc/include/specifics/itanium/CxxAbi.hpp +++ b/core/libs/prx/libc/include/specifics/itanium/CxxAbi.hpp @@ -3,4 +3,51 @@ #include +#ifdef _LIBCPPABI_VERSION +#include + +namespace __cxxabiv1 { +class __class_type_info : public std::type_info { +public: + enum __sub_kind { __unknown = 0 }; + struct __upcast_result; + struct __dyncast_result; +}; + +class __si_class_type_info : public __class_type_info { +public: + const __class_type_info* __base_type; +}; + +struct __base_class_type_info { + const __class_type_info* __base_type; + long __offset_flags; +}; + +class __vmi_class_type_info : public __class_type_info { +public: + unsigned int __flags; + unsigned int __base_count; + __base_class_type_info __base_info[1]; +}; + +class __pbase_type_info : public std::type_info { +public: + unsigned int __flags; + const std::type_info* __pointee; +}; + +class __pointer_type_info : public __pbase_type_info {}; + +class __pointer_to_member_type_info : public __pbase_type_info { +public: + const __class_type_info* __context; +}; + +#ifdef __APPLE__ +extern "C" int __cxa_thread_atexit(void (*)(void*), void*, void*) noexcept; +#endif +} +#endif + #endif diff --git a/core/libs/prx/libc/include/specifics/linux/ElfTypes.hpp b/core/libs/prx/libc/include/specifics/linux/ElfTypes.hpp index 5263b92c4..89dd7d3bc 100644 --- a/core/libs/prx/libc/include/specifics/linux/ElfTypes.hpp +++ b/core/libs/prx/libc/include/specifics/linux/ElfTypes.hpp @@ -27,6 +27,10 @@ static constexpr std::uint32_t PT_GNU_EH_FRAME = 0x6474e550; static constexpr std::uint32_t PF_X = 1; static constexpr std::uint32_t PF_W = 2; +#ifdef __APPLE__ +extern "C" int dl_iterate_phdr(int (*callback)(dl_phdr_info*, std::size_t, void*), void* data) __asm__("_dl_iterate_phdr_nid_no_patch"); +#else extern "C" int dl_iterate_phdr(int (*callback)(dl_phdr_info*, std::size_t, void*), void* data); +#endif #endif diff --git a/core/libs/prx/libc/include/specifics/x86_64/SimdTypes.hpp b/core/libs/prx/libc/include/specifics/x86_64/SimdTypes.hpp deleted file mode 100644 index b17de52e8..000000000 --- a/core/libs/prx/libc/include/specifics/x86_64/SimdTypes.hpp +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_X86_64_SIMDTYPES_HPP -#define CORE_LIBS_PRX_LIBC_INCLUDE_X86_64_SIMDTYPES_HPP - -#include - -namespace X86_64 { -using Xmm = __m128; -} - -#endif diff --git a/core/libs/prx/libc/src/CxxAbiSupport.cpp b/core/libs/prx/libc/src/CxxAbiSupport.cpp index 008cc422e..c8e7e0801 100644 --- a/core/libs/prx/libc/src/CxxAbiSupport.cpp +++ b/core/libs/prx/libc/src/CxxAbiSupport.cpp @@ -4,7 +4,7 @@ #define _UNWIND_H #endif -#include +#include "prx/libc/include/specifics/itanium/CxxAbi.hpp" #include #include #include @@ -18,20 +18,72 @@ #include #include #include +#include #include "prx/libc/include/General.hpp" +#include "prx/libc/include/ApplicationHeap.hpp" extern "C" { void* APS5_VABI __cxa_demangle_nid_postfix(const char* mangled, char* buf, std::size_t* len, int* status) { - return abi::__cxa_demangle(mangled, buf, len, status); + if (mangled == nullptr || (buf != nullptr && len == nullptr)) { + if (status != nullptr) *status = -3; + return nullptr; + } + + int nativeStatus = 0; + std::unique_ptr nativeResult( + abi::__cxa_demangle(mangled, nullptr, nullptr, &nativeStatus), &std::free); + if (!nativeResult) { + if (status != nullptr) *status = nativeStatus; + return nullptr; + } + + const std::size_t required = std::strlen(nativeResult.get()) + 1; + char* result = buf; + bool updateLength = false; + try { + if (result == nullptr) { + result = static_cast(ApplicationHeapAllocate_nid_no_patch(required)); + updateLength = len != nullptr; + } else if (*len < required) { + result = static_cast(ApplicationHeapReallocate_nid_no_patch(result, required)); + updateLength = true; + } + } catch (const std::bad_alloc&) { + if (status != nullptr) *status = -1; + return nullptr; + } + + std::memcpy(result, nativeResult.get(), required); + if (updateLength) *len = required; + if (status != nullptr) *status = 0; + return result; } -int APS5_VABI __cxa_thread_atexit_impl_nid_postfix(void (*func)(void*), void* arg, void* dso) { - return __cxxabiv1::__cxa_thread_atexit(func, arg, dso); +int APS5_VABI __cxa_thread_atexit_impl_nid_postfix(void (APS5_VABI *func)(void*), void* arg, void* dso) { + struct ThreadAtexitContext { + void (APS5_VABI *destructor)(void*); + void* object; + }; + auto* context = new (std::nothrow) ThreadAtexitContext{func, arg}; + if (context == nullptr) + return -1; + const int result = __cxxabiv1::__cxa_thread_atexit( + [](void* opaque) { + auto* context = static_cast(opaque); + const auto destructor = context->destructor; + void* object = context->object; + delete context; + destructor(object); + }, + context, dso); + if (result != 0) + delete context; + return result; } -int APS5_VABI LibcInternalExtCxaThreadAtexit_nid_postfix(void (*destructor)(void*), void* object, void* module_id) { +int APS5_VABI LibcInternalExtCxaThreadAtexit_nid_postfix(void (APS5_VABI *destructor)(void*), void* object, void* module_id) { #ifdef _WIN32 (void)module_id; return __cxa_thread_atexit_impl_nid_postfix(destructor, object, nullptr); diff --git a/core/libs/prx/libc/src/FileAndHeap.cpp b/core/libs/prx/libc/src/FileAndHeap.cpp index c6a5de95d..7c15cffde 100644 --- a/core/libs/prx/libc/src/FileAndHeap.cpp +++ b/core/libs/prx/libc/src/FileAndHeap.cpp @@ -7,10 +7,12 @@ #include #include #include +#include #include "prx/libc/include/FileStream.hpp" #include "prx/libc/include/ApplicationHeap.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libc/include/GuestArena.hpp" static std::string NativeFileMode(const char* mode) { std::string result(mode); @@ -20,6 +22,10 @@ static std::string NativeFileMode(const char* mode) { return result; } +static bool WritesFile(const char* mode) { + return std::strpbrk(mode, "wa+") != nullptr; +} + extern "C" { [[noreturn]] void APS5_VABI _ZSt11_Xbad_allocv_nid_postfix(); @@ -57,7 +63,10 @@ FileStream* APS5_VABI freopen_nid_postfix(const char* filename, const char* mode if (!valid) { errno = 22; return nullptr; } try { const auto path = *filename ? ResolvePath_nid_no_patch(filename).string() : std::string{}; - if (stream->Reopen(path.c_str(), NativeFileMode(mode).c_str())) return stream; + if (stream->Reopen(path.c_str(), NativeFileMode(mode).c_str())) { + if (!path.empty() && WritesFile(mode)) RecordWrittenPath_nid_no_patch(path); + return stream; + } const int error = errno; if (stream->IsDynamic()) delete stream; errno = error; @@ -94,6 +103,7 @@ FileStream* APS5_VABI fopen_nid_postfix(const char* filename, const char* mode) throw std::runtime_error(std::string(__func__) + ": " + FOPEN_MSG_OPEN_FAILED + ": \"" + abs_path + "\": " + reason); } // APS5_LOG_OUT("success: \"%s\"", abs_path.c_str()); + if (WritesFile(mode)) RecordWrittenPath_nid_no_patch(fpath); auto stream = std::make_unique(handle.get(), true); handle.release(); return stream.release(); @@ -145,6 +155,11 @@ size_t APS5_VABI fread_nid_postfix(void* buffer, size_t size, size_t count, File auto* handle = GetNativeStream(stream); if (size == 0 || count == 0) return 0; if (!buffer) throw std::runtime_error("fread: null buffer"); + if (count > std::numeric_limits::max() / size) throw std::overflow_error("fread: buffer size overflow"); + const auto bytes = size * count; + if (bytes > std::numeric_limits::max() - reinterpret_cast(buffer)) throw std::overflow_error("fread: buffer address overflow"); + const GuestArena::HostWrite destination(buffer, bytes); + if (!destination.Open()) throw std::runtime_error("fread: buffer is not writable"); const auto result = std::fread(buffer, size, count, handle); stream->SyncStatus(); if (std::ferror(handle)) throw std::runtime_error("fread: read failed"); diff --git a/core/libs/prx/libc/src/Filesystem.cpp b/core/libs/prx/libc/src/Filesystem.cpp index d7f191170..2e2d2bd3b 100644 --- a/core/libs/prx/libc/src/Filesystem.cpp +++ b/core/libs/prx/libc/src/Filesystem.cpp @@ -99,6 +99,8 @@ extern "C" int APS5_VABI rename_nid_postfix(const char* from, const char* to) { std::error_code error; std::filesystem::rename(source, destination, error); if (error) { errno = FilesystemError(error); return -1; } + RecordWrittenPath_nid_no_patch(source); + RecordWrittenPath_nid_no_patch(destination); return 0; } catch (const std::bad_alloc&) { errno = 12; return -1; } catch (const std::filesystem::filesystem_error& error) { @@ -132,6 +134,7 @@ extern "C" int APS5_VABI remove_nid_postfix(const char* path) { errno = error ? FilesystemError(error) : 2; return -1; } + RecordWrittenPath_nid_no_patch(resolved); return 0; } catch (const std::bad_alloc&) { errno = 12; return -1; } catch (const std::filesystem::filesystem_error& error) { diff --git a/core/libs/prx/libc/src/Formatting.cpp b/core/libs/prx/libc/src/Formatting.cpp index 875bdf156..7395115b5 100644 --- a/core/libs/prx/libc/src/Formatting.cpp +++ b/core/libs/prx/libc/src/Formatting.cpp @@ -12,7 +12,6 @@ #include "prx/libc/include/General.hpp" #include "SceTypes.hpp" -#include "prx/libc/include/VarArgsAbi.hpp" #include "prx/libc/include/FileStream.hpp" #include "prx/libc/include/ApplicationHeap.hpp" @@ -95,6 +94,7 @@ int ScanGuest(const char* buffer, const char* format, bool secure, NextPointer n if (*probe == '\0') return finish(converted ? assigned : EOF); } const bool sized = secure && (conversion == 's' || conversion == 'c' || conversion == '['); + bool capped = false; std::string directive = "%"; void* target = nullptr; if (suppress) directive += '*'; @@ -107,6 +107,7 @@ int ScanGuest(const char* buffer, const char* format, bool secure, NextPointer n if (capacity != 0) static_cast(target)[0] = '\0'; return finish(assigned); } + capped = wanted == 0 || wanted > limit; width = wanted == 0 ? limit : std::min(wanted, limit); } if (width != 0) directive += std::to_string(width); @@ -116,7 +117,7 @@ int ScanGuest(const char* buffer, const char* format, bool secure, NextPointer n int consumed = -1; const int matched = suppress ? std::sscanf(input, directive.c_str(), &consumed) : std::sscanf(input, directive.c_str(), target, &consumed); if (consumed < 0 || (!suppress && matched != 1)) return finish(!converted && *input == '\0' ? EOF : assigned); - if (sized && !suppress && conversion != 'c') { + if (capped && conversion != 'c') { const char next = input[consumed]; bool overflow = false; if (conversion == 's') overflow = next != '\0' && !std::isspace(static_cast(next)); @@ -167,7 +168,10 @@ int APS5_VABI vasprintf_nid_postfix(char** destination, const char* format, VaLi const char* source = text.c_str(); #else char* text = nullptr; - const int count = ::vasprintf(&text, format, *reinterpret_cast(args)); + std::va_list copy; + va_copy(copy, *reinterpret_cast(args)); + const int count = ::vasprintf(&text, format, copy); + va_end(copy); std::unique_ptr owner(text, std::free); if (count < 0) return -1; const char* source = text; @@ -190,7 +194,10 @@ int APS5_VABI vfprintf_nid_postfix(FileStream* stream, const char* format, VaLis const int result = std::fwrite(buffer.data(), 1, static_cast(count), native) == static_cast(count) ? count : -1; #else - const int result = std::vfprintf(native, format, *reinterpret_cast(args)); + std::va_list copy; + va_copy(copy, *reinterpret_cast(args)); + const int result = std::vfprintf(native, format, copy); + va_end(copy); #endif stream->SyncStatus(); return result; @@ -274,44 +281,28 @@ int APS5_VABI printf_nid_postfix(const char* format, ...) { return result; } -int APS5_VABI libc_printf_nid_postfix(VA_ARGS) { - (void)rcx; (void)r8; (void)r9; - LibcDetail::RegSaveArea regs; - LibcDetail::FillRegSaveArea(regs, rsi, rdx, rcx, r8, r9, 0, - xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); - LibcDetail::VaListLayout layout; - std::va_list* va = LibcDetail::BuildVaList(layout, regs, 0u, - reinterpret_cast(overflow_arg_area)); - return std::vprintf(reinterpret_cast(rdi), *va); +int APS5_VABI libc_printf_nid_postfix(const char* format, ...) { + std::va_list args; + va_start(args, format); + const int result = std::vprintf(format, args); + va_end(args); + return result; } -int APS5_VABI snprintf_nid_postfix(VA_ARGS) { - LibcDetail::RegSaveArea regs; - LibcDetail::FillRegSaveArea(regs, rcx, r8, r9, 0, 0, 0, - xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); - LibcDetail::VaListLayout layout; - std::va_list* va = LibcDetail::BuildVaList(layout, regs, 0u, - reinterpret_cast(overflow_arg_area)); - return std::vsnprintf( - reinterpret_cast(rdi), - static_cast(rsi), - reinterpret_cast(rdx), - *va - ); +int APS5_VABI snprintf_nid_postfix(char* buffer, size_t size, const char* format, ...) { + std::va_list args; + va_start(args, format); + const int result = std::vsnprintf(buffer, size, format, args); + va_end(args); + return result; } -int APS5_VABI sprintf_nid_postfix(VA_ARGS) { - LibcDetail::RegSaveArea regs; - LibcDetail::FillRegSaveArea(regs, rdx, rcx, r8, r9, 0, 0, - xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); - LibcDetail::VaListLayout layout; - std::va_list* va = LibcDetail::BuildVaList(layout, regs, 0u, - reinterpret_cast(overflow_arg_area)); - return std::vsprintf( - reinterpret_cast(rdi), - reinterpret_cast(rsi), - *va - ); +int APS5_VABI sprintf_nid_postfix(char* buffer, const char* format, ...) { + std::va_list args; + va_start(args, format); + const int result = std::vsprintf(buffer, format, args); + va_end(args); + return result; } #endif @@ -320,7 +311,11 @@ int APS5_VABI vsscanf_nid_postfix(const char* input, const char* format, VaList* #ifdef _WIN32 return LibcDetail::ScanWindows(input, format, args); #else - return std::vsscanf(input, format, *reinterpret_cast(args)); + std::va_list copy; + va_copy(copy, *reinterpret_cast(args)); + const int result = std::vsscanf(input, format, copy); + va_end(copy); + return result; #endif } @@ -333,18 +328,12 @@ int APS5_VABI sscanf_nid_postfix(const char* input, const char* format, ...) { return result; } #else -int APS5_VABI sscanf_nid_postfix(VA_ARGS) { - LibcDetail::RegSaveArea regs; - LibcDetail::FillRegSaveArea(regs, rdx, rcx, r8, r9, 0, 0, - xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7); - LibcDetail::VaListLayout layout; - std::va_list* va = LibcDetail::BuildVaList(layout, regs, 0u, - reinterpret_cast(overflow_arg_area)); - return std::vsscanf( - reinterpret_cast(rdi), - reinterpret_cast(rsi), - *va - ); +int APS5_VABI sscanf_nid_postfix(const char* input, const char* format, ...) { + std::va_list args; + va_start(args, format); + const int result = std::vsscanf(input, format, args); + va_end(args); + return result; } #endif @@ -374,8 +363,11 @@ int APS5_VABI vprintf_nid_postfix(const char* str, VaList* c) { #ifdef _WIN32 return LibcDetail::PrintWindows(str, c); #else - std::va_list* va = reinterpret_cast(c); - return std::vprintf(str, *va); + std::va_list copy; + va_copy(copy, *reinterpret_cast(c)); + const int result = std::vprintf(str, copy); + va_end(copy); + return result; #endif } @@ -383,7 +375,11 @@ int APS5_VABI vsprintf_nid_postfix(char* str, const char* format, VaList* args) #ifdef _WIN32 return LibcDetail::FormatWindows(str, SIZE_MAX, format, args); #else - return std::vsprintf(str, format, *reinterpret_cast(args)); + std::va_list copy; + va_copy(copy, *reinterpret_cast(args)); + const int result = std::vsprintf(str, format, copy); + va_end(copy); + return result; #endif } @@ -391,8 +387,11 @@ int APS5_VABI vsnprintf_nid_postfix(char* str, size_t size, const char* format, #ifdef _WIN32 return LibcDetail::FormatWindows(str, size, format, c); #else - std::va_list* va = reinterpret_cast(c); - return std::vsnprintf(str, size, format, *va); + std::va_list copy; + va_copy(copy, *reinterpret_cast(c)); + const int result = std::vsnprintf(str, size, format, copy); + va_end(copy); + return result; #endif } @@ -417,7 +416,11 @@ int APS5_VABI printf_s_nid_postfix(const char* format, ...) { #else int APS5_VABI vsprintf_s_nid_postfix(char* buffer, size_t size, const char* format, VaList* args) { - return std::vsnprintf(buffer, size, format, *reinterpret_cast(args)); + std::va_list copy; + va_copy(copy, *reinterpret_cast(args)); + const int result = std::vsnprintf(buffer, size, format, copy); + va_end(copy); + return result; } int APS5_VABI sprintf_s_nid_postfix(char* buffer, size_t size, const char* format, ...) { diff --git a/core/libs/prx/libc/src/FormattingWide.cpp b/core/libs/prx/libc/src/FormattingWide.cpp index 35912693c..7808ba4ba 100644 --- a/core/libs/prx/libc/src/FormattingWide.cpp +++ b/core/libs/prx/libc/src/FormattingWide.cpp @@ -10,7 +10,6 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" -#include "prx/libc/include/VarArgsAbi.hpp" #include "prx/libc/include/FileStream.hpp" #include "prx/libc/include/WindowsFormatting.hpp" diff --git a/core/libs/prx/libc/src/General.cpp b/core/libs/prx/libc/src/General.cpp index 7d9d8d137..a190e8fcc 100644 --- a/core/libs/prx/libc/src/General.cpp +++ b/core/libs/prx/libc/src/General.cpp @@ -1,12 +1,28 @@ #include #include #include +#include #include #include #include #include #include #include +#include +#include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#endif #include "prx/libc/include/General.hpp" #include "prx/libc/include/GuestHeap.hpp" @@ -16,6 +32,7 @@ namespace { struct PathAliases { std::mutex mutex; std::vector> entries; + std::vector blocked; }; PathAliases& Aliases() { @@ -36,15 +53,54 @@ std::string TrimSlashes(const char* path) { return s.substr(start, end - start); } +bool SameName(std::string_view left, std::string_view right) { + if (left.size() != right.size()) return false; + const auto fold = [](unsigned char value) { return value >= 'A' && value <= 'Z' ? value + ('a' - 'A') : value; }; + for (std::size_t i = 0; i < left.size(); ++i) if (fold(left[i]) != fold(right[i])) return false; + return true; +} + +bool HasPrefix(const std::string& path, const std::string& prefix) { + return path.size() >= prefix.size() && SameName(std::string_view(path).substr(0, prefix.size()), prefix) && + (path.size() == prefix.size() || path[prefix.size()] == '/'); +} + +std::filesystem::path ResolveHostPath(std::filesystem::path root, const std::filesystem::path& relative) { + const auto direct = root / relative; + std::error_code error; + if (std::filesystem::exists(direct, error)) return direct; + for (const auto& part : relative) { + const auto candidate = root / part; + const auto status = std::filesystem::symlink_status(candidate, error); + if (std::filesystem::exists(status) || (error && error != std::errc::no_such_file_or_directory)) { + root = candidate; + continue; + } + std::optional matched; + const auto name = part.string(); + std::filesystem::directory_iterator entry(root, error), end; + for (; !error && entry != end; entry.increment(error)) { + if (!SameName(entry->path().filename().string(), name)) continue; + if (matched) throw std::runtime_error("Ambiguous case-insensitive guest path: " + candidate.string()); + matched = entry->path(); + } + root = !error && matched ? *matched : candidate; + } + return root; +} + std::optional ResolveAlias(const std::string& guestPath) { const auto relative = TrimSlashes(guestPath.c_str()); auto& aliases = Aliases(); std::lock_guard lock(aliases.mutex); + for (const auto& prefix : aliases.blocked) { + if (HasPrefix(relative, prefix)) + throw std::filesystem::filesystem_error("Guest mount is unavailable", guestPath, std::make_error_code(std::errc::no_such_file_or_directory)); + } for (const auto& [prefix, host] : aliases.entries) { - if (relative.size() < prefix.size() || relative.compare(0, prefix.size(), prefix) != 0) continue; + if (!HasPrefix(relative, prefix)) continue; if (relative.size() == prefix.size()) return std::filesystem::path(host).make_preferred(); - if (relative[prefix.size()] != '/') continue; - std::filesystem::path result = std::filesystem::path(host) / std::filesystem::path(relative.substr(prefix.size() + 1)); + std::filesystem::path result = ResolveHostPath(host, relative.substr(prefix.size() + 1)); return result.make_preferred(); } return std::nullopt; @@ -67,8 +123,43 @@ std::filesystem::path Resolve(WorkingDirectory& state, const char* path) { auto guest = (std::filesystem::path("/") / state.current.lexically_relative(state.root)); guest = (input.is_absolute() ? input : guest / input).lexically_normal(); if (auto aliased = ResolveAlias(guest.relative_path().generic_string())) return *aliased; - return (state.root / guest.relative_path()).make_preferred(); + return ResolveHostPath(state.root, guest.relative_path()).make_preferred(); } +struct WrittenPathRegistry { + std::mutex syncMutex; + std::mutex mutex; + std::map dirty; +#ifdef _WIN32 + FILETIME mark = [] { FILETIME now; GetSystemTimeAsFileTime(&now); return now; }(); +#endif +}; +WrittenPathRegistry& Written() { static WrittenPathRegistry registry; return registry; } +#ifdef _WIN32 +std::optional WriteTime(const std::filesystem::path& path) { + WIN32_FILE_ATTRIBUTE_DATA data; + if (GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &data)) return data.ftLastWriteTime; + const auto error = GetLastError(); + if (error == ERROR_FILE_NOT_FOUND || error == ERROR_PATH_NOT_FOUND) return std::nullopt; + throw std::system_error(static_cast(error), std::system_category(), "Reading the write time of " + path.string()); +} +bool ChangedSince(const std::filesystem::path& path, const FILETIME& mark) { + const auto time = WriteTime(path); + return time && CompareFileTime(&*time, &mark) >= 0; +} +void FlushPath(const std::filesystem::path& path) { + const auto handle = CreateFileW(path.c_str(), GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, nullptr); + if (handle == INVALID_HANDLE_VALUE) { + const auto error = GetLastError(); + if (error == ERROR_FILE_NOT_FOUND || error == ERROR_PATH_NOT_FOUND) return; + if (error == ERROR_ACCESS_DENIED || error == ERROR_SHARING_VIOLATION) return; + throw std::system_error(static_cast(error), std::system_category(), "Opening " + path.string() + " for sync"); + } + const std::unique_ptr owner(handle, &CloseHandle); + if (!FlushFileBuffers(handle)) + throw std::system_error(static_cast(GetLastError()), std::system_category(), "Flushing " + path.string()); +} +#endif int DirectoryFailure(const std::error_code& error) { if (error == std::errc::permission_denied) return 13; if (error == std::errc::not_a_directory) return 20; @@ -86,8 +177,9 @@ extern "C" void AddPathAlias_nid_no_patch(const char* guestPrefix, const char* h auto& aliases = Aliases(); std::lock_guard lock(aliases.mutex); const auto prefix = TrimSlashes(guestPrefix); + std::erase_if(aliases.blocked, [&](const auto& entry) { return SameName(entry, prefix); }); for (auto& entry : aliases.entries) { - if (entry.first == prefix) { + if (SameName(entry.first, prefix)) { entry.second = hostPath; return; } @@ -102,7 +194,74 @@ extern "C" void RemovePathAlias_nid_no_patch(const char* guestPrefix) { auto& aliases = Aliases(); std::lock_guard lock(aliases.mutex); const auto prefix = TrimSlashes(guestPrefix); - std::erase_if(aliases.entries, [&](const auto& entry) { return entry.first == prefix; }); + std::erase_if(aliases.entries, [&](const auto& entry) { return SameName(entry.first, prefix); }); +} + +extern "C" void RecordWrittenPath_nid_no_patch(const std::filesystem::path& path) { + auto& registry = Written(); + std::lock_guard lock(registry.mutex); + registry.dirty[path.lexically_normal()] = true; +} + +extern "C" std::vector WrittenPaths_nid_no_patch() { + auto& registry = Written(); + std::lock_guard lock(registry.mutex); + std::vector paths; + for (const auto& entry : registry.dirty) paths.push_back(entry.first); + return paths; +} + +extern "C" void SyncWrittenPaths_nid_no_patch() { + auto& registry = Written(); + std::lock_guard serial(registry.syncMutex); +#ifdef _WIN32 + FILETIME start; + GetSystemTimeAsFileTime(&start); + FILETIME mark; + std::vector> entries; + { + std::lock_guard lock(registry.mutex); + mark = registry.mark; + registry.mark = start; + for (auto& [path, dirty] : registry.dirty) { + entries.emplace_back(path, dirty); + dirty = false; + } + } + std::set flushed; + std::vector missing; + for (const auto& [path, dirty] : entries) { + const auto time = WriteTime(path); + if (!time) missing.push_back(path); + if (!dirty && !(time && CompareFileTime(&*time, &mark) >= 0)) continue; + if (time && flushed.insert(path).second) FlushPath(path); + for (auto ancestor = path.parent_path(); ancestor.has_relative_path() && !flushed.contains(ancestor) && ChangedSince(ancestor, mark); + ancestor = ancestor.parent_path()) { + flushed.insert(ancestor); + FlushPath(ancestor); + } + } + std::lock_guard lock(registry.mutex); + for (const auto& path : missing) { + const auto entry = registry.dirty.find(path); + if (entry != registry.dirty.end() && !entry->second) registry.dirty.erase(entry); + } +#else + ::sync(); + std::lock_guard lock(registry.mutex); + registry.dirty.clear(); +#endif +} + +extern "C" void BlockPathAlias_nid_no_patch(const char* guestPrefix) { + if (guestPrefix == nullptr) { APS5_INVALID_ARG_EX; } + const auto prefix = TrimSlashes(guestPrefix); + if (prefix.empty()) { APS5_INVALID_ARG_EX; } + auto& aliases = Aliases(); + std::lock_guard lock(aliases.mutex); + std::erase_if(aliases.entries, [&](const auto& entry) { return SameName(entry.first, prefix); }); + std::erase_if(aliases.blocked, [&](const auto& entry) { return SameName(entry, prefix); }); + aliases.blocked.push_back(prefix); } extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path) { diff --git a/core/libs/prx/libc/src/GuestAllocations.cpp b/core/libs/prx/libc/src/GuestAllocations.cpp index 344fcaa82..a29a4cf1c 100644 --- a/core/libs/prx/libc/src/GuestAllocations.cpp +++ b/core/libs/prx/libc/src/GuestAllocations.cpp @@ -15,7 +15,15 @@ #endif #include #else +#include +#include +#include +#include +#include +#include +#include #include +#include #include #endif @@ -61,6 +69,38 @@ void recordChange(void* mutation, const void* pointer, std::size_t bytes) { if (mutation != nullptr && bytes != 0) static_cast(mutation)->changed.emplace_back(reinterpret_cast(pointer), bytes); } +#ifndef _WIN32 +std::vector> fileBackedWritableImage() { + const auto image = std::filesystem::read_symlink("/proc/self/exe").string(); + std::ifstream maps("/proc/self/maps"); + std::vector> mappings; + for (std::string line; std::getline(maps, line);) { + std::istringstream fields(line); + std::string span, permissions, offset, device, inode, path; + fields >> span >> permissions >> offset >> device >> inode >> std::ws; + std::getline(fields, path); + if (permissions != "rw-p" || path != image) continue; + const auto dash = span.find('-'); + const auto begin = std::stoull(span.substr(0, dash), nullptr, 16); + mappings.emplace_back(begin, std::stoull(span.substr(dash + 1), nullptr, 16) - begin); + } + return mappings; +} + +bool backWritableImageAnonymously() { + for (const auto& [address, bytes] : fileBackedWritableImage()) { + auto* image = reinterpret_cast(address); + auto* copy = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (copy == MAP_FAILED) throw std::system_error(errno, std::generic_category(), "mmap of an anonymous main image copy failed"); + std::memcpy(copy, image, bytes); + if (mremap(copy, bytes, bytes, MREMAP_MAYMOVE | MREMAP_FIXED, image) == MAP_FAILED) throw std::system_error(errno, std::generic_category(), "mremap of an anonymous main image copy failed"); + } + return true; +} + +[[maybe_unused]] const bool writableImageAnonymous = backWritableImageAnonymously(); +#endif + } void* GuestAllocationsBegin_nid_postfix() { @@ -227,9 +267,10 @@ void GuestAllocationsRequireUnpinned_nid_postfix(void* mutation, const void* poi if (std::chrono::steady_clock::now() >= deadline) PinnedFailure(address, bytes, ("waited " + std::to_string(pinWait().count()) + " ms for the lease (APS5_PIN_WAIT_MS)").c_str()); const auto waiter = pinWaiter.load(std::memory_order_acquire); bool progressed = false; - if (waiter != nullptr && state != nullptr && state->lock.owns_lock()) { + if (state != nullptr && state->lock.owns_lock()) { state->lock.unlock(); - progressed = waiter(); + if (waiter != nullptr) progressed = waiter(); + else std::this_thread::yield(); state->lock.lock(); } else { std::this_thread::yield(); diff --git a/core/libs/prx/libc/src/GuestArena.cpp b/core/libs/prx/libc/src/GuestArena.cpp index ff7a0b4f0..b815e0fab 100644 --- a/core/libs/prx/libc/src/GuestArena.cpp +++ b/core/libs/prx/libc/src/GuestArena.cpp @@ -19,6 +19,11 @@ namespace GuestArena { namespace { +#ifdef _WIN32 +std::atomic commitGeneration{1}; +std::atomic privateMappingObserver{nullptr}; +#endif + constexpr std::uintptr_t ArenaStart = 0x0000000200000000ull; constexpr std::uintptr_t SystemReservedStart = 0x00000007FFFFC000ull; constexpr std::uintptr_t SystemReservedEnd = 0x0000001000000000ull; @@ -108,13 +113,19 @@ private: GetSystemInfo(&system); const std::uintptr_t granularity = system.dwAllocationGranularity; const std::uintptr_t end = ApplicationAreaEnd; + std::uint32_t retries = 0; for (std::uintptr_t cursor = ArenaStart; cursor < end;) { MEMORY_BASIC_INFORMATION info{}; if (VirtualQuery(reinterpret_cast(cursor), &info, sizeof(info)) == 0) throw std::system_error(static_cast(GetLastError()), std::system_category(), "query the guest arena range"); const auto regionEnd = std::min(end, reinterpret_cast(info.BaseAddress) + info.RegionSize); const auto first = info.State == MEM_FREE ? std::min(regionEnd, alignUp(cursor, granularity)) : regionEnd; const auto last = std::max(first, regionEnd & ~(granularity - 1)); - if (first < last && WindowsMappings::Get().Reserve(reinterpret_cast(first), last - first) == nullptr) throw std::system_error(static_cast(GetLastError()), std::system_category(), "reserve the guest arena range"); + if (first < last && WindowsMappings::Get().Reserve(reinterpret_cast(first), last - first) == nullptr) { + const auto error = GetLastError(); + if (error == ERROR_INVALID_ADDRESS && ++retries < 64u) continue; + throw std::system_error(static_cast(error), std::system_category(), "reserve the guest arena range"); + } + retries = 0; if (cursor < first) _holes.emplace_back(cursor, first); if (last < regionEnd) _holes.emplace_back(last, regionEnd); cursor = regionEnd; @@ -221,9 +232,21 @@ std::invalid_argument OutsideArena(const char* operation, const void* pointer, s } +std::uint64_t GuestArenaCommitGeneration_nid_postfix() { + return commitGeneration.load(std::memory_order_acquire); +} + +void GuestArenaSetPrivateMappingObserver_nid_postfix(void (*callback)(std::uintptr_t, std::size_t, std::uint64_t)) { + privateMappingObserver.store(callback, std::memory_order_release); +} + void GuestArenaCommit_nid_postfix(void* pointer, std::size_t bytes, std::uint32_t protection, std::size_t granule) { if (!Arena::Get().Contains(pointer, bytes)) throw OutsideArena("commit", pointer, bytes); - WindowsMappings::Get().Commit(pointer, bytes, protection, granule, Arena::Get().WriteWatched()); + const auto generation = commitGeneration.fetch_add(1, std::memory_order_acq_rel) + 1; + const auto created = WindowsMappings::Get().Commit(pointer, bytes, protection, granule, Arena::Get().WriteWatched()); + if (const auto callback = privateMappingObserver.load(std::memory_order_acquire)) { + for (const auto& [address, size] : created) callback(address, size, generation); + } } void GuestArenaReset_nid_postfix(void* pointer, std::size_t bytes) { diff --git a/core/libs/prx/libc/src/GuestHeap.cpp b/core/libs/prx/libc/src/GuestHeap.cpp index b5d5f03a1..ae9d1674c 100644 --- a/core/libs/prx/libc/src/GuestHeap.cpp +++ b/core/libs/prx/libc/src/GuestHeap.cpp @@ -43,6 +43,9 @@ public: } void* Allocate(std::size_t total, std::size_t& blockBytes) { + if (total > MaximumSmallBytes && + total > std::numeric_limits::max() - (LargeGranuleBytes - 1)) + throw std::length_error("guest heap allocation overflow"); if (total > MaximumSmallBytes) { blockBytes = alignUp(total, LargeGranuleBytes); if (void* cached = _large.Take(blockBytes)) return cached; diff --git a/core/libs/prx/libc/src/HeapDiagnostics.cpp b/core/libs/prx/libc/src/HeapDiagnostics.cpp index 0288d5c0a..39aaf3951 100644 --- a/core/libs/prx/libc/src/HeapDiagnostics.cpp +++ b/core/libs/prx/libc/src/HeapDiagnostics.cpp @@ -42,10 +42,4 @@ void APS5_VABI sceLibcInternalHeapErrorReportForGame_nid_postfix(void* heap, voi NotImplemented_nid_no_patch(__func__); } -APS5_EXPORT("BnMAMrsfVWo", sceLibcUnknown_BnMAMrsfVWo); -int32_t APS5_VABI sceLibcUnknown_BnMAMrsfVWo(void) { - NotImplemented_nid_no_patch(__func__); - return 0; -} - } diff --git a/core/libs/prx/libc/src/Math.cpp b/core/libs/prx/libc/src/Math.cpp index 37e89f563..1ae4fb350 100644 --- a/core/libs/prx/libc/src/Math.cpp +++ b/core/libs/prx/libc/src/Math.cpp @@ -18,6 +18,8 @@ float APS5_VABI fmodf_nid_postfix(float x, float y) { return std::fmod(x, y); } float APS5_VABI asinf_nid_postfix(float x) { return std::asin(x); } float APS5_VABI acosf_nid_postfix(float x) { return std::acos(x); } float APS5_VABI atan2f_nid_postfix(float y, float x) { return std::atan2(y, x); } +float APS5_VABI hypotf_nid_postfix(float x, float y) { return static_cast(std::hypot(static_cast(x), static_cast(y))); } +double APS5_VABI hypot_nid_postfix(double x, double y) { return std::hypot(x, y); } float APS5_VABI tanf_nid_postfix(float x) { return std::tan(x); } float APS5_VABI log10f_nid_postfix(float x) { return std::log10(x); } float APS5_VABI logbf_nid_postfix(float x) { return std::logb(x); } diff --git a/core/libs/prx/libc/src/Mspace.cpp b/core/libs/prx/libc/src/Mspace.cpp index c5cae4155..d64d02d4a 100644 --- a/core/libs/prx/libc/src/Mspace.cpp +++ b/core/libs/prx/libc/src/Mspace.cpp @@ -216,6 +216,7 @@ void* APS5_VABI sceLibcMspaceRealloc_nid_postfix(void* handle, void* pointer, st if (!FindUsed(arena, pointer, chunk)) return nullptr; if (!size) { Release(*arena, chunk); return nullptr; } const auto needed = AlignUp(size, Granule); + if (needed < size) { Error(12); return nullptr; } const auto start = chunk->first; const auto capacity = chunk->second.end - start; if (needed <= capacity) { @@ -276,8 +277,10 @@ void* APS5_VABI sceLibcMspaceReallocalign_nid_postfix(void* handle, void* pointe std::map::iterator chunk; if (!FindUsed(arena, pointer, chunk)) return nullptr; if (!size) { Release(*arena, chunk); return nullptr; } + const auto needed = AlignUp(size, Granule); + if (needed < size) { Error(12); return nullptr; } const auto start = chunk->first; - if (AlignUp(size, Granule) <= chunk->second.end - start && (start & (effective - 1)) == 0) { + if (needed <= chunk->second.end - start && (start & (effective - 1)) == 0) { chunk->second.requested = size; return pointer; } diff --git a/core/libs/prx/libc/src/StandardStreams.cpp b/core/libs/prx/libc/src/StandardStreams.cpp index 32837f9cf..856937ab4 100644 --- a/core/libs/prx/libc/src/StandardStreams.cpp +++ b/core/libs/prx/libc/src/StandardStreams.cpp @@ -78,6 +78,10 @@ int APS5_VABI fileno_nid_postfix(FileStream* stream) { int APS5_VABI setvbuf_nid_postfix(FileStream* stream, char* buffer, int mode, std::size_t size) { if (mode < 0 || mode > 2) return -1; const int native = mode == 0 ? _IOFBF : mode == 1 ? _IOLBF : _IONBF; + if (native != _IONBF && size == 0) { + buffer = nullptr; + size = BUFSIZ; + } return std::setvbuf(GetNativeStream(stream), buffer, native, size); } void APS5_VABI setbuf_nid_postfix(FileStream* stream, char* buffer) { diff --git a/core/libs/prx/libc/src/StdioExtras.cpp b/core/libs/prx/libc/src/StdioExtras.cpp index 09e1b5799..0b35feabc 100644 --- a/core/libs/prx/libc/src/StdioExtras.cpp +++ b/core/libs/prx/libc/src/StdioExtras.cpp @@ -114,4 +114,9 @@ void APS5_VABI _ZdlPvSt11align_val_t_nid_postfix(void* pointer, std::size_t alig if (pointer != nullptr) ApplicationHeapFree_nid_no_patch(pointer); } +void APS5_VABI _ZdlPvmSt11align_val_t_nid_postfix(void* pointer, std::size_t, std::size_t alignment) { + (void)alignment; + if (pointer != nullptr) ApplicationHeapFree_nid_no_patch(pointer); +} + } diff --git a/core/libs/prx/libc/src/Strings.cpp b/core/libs/prx/libc/src/Strings.cpp index 8cba0d618..533611a17 100644 --- a/core/libs/prx/libc/src/Strings.cpp +++ b/core/libs/prx/libc/src/Strings.cpp @@ -6,9 +6,46 @@ #include #include #include +#include #include "prx/libc/include/General.hpp" +namespace { + +int CompareUnits(char16_t left, char16_t right) { + return left < right ? -1 : 1; +} + +bool Contains(const char16_t* set, char16_t character) { + for (; *set != 0; ++set) { + if (*set == character) return true; + } + return false; +} + +size_t Length(const char16_t* s) { + size_t length = 0; + while (s[length] != 0) ++length; + return length; +} + +std::string AsciiPrefix(const char16_t* text) { + std::string prefix; + for (; *text != 0 && *text < 0x80; ++text) prefix.push_back(static_cast(*text)); + return prefix; +} + +template +auto ParseAsciiPrefix(const char16_t* text, char16_t** end, TParse parse) { + const std::string prefix = AsciiPrefix(text); + char* parsedEnd = nullptr; + const auto value = parse(prefix.c_str(), &parsedEnd); + if (end != nullptr) *end = const_cast(text) + (parsedEnd - prefix.c_str()); + return value; +} + +} + extern "C" { void* APS5_VABI memset_nid_postfix(void* s, int c, size_t n) { @@ -117,20 +154,28 @@ std::div_t APS5_VABI div_nid_postfix(int numerator, int denominator) { return std::div(numerator, denominator); } -const wchar_t* APS5_VABI wmemchr_nid_postfix(const wchar_t* s, wchar_t c, size_t n) { - return std::wmemchr(s, c, n); +const char16_t* APS5_VABI wmemchr_nid_postfix(const char16_t* s, char16_t c, size_t n) { + for (; n != 0; ++s, --n) { + if (*s == c) return s; + } + return nullptr; } -int APS5_VABI wmemcmp_nid_postfix(const wchar_t* s1, const wchar_t* s2, size_t n) { - return std::wmemcmp(s1, s2, n); +int APS5_VABI wmemcmp_nid_postfix(const char16_t* s1, const char16_t* s2, size_t n) { + for (; n != 0; ++s1, ++s2, --n) { + if (*s1 != *s2) return CompareUnits(*s1, *s2); + } + return 0; } -wchar_t* APS5_VABI wmemcpy_nid_postfix(wchar_t* dest, const wchar_t* src, size_t n) { - return std::wmemcpy(dest, src, n); +char16_t* APS5_VABI wmemcpy_nid_postfix(char16_t* dest, const char16_t* src, size_t n) { + if (n != 0) std::memcpy(dest, src, n * sizeof(char16_t)); + return dest; } -wchar_t* APS5_VABI wmemmove_nid_postfix(wchar_t* dest, const wchar_t* src, size_t n) { - return std::wmemmove(dest, src, n); +char16_t* APS5_VABI wmemmove_nid_postfix(char16_t* dest, const char16_t* src, size_t n) { + if (n != 0) std::memmove(dest, src, n * sizeof(char16_t)); + return dest; } } @@ -274,86 +319,119 @@ unsigned long long APS5_VABI _Stoull_nid_postfix(const char* str, char** endptr, return std::strtoull(str, endptr, base); } -size_t APS5_VABI wcslen_nid_postfix(const wchar_t* s) { - return std::wcslen(s); +size_t APS5_VABI wcslen_nid_postfix(const char16_t* s) { + return Length(s); } -int APS5_VABI wcscmp_nid_postfix(const wchar_t* s1, const wchar_t* s2) { - return std::wcscmp(s1, s2); +int APS5_VABI wcscmp_nid_postfix(const char16_t* s1, const char16_t* s2) { + for (; *s1 == *s2; ++s1, ++s2) { + if (*s1 == 0) return 0; + } + return CompareUnits(*s1, *s2); } -int APS5_VABI wcsncmp_nid_postfix(const wchar_t* s1, const wchar_t* s2, size_t n) { - return std::wcsncmp(s1, s2, n); +int APS5_VABI wcsncmp_nid_postfix(const char16_t* s1, const char16_t* s2, size_t n) { + for (; n != 0; ++s1, ++s2, --n) { + if (*s1 != *s2) return CompareUnits(*s1, *s2); + if (*s1 == 0) return 0; + } + return 0; } -wchar_t* APS5_VABI wcscpy_nid_postfix(wchar_t* dest, const wchar_t* src) { - return std::wcscpy(dest, src); +char16_t* APS5_VABI wcscpy_nid_postfix(char16_t* dest, const char16_t* src) { + std::memcpy(dest, src, (Length(src) + 1) * sizeof(char16_t)); + return dest; } -wchar_t* APS5_VABI wcsncpy_nid_postfix(wchar_t* dest, const wchar_t* src, size_t n) { - return std::wcsncpy(dest, src, n); +char16_t* APS5_VABI wcsncpy_nid_postfix(char16_t* dest, const char16_t* src, size_t n) { + size_t index = 0; + for (; index < n && src[index] != 0; ++index) dest[index] = src[index]; + for (; index < n; ++index) dest[index] = 0; + return dest; } -const wchar_t* APS5_VABI wcschr_nid_postfix(const wchar_t* s, wchar_t c) { - return std::wcschr(s, c); +const char16_t* APS5_VABI wcschr_nid_postfix(const char16_t* s, char16_t c) { + for (;; ++s) { + if (*s == c) return s; + if (*s == 0) return nullptr; + } } -const wchar_t* APS5_VABI wcsrchr_nid_postfix(const wchar_t* s, wchar_t c) { - return std::wcsrchr(s, c); +const char16_t* APS5_VABI wcsrchr_nid_postfix(const char16_t* s, char16_t c) { + const char16_t* found = nullptr; + for (;; ++s) { + if (*s == c) found = s; + if (*s == 0) return found; + } } -const wchar_t* APS5_VABI wcsstr_nid_postfix(const wchar_t* haystack, const wchar_t* needle) { - return std::wcsstr(haystack, needle); +const char16_t* APS5_VABI wcsstr_nid_postfix(const char16_t* haystack, const char16_t* needle) { + const size_t length = Length(needle); + if (length == 0) return haystack; + for (; *haystack != 0; ++haystack) { + size_t matched = 0; + while (matched < length && haystack[matched] == needle[matched]) ++matched; + if (matched == length) return haystack; + } + return nullptr; } -const wchar_t* APS5_VABI wcspbrk_nid_postfix(const wchar_t* s, const wchar_t* accept) { - return std::wcspbrk(s, accept); +const char16_t* APS5_VABI wcspbrk_nid_postfix(const char16_t* s, const char16_t* accept) { + for (; *s != 0; ++s) { + if (Contains(accept, *s)) return s; + } + return nullptr; } -size_t APS5_VABI wcsspn_nid_postfix(const wchar_t* s, const wchar_t* accept) { - return std::wcsspn(s, accept); +size_t APS5_VABI wcsspn_nid_postfix(const char16_t* s, const char16_t* accept) { + size_t count = 0; + while (s[count] != 0 && Contains(accept, s[count])) ++count; + return count; } -wchar_t* APS5_VABI wmemset_nid_postfix(wchar_t* s, wchar_t c, size_t n) { - return std::wmemset(s, c, n); +char16_t* APS5_VABI wmemset_nid_postfix(char16_t* s, char16_t c, size_t n) { + for (size_t index = 0; index < n; ++index) s[index] = c; + return s; } -double APS5_VABI wcstod_nid_postfix(const wchar_t* str, wchar_t** endptr) { - return std::wcstod(str, endptr); +double APS5_VABI wcstod_nid_postfix(const char16_t* str, char16_t** endptr) { + return ParseAsciiPrefix(str, endptr, [](const char* text, char** end) { return std::strtod(text, end); }); } -float APS5_VABI wcstof_nid_postfix(const wchar_t* str, wchar_t** endptr) { - return std::wcstof(str, endptr); +float APS5_VABI wcstof_nid_postfix(const char16_t* str, char16_t** endptr) { + return ParseAsciiPrefix(str, endptr, [](const char* text, char** end) { return std::strtof(text, end); }); } -long double APS5_VABI wcstold_nid_postfix(const wchar_t* str, wchar_t** endptr) { +long double APS5_VABI wcstold_nid_postfix(const char16_t* str, char16_t** endptr) { static_assert(sizeof(long double) == 16); static_assert(std::numeric_limits::digits == 64); - return std::wcstold(str, endptr); + return ParseAsciiPrefix(str, endptr, [](const char* text, char** end) { return std::strtold(text, end); }); } -long long APS5_VABI wcstol_nid_postfix(const wchar_t* str, wchar_t** endptr, int base) { - return std::wcstoll(str, endptr, base); +long long APS5_VABI wcstol_nid_postfix(const char16_t* str, char16_t** endptr, int base) { + return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return std::strtoll(text, end, base); }); } -long long APS5_VABI wcstoll_nid_postfix(const wchar_t* str, wchar_t** endptr, int base) { - return std::wcstoll(str, endptr, base); +long long APS5_VABI wcstoll_nid_postfix(const char16_t* str, char16_t** endptr, int base) { + return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return std::strtoll(text, end, base); }); } -unsigned long long APS5_VABI wcstoul_nid_postfix(const wchar_t* str, wchar_t** endptr, int base) { - return std::wcstoull(str, endptr, base); +unsigned long long APS5_VABI wcstoul_nid_postfix(const char16_t* str, char16_t** endptr, int base) { + return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return std::strtoull(text, end, base); }); } -unsigned long long APS5_VABI wcstoull_nid_postfix(const wchar_t* str, wchar_t** endptr, int base) { - return std::wcstoull(str, endptr, base); +unsigned long long APS5_VABI wcstoull_nid_postfix(const char16_t* str, char16_t** endptr, int base) { + return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return std::strtoull(text, end, base); }); } -int APS5_VABI wcscoll_nid_postfix(const wchar_t* first, const wchar_t* second) { - return std::wcscoll(first, second); +int APS5_VABI wcscoll_nid_postfix(const char16_t* first, const char16_t* second) { + return wcscmp_nid_postfix(first, second); } -size_t APS5_VABI wcsxfrm_nid_postfix(wchar_t* destination, const wchar_t* source, size_t count) { - return std::wcsxfrm(destination, source, count); +size_t APS5_VABI wcsxfrm_nid_postfix(char16_t* destination, const char16_t* source, size_t count) { + const size_t length = Length(source); + if (length < count) std::memcpy(destination, source, (length + 1) * sizeof(char16_t)); + return length; } size_t APS5_VABI strxfrm_nid_postfix(char* destination, const char* source, size_t count) { diff --git a/core/libs/prx/libc/src/Time.cpp b/core/libs/prx/libc/src/Time.cpp index 53199eed7..4710acdd0 100644 --- a/core/libs/prx/libc/src/Time.cpp +++ b/core/libs/prx/libc/src/Time.cpp @@ -11,6 +11,9 @@ extern "C" { +std::tm* APS5_VABI localtime_s_nid_postfix(const int64_t* timer, std::tm* result); +std::tm* APS5_VABI gmtime_s_nid_postfix(const int64_t* timer, std::tm* result); + int64_t APS5_VABI libc_time_nid_postfix(int64_t* timer) { std::time_t t = std::time(nullptr); if (timer != nullptr) *timer = static_cast(t); @@ -44,20 +47,12 @@ double APS5_VABI difftime_nid_postfix(int64_t time1, int64_t time0) { std::tm* APS5_VABI libc_gmtime_nid_postfix(const int64_t* timer) { static thread_local std::tm result; - const std::time_t t = static_cast(*timer); - const std::tm* converted = std::gmtime(&t); - if (converted == nullptr) return nullptr; - result = *converted; - return &result; + return gmtime_s_nid_postfix(timer, &result); } std::tm* APS5_VABI libc_localtime_nid_postfix(const int64_t* timer) { static thread_local std::tm result; - const std::time_t t = static_cast(*timer); - const std::tm* converted = std::localtime(&t); - if (converted == nullptr) return nullptr; - result = *converted; - return &result; + return localtime_s_nid_postfix(timer, &result); } std::tm* APS5_VABI localtime_nid_postfix(const int64_t* timer) { diff --git a/core/libs/prx/libc/src/exception/ExceptionPointer.cpp b/core/libs/prx/libc/src/exception/ExceptionPointer.cpp index 50bfcdffa..eba668749 100644 --- a/core/libs/prx/libc/src/exception/ExceptionPointer.cpp +++ b/core/libs/prx/libc/src/exception/ExceptionPointer.cpp @@ -1,5 +1,6 @@ #include +#ifndef _LIBCPP_VERSION extern "C" { void ExceptionPointerAddref(std::exception_ptr* self) noexcept asm("_ZNSt15__exception_ptr13exception_ptr9_M_addrefEv"); void ExceptionPointerRelease(std::exception_ptr* self) noexcept asm("_ZNSt15__exception_ptr13exception_ptr10_M_releaseEv"); @@ -79,3 +80,53 @@ const type_info* exception_ptr::__cxa_exception_type() const noexcept { } } + +#else + +namespace { +struct GuestExceptionPointer { + void* object {}; + explicit GuestExceptionPointer(void* primary) noexcept : object(primary) {} + GuestExceptionPointer(const GuestExceptionPointer&) = delete; + ~GuestExceptionPointer() {} +}; +} + +extern "C" { +void APS5_VABI _ZNSt15__exception_ptr13exception_ptr9_M_addrefEv_nid_postfix(GuestExceptionPointer* self) noexcept { + __cxa_increment_exception_refcount_nid_postfix(self->object); +} + +void APS5_VABI _ZNSt15__exception_ptr13exception_ptr10_M_releaseEv_nid_postfix(GuestExceptionPointer* self) noexcept { + __cxa_decrement_exception_refcount_nid_postfix(self->object); + self->object = nullptr; +} + +void* APS5_VABI _ZNKSt15__exception_ptr13exception_ptr6_M_getEv_nid_postfix(const GuestExceptionPointer* self) noexcept { + return self->object; +} + +void APS5_VABI _ZNSt15__exception_ptr13exception_ptrC1EPv_nid_postfix(GuestExceptionPointer* self, void* exception) noexcept { + self->object = exception; + __cxa_increment_exception_refcount_nid_postfix(exception); +} + +void APS5_VABI _ZNSt15__exception_ptr13exception_ptrC2EPv_nid_postfix(GuestExceptionPointer* self, void* exception) noexcept { + _ZNSt15__exception_ptr13exception_ptrC1EPv_nid_postfix(self, exception); +} + +const std::type_info* APS5_VABI _ZNKSt15__exception_ptr13exception_ptr20__cxa_exception_typeEv_nid_postfix(const GuestExceptionPointer* self) noexcept { + return LibcException::FromObject(self->object)->type; +} + +GuestExceptionPointer APS5_VABI _ZSt17current_exceptionv_nid_postfix() noexcept { + return GuestExceptionPointer(__cxa_current_primary_exception_nid_postfix()); +} + +[[noreturn]] void APS5_VABI _ZSt17rethrow_exceptionNSt15__exception_ptr13exception_ptrE_nid_postfix(GuestExceptionPointer& exception) { + __cxa_rethrow_primary_exception_nid_postfix(exception.object); + LibcException::Terminate(); +} +} + +#endif diff --git a/core/libs/prx/libc/src/exception/Personality.cpp b/core/libs/prx/libc/src/exception/Personality.cpp index 26377ad0c..87826a2dc 100644 --- a/core/libs/prx/libc/src/exception/Personality.cpp +++ b/core/libs/prx/libc/src/exception/Personality.cpp @@ -71,14 +71,7 @@ extern "C" _Unwind_Reason_Code APS5_VABI __gxx_personality_v0_nid_postfix( return _URC_CONTINUE_UNWIND; } -extern "C" _Unwind_Reason_Code __gxx_personality_v0( - int version, _Unwind_Action actions, std::uint64_t exceptionClass, - _Unwind_Exception* exception, _Unwind_Context* context -) { - return __gxx_personality_v0_nid_postfix(version, actions, exceptionClass, exception, context); -} - -extern "C" _Unwind_Reason_Code __gcc_personality_v0( +extern "C" _Unwind_Reason_Code APS5_VABI __gcc_personality_v0_nid_postfix( int version, _Unwind_Action actions, std::uint64_t, _Unwind_Exception* exception, _Unwind_Context* context ) { @@ -107,3 +100,19 @@ extern "C" _Unwind_Reason_Code __gcc_personality_v0( } return _URC_CONTINUE_UNWIND; } + +#ifndef __APPLE__ +extern "C" _Unwind_Reason_Code __gxx_personality_v0( + int version, _Unwind_Action actions, std::uint64_t exceptionClass, + _Unwind_Exception* exception, _Unwind_Context* context +) { + return __gxx_personality_v0_nid_postfix(version, actions, exceptionClass, exception, context); +} + +extern "C" _Unwind_Reason_Code __gcc_personality_v0( + int version, _Unwind_Action actions, std::uint64_t exceptionClass, + _Unwind_Exception* exception, _Unwind_Context* context +) { + return __gcc_personality_v0_nid_postfix(version, actions, exceptionClass, exception, context); +} +#endif diff --git a/core/libs/prx/libc/src/exception/StandardExceptions.cpp b/core/libs/prx/libc/src/exception/StandardExceptions.cpp index 16da81ad5..b8bf43028 100644 --- a/core/libs/prx/libc/src/exception/StandardExceptions.cpp +++ b/core/libs/prx/libc/src/exception/StandardExceptions.cpp @@ -82,6 +82,7 @@ const char* PlainWhat(const ExceptionObject* object) { if (std::strcmp(name, "St9bad_alloc") == 0) return "std::bad_alloc"; if (std::strcmp(name, "St20bad_array_new_length") == 0) return "std::bad_array_new_length"; if (std::strcmp(name, "St17bad_function_call") == 0) return "std::bad_function_call"; + if (std::strcmp(name, "St12bad_weak_ptr") == 0) return "bad_weak_ptr"; return name; } const char* MessageWhat(const ExceptionObject* object) { return object->message ? object->message : ""; } @@ -175,6 +176,13 @@ const char* APS5_VABI _ZNKSt17bad_function_call4whatEv_nid_postfix(const LibcExc void APS5_VABI _ZNSt17bad_function_callC1Ev_nid_postfix(LibcException::ExceptionObject* self) { self->vtable = &_ZTVSt17bad_function_call_nid_postfix.destroy; } void APS5_VABI _ZNSt17bad_function_callC2Ev_nid_postfix(LibcException::ExceptionObject* self) { self->vtable = &_ZTVSt17bad_function_call_nid_postfix.destroy; } +LibcException::TypeRecord _ZTISt12bad_weak_ptr_nid_postfix {LibcException::SingleTypeVtable + 2, "St12bad_weak_ptr", &_ZTISt9exception_nid_postfix}; +LibcException::ExceptionVtable _ZTVSt12bad_weak_ptr_nid_postfix {0, &_ZTISt12bad_weak_ptr_nid_postfix, LibcException::DestroyPlain, LibcException::DeletePlain, LibcException::PlainWhat}; +void APS5_VABI _ZNSt12bad_weak_ptrD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } +void APS5_VABI _ZNSt12bad_weak_ptrD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } +void APS5_VABI _ZNSt12bad_weak_ptrD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeletePlain(self); } +const char* APS5_VABI _ZNKSt12bad_weak_ptr4whatEv_nid_postfix(const LibcException::ExceptionObject* self) { return LibcException::PlainWhat(self); } + LibcException::TypeRecord _ZTISt11logic_error_nid_postfix {LibcException::SingleTypeVtable + 2, "St11logic_error", &_ZTISt9exception_nid_postfix}; LibcException::ExceptionVtable _ZTVSt11logic_error_nid_postfix {0, &_ZTISt11logic_error_nid_postfix, LibcException::DestroyMessage, LibcException::DeleteMessage, LibcException::MessageWhat}; void APS5_VABI _ZNSt11logic_errorD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } @@ -289,6 +297,8 @@ const char* APS5_VABI _ZNKSt8ios_base7failure4whatEv_nid_postfix(const LibcExcep [[noreturn]] void APS5_VABI _ZSt18_Xinvalid_argumentPKc_nid_postfix(const char* message) { LibcException::ThrowMessage(_ZTVSt16invalid_argument_nid_postfix, message); } [[noreturn]] void APS5_VABI _ZSt13_Xrange_errorPKc_nid_postfix(const char* message) { LibcException::ThrowMessage(_ZTVSt11range_error_nid_postfix, message); } [[noreturn]] void APS5_VABI _ZSt19_Xbad_function_callv_nid_postfix() { LibcException::ThrowPlain(_ZTVSt17bad_function_call_nid_postfix); } +[[noreturn]] void APS5_VABI _ZNSt8__sce_v219_Xbad_function_callEv_nid_postfix() { LibcException::ThrowPlain(_ZTVSt17bad_function_call_nid_postfix); } +[[noreturn]] void APS5_VABI _ZSt19_Throw_bad_weak_ptrv_nid_postfix() { LibcException::ThrowPlain(_ZTVSt12bad_weak_ptr_nid_postfix); } [[noreturn]] void APS5_VABI _ZSt13_Xregex_errorNSt15regex_constants10error_typeE_nid_postfix(std::regex_constants::error_type code) { struct RegexObject { LibcException::ExceptionObject base; std::regex_constants::error_type code; }; auto* object = static_cast(__cxa_allocate_exception_nid_postfix(sizeof(RegexObject))); diff --git a/core/libs/prx/libc/src/exception/Unwind.cpp b/core/libs/prx/libc/src/exception/Unwind.cpp index 53b741735..9da514f5b 100644 --- a/core/libs/prx/libc/src/exception/Unwind.cpp +++ b/core/libs/prx/libc/src/exception/Unwind.cpp @@ -5,13 +5,16 @@ #ifdef _WIN32 #include #endif +#ifdef __APPLE__ +#include +#include +#include +#endif -#if defined(__linux__) || defined(_WIN32) +#if defined(__linux__) || defined(_WIN32) || defined(__APPLE__) -#if defined(_WIN32) || defined(__linux__) extern "C" _Unwind_Reason_Code __gxx_personality_v0(int, _Unwind_Action, std::uint64_t, _Unwind_Exception*, _Unwind_Context*); extern "C" _Unwind_Reason_Code __gcc_personality_v0(int, _Unwind_Action, std::uint64_t, _Unwind_Exception*, _Unwind_Context*); -#endif namespace LibcUnwind { _Unwind_Reason_Code CallPersonality(Word personality, _Unwind_Action actions, _Unwind_Exception* exception, _Unwind_Context* context) { @@ -25,13 +28,13 @@ _Unwind_Reason_Code CallPersonality(Word personality, _Unwind_Action actions, _U #endif if (personality == reinterpret_cast(__gxx_personality_v0_nid_postfix) || personality == reinterpret_cast(__gxx_personality_v0)) return __gxx_personality_v0_nid_postfix(1, actions, exception->exception_class, exception, context); - if (personality == reinterpret_cast(__gcc_personality_v0)) - return __gcc_personality_v0(1, actions, exception->exception_class, exception, context); + if (personality == reinterpret_cast(__gcc_personality_v0_nid_postfix) || personality == reinterpret_cast(__gcc_personality_v0)) + return __gcc_personality_v0_nid_postfix(1, actions, exception->exception_class, exception, context); return (actions & _UA_SEARCH_PHASE) ? _URC_FATAL_PHASE1_ERROR : _URC_FATAL_PHASE2_ERROR; } struct Lookup { Word pc; const Byte* fde {}; Word text {}; Word data {}; }; -#ifdef __linux__ +#if defined(__linux__) || defined(__APPLE__) int FindFrame(dl_phdr_info* info, std::size_t, void* argument) { auto& query = *static_cast(argument); const Byte* header = nullptr; @@ -141,8 +144,32 @@ bool DecodeCandidate(_Unwind_Context& context, Frame& frame, const Lookup& query bool DecodeFrame(_Unwind_Context& context, Frame& frame) { Lookup query {context.registers[16] - !context.signalFrame}; -#ifdef __linux__ +#if defined(__linux__) || defined(__APPLE__) dl_iterate_phdr(FindFrame, &query); +#ifdef __APPLE__ + if (!query.fde) { + Dl_info image {}; + if (!dladdr(reinterpret_cast(query.pc), &image) || !image.dli_fbase) return false; + unsigned long size = 0; + const Byte* p = getsectiondata(static_cast(image.dli_fbase), "__TEXT", "__eh_frame", &size); + if (!p) return false; + const Byte* end = p + size; + while (end - p >= 8) { + const Byte* record = p; + const auto length = Read(p); + if (length == 0) return false; + if (length == 0xffffffff || length < 4 || Word(end - p) < length) return false; + const Byte* next = p + length; + if (Read(p)) { + query.fde = record; + frame = {}; + if (DecodeCandidate(context, frame, query)) return true; + } + p = next; + } + return false; + } +#endif return DecodeCandidate(context, frame, query); #else MEMORY_BASIC_INFORMATION memory{}; diff --git a/core/libs/prx/libc/src/specifics/linux/DlIteratePhdr.cpp b/core/libs/prx/libc/src/specifics/linux/DlIteratePhdr.cpp index 5556e14d3..cc2ff7b9e 100644 --- a/core/libs/prx/libc/src/specifics/linux/DlIteratePhdr.cpp +++ b/core/libs/prx/libc/src/specifics/linux/DlIteratePhdr.cpp @@ -7,7 +7,7 @@ extern "C" { int APS5_VABI dl_iterate_phdr_nid_postfix(int (*callback)(dl_phdr_info*, std::size_t, void*), void* data) { -#if defined(__linux__) +#if defined(__linux__) || defined(__APPLE__) return dl_iterate_phdr(callback, data); #else (void)callback; diff --git a/core/libs/prx/libc/src/specifics/macos/GuestImages.cpp b/core/libs/prx/libc/src/specifics/macos/GuestImages.cpp new file mode 100644 index 000000000..57a60699f --- /dev/null +++ b/core/libs/prx/libc/src/specifics/macos/GuestImages.cpp @@ -0,0 +1,42 @@ +#ifdef __APPLE__ + +#include "prx/libc/include/specifics/linux/ElfTypes.hpp" +#include +#include +#include +#include + +namespace { + +constexpr std::uint64_t MetaVersion = 2; +constexpr std::size_t LoadBiasOffset = 0x08; +constexpr std::size_t HeaderCountOffset = 0x58; +constexpr std::size_t HeadersOffset = 0x60; + +} + +extern "C" int dl_iterate_phdr(int (*callback)(dl_phdr_info*, std::size_t, void*), void* data) { + for (std::uint32_t index = 0; index < _dyld_image_count(); ++index) { + const auto* header = reinterpret_cast(_dyld_get_image_header(index)); + if (header == nullptr || header->magic != MH_MAGIC_64 || (header->flags & MH_DYLIB_IN_CACHE) != 0) continue; + unsigned long size = 0; + const auto* meta = getsectiondata(header, "__APS5DATA", "__meta", &size); + if (meta == nullptr || size < HeadersOffset) continue; + std::uint64_t version; + std::uint64_t loadBias; + std::uint64_t count; + std::memcpy(&version, meta, sizeof(version)); + std::memcpy(&loadBias, meta + LoadBiasOffset, sizeof(loadBias)); + std::memcpy(&count, meta + HeaderCountOffset, sizeof(count)); + if (version != MetaVersion || count > 0xffff || count > (size - HeadersOffset) / sizeof(Elf64_Phdr)) continue; + dl_phdr_info info {}; + info.dlpi_addr = static_cast(loadBias); + info.dlpi_name = header->filetype == MH_EXECUTE ? "" : _dyld_get_image_name(index); + info.dlpi_phdr = reinterpret_cast(meta + HeadersOffset); + info.dlpi_phnum = static_cast(count); + if (const int result = callback(&info, sizeof(info), data); result != 0) return result; + } + return 0; +} + +#endif diff --git a/core/libs/prx/libkernel/Aio/src/Aio.cpp b/core/libs/prx/libkernel/Aio/src/Aio.cpp index 0dc7d97d8..bee34799c 100644 --- a/core/libs/prx/libkernel/Aio/src/Aio.cpp +++ b/core/libs/prx/libkernel/Aio/src/Aio.cpp @@ -13,9 +13,9 @@ #include "prx/libkernel/KernelErrors.hpp" #ifdef _WIN32 -#include #include #include +#include #else #include #include @@ -54,6 +54,35 @@ void SetState(std::int32_t id, std::int32_t state) { g_states[id] = state; } +#ifdef _WIN32 +std::int64_t NativePositioned(std::int32_t fd, void* buf, std::size_t nbyte, std::int64_t offset, bool write, const char* name) { + if (nbyte > static_cast(std::numeric_limits::max())) { + throw std::runtime_error(std::string(name) + ": nbytes exceeds platform limit"); + } + if (offset < 0) { + errno = EINVAL; + return -1; + } + const auto handle = reinterpret_cast(::_get_osfhandle(fd)); + if (handle == INVALID_HANDLE_VALUE) { + errno = EBADF; + return -1; + } + OVERLAPPED overlapped{}; + overlapped.Offset = static_cast(offset); + overlapped.OffsetHigh = static_cast(static_cast(offset) >> 32u); + DWORD done = 0; + const BOOL ok = write ? ::WriteFile(handle, buf, static_cast(nbyte), &done, &overlapped) + : ::ReadFile(handle, buf, static_cast(nbyte), &done, &overlapped); + if (!ok) { + if (!write && ::GetLastError() == ERROR_HANDLE_EOF) return 0; + errno = EIO; + return -1; + } + return done; +} +#endif + std::int64_t NativePread(std::int32_t fd, void* buf, std::size_t nbyte, std::int64_t offset) { const GuestArena::HostWrite destination(buf, nbyte); if (!destination.Open()) { @@ -61,22 +90,7 @@ std::int64_t NativePread(std::int32_t fd, void* buf, std::size_t nbyte, std::int return -1; } #ifdef _WIN32 - if (nbyte > static_cast(std::numeric_limits::max())) { - throw std::runtime_error("sceKernelAioSubmitReadCommands: nbytes exceeds platform limit"); - } - const int duped = ::_dup(fd); - if (duped < 0) return -1; - if (::_lseeki64(duped, offset, SEEK_SET) < 0) { - const int error = errno; - ::_close(duped); - errno = error; - return -1; - } - const int result = ::_read(duped, buf, static_cast(nbyte)); - const int error = errno; - ::_close(duped); - errno = error; - return result; + return NativePositioned(fd, buf, nbyte, offset, false, "sceKernelAioSubmitReadCommands"); #else return static_cast(::pread(fd, buf, nbyte, static_cast(offset))); #endif @@ -84,22 +98,7 @@ std::int64_t NativePread(std::int32_t fd, void* buf, std::size_t nbyte, std::int std::int64_t NativePwrite(std::int32_t fd, const void* buf, std::size_t nbyte, std::int64_t offset) { #ifdef _WIN32 - if (nbyte > static_cast(std::numeric_limits::max())) { - throw std::runtime_error("sceKernelAioSubmitWriteCommands: nbytes exceeds platform limit"); - } - const int duped = ::_dup(fd); - if (duped < 0) return -1; - if (::_lseeki64(duped, offset, SEEK_SET) < 0) { - const int error = errno; - ::_close(duped); - errno = error; - return -1; - } - const int result = ::_write(duped, buf, static_cast(nbyte)); - const int error = errno; - ::_close(duped); - errno = error; - return result; + return NativePositioned(fd, const_cast(buf), nbyte, offset, true, "sceKernelAioSubmitWriteCommands"); #else return static_cast(::pwrite(fd, buf, nbyte, static_cast(offset))); #endif diff --git a/core/libs/prx/libkernel/AppMetadata/include/AppMetadata.hpp b/core/libs/prx/libkernel/AppMetadata/include/AppMetadata.hpp index dc5540138..d18f2e520 100644 --- a/core/libs/prx/libkernel/AppMetadata/include/AppMetadata.hpp +++ b/core/libs/prx/libkernel/AppMetadata/include/AppMetadata.hpp @@ -23,6 +23,7 @@ AppTitleId GetAppTitleId_nid_postfix(); bool HasAppIcon_nid_postfix(); AppIconData GetAppIconData_nid_postfix(); std::uint64_t GetAppDownloadDataSizeMiB_nid_postfix(); +std::int32_t GetAppUserDefinedParam_nid_postfix(std::uint32_t index); } diff --git a/core/libs/prx/libkernel/AppMetadata/include/ParamJsonParser.hpp b/core/libs/prx/libkernel/AppMetadata/include/ParamJsonParser.hpp index a52b5da50..1737a427d 100644 --- a/core/libs/prx/libkernel/AppMetadata/include/ParamJsonParser.hpp +++ b/core/libs/prx/libkernel/AppMetadata/include/ParamJsonParser.hpp @@ -9,6 +9,7 @@ struct ParsedParamJson { std::string title; std::string titleId; std::uint64_t downloadDataSizeMiB = 0; + std::int32_t userDefinedParams[4] = {}; }; ParsedParamJson parseParamJson(const std::filesystem::path& paramJsonPath); diff --git a/core/libs/prx/libkernel/AppMetadata/src/AppMetadata.cpp b/core/libs/prx/libkernel/AppMetadata/src/AppMetadata.cpp index b99d5aa7f..bc423779a 100644 --- a/core/libs/prx/libkernel/AppMetadata/src/AppMetadata.cpp +++ b/core/libs/prx/libkernel/AppMetadata/src/AppMetadata.cpp @@ -18,6 +18,7 @@ bool g_iconLoaded = false; char g_title[128] = {}; char g_titleId[12] = {}; std::uint64_t g_downloadDataSizeMiB = 0; +std::int32_t g_userDefinedParams[4] = {}; std::vector g_iconBytes; constexpr const char* AppMetadataParamJsonGuestPath = "/app0/sce_sys/param.json"; @@ -37,6 +38,7 @@ void ensureTitleLoaded() { copyToFixedBuffer(g_title, sizeof(g_title), parsed.title); copyToFixedBuffer(g_titleId, sizeof(g_titleId), parsed.titleId); g_downloadDataSizeMiB = parsed.downloadDataSizeMiB; + std::memcpy(g_userDefinedParams, parsed.userDefinedParams, sizeof(g_userDefinedParams)); g_titleLoaded = true; } @@ -75,6 +77,12 @@ std::uint64_t GetAppDownloadDataSizeMiB_nid_postfix() { return g_downloadDataSizeMiB; } +std::int32_t GetAppUserDefinedParam_nid_postfix(std::uint32_t index) { + if (index >= 4) throw std::out_of_range("user defined param index out of range"); + ensureTitleLoaded(); + return g_userDefinedParams[index]; +} + AppTitleId GetAppTitleId_nid_postfix() { ensureTitleLoaded(); AppTitleId result{}; diff --git a/core/libs/prx/libkernel/AppMetadata/src/ParamJsonParser.cpp b/core/libs/prx/libkernel/AppMetadata/src/ParamJsonParser.cpp index a584cf3b2..3645c476d 100644 --- a/core/libs/prx/libkernel/AppMetadata/src/ParamJsonParser.cpp +++ b/core/libs/prx/libkernel/AppMetadata/src/ParamJsonParser.cpp @@ -181,7 +181,12 @@ JsonValue parseJsonNumber(const std::string& text, std::size_t& position) { const std::size_t start = position; if (peekChar(text, position) == '-') ++position; if (position >= text.size() || text[position] < '0' || text[position] > '9') throw std::runtime_error("malformed number in param.json"); - while (position < text.size() && text[position] >= '0' && text[position] <= '9') ++position; + if (text[position] == '0') { + ++position; + if (position < text.size() && text[position] >= '0' && text[position] <= '9') throw std::runtime_error("malformed number in param.json"); + } else { + while (position < text.size() && text[position] >= '0' && text[position] <= '9') ++position; + } if (position < text.size() && text[position] == '.') { ++position; if (position >= text.size() || text[position] < '0' || text[position] > '9') throw std::runtime_error("malformed number in param.json"); @@ -282,5 +287,14 @@ ParsedParamJson parseParamJson(const std::filesystem::path& paramJsonPath) { result.downloadDataSizeMiB = static_cast(downloadData->numberValue); } } + for (int index = 0; index < 4; ++index) { + const std::string key = "userDefinedParam" + std::to_string(index + 1); + const JsonValue* param = findObjectMember(root, key); + if (param == nullptr) continue; + const auto& number = param->numberText; + if (param->type != JsonType::Number || number.find_first_of(".eE") != std::string::npos) throw std::runtime_error("param.json " + key + " is not an integer"); + const auto parsed = std::from_chars(number.data(), number.data() + number.size(), result.userDefinedParams[index]); + if (parsed.ec != std::errc{} || parsed.ptr != number.data() + number.size()) throw std::runtime_error("param.json " + key + " exceeds int32 range"); + } return result; } diff --git a/core/libs/prx/libkernel/Apr/src/Apr.cpp b/core/libs/prx/libkernel/Apr/src/Apr.cpp index 0fe1b7c5b..a363671b9 100644 --- a/core/libs/prx/libkernel/Apr/src/Apr.cpp +++ b/core/libs/prx/libkernel/Apr/src/Apr.cpp @@ -590,7 +590,7 @@ void _execute(const Apr::CommandBufferObject& buffer) { } case Apr::Opcode::WriteKernelEventQueue: { const auto command = _read(buffer, cursor); - EqueueTriggerEvent_nid_postfix(static_cast(command.equeue), static_cast(command.ident), EVFILT_USER, reinterpret_cast(command.data)); + EqueueTriggerEvent_nid_postfix(static_cast(command.equeue), static_cast(command.ident), EVFILT_AMPR, reinterpret_cast(command.data)); break; } case Apr::Opcode::WriteAddressFromTimeCounter: { diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index f16f611dd..57dd08890 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -96,12 +96,17 @@ if(BUILD_TESTING) target_link_options(guest_module_fixture PRIVATE -Wl,--no-as-needed) add_test(NAME nid_patcher_dynamic_strings COMMAND ${Python3_EXECUTABLE} ${LIBS_INCLUDE_DIR}/nid/tests/test_dynamic_strings.py $) + add_test(NAME nid_patcher_symbol_versions COMMAND ${Python3_EXECUTABLE} + ${LIBS_INCLUDE_DIR}/nid/tests/test_symbol_versions.py $) endif() add_test_executable(guest_dynamic_loader_tests ${LIBS_INCLUDE_DIR}/tests/GuestDynamicLoader.cpp) target_include_directories(guest_dynamic_loader_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_dynamic_loader_tests PRIVATE libkernel libc) configure_windows_unwind(guest_dynamic_loader_tests) add_dependencies(guest_dynamic_loader_tests guest_module_fixture) + if(NOT WIN32) + set_target_properties(guest_dynamic_loader_tests PROPERTIES ENABLE_EXPORTS ON) + endif() add_test(NAME guest_dynamic_loader COMMAND guest_dynamic_loader_tests $) set_tests_properties(guest_dynamic_loader PROPERTIES WORKING_DIRECTORY "$") @@ -127,6 +132,15 @@ if(BUILD_TESTING) set_tests_properties(guest_filesystem PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_kernel_sync_tests ${LIBS_INCLUDE_DIR}/tests/GuestKernelSync.cpp) + target_include_directories(guest_kernel_sync_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_kernel_sync_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_kernel_sync_tests) + add_test(NAME guest_kernel_sync COMMAND guest_kernel_sync_tests) + if(WIN32) + set_tests_properties(guest_kernel_sync PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_aio_tests ${LIBS_INCLUDE_DIR}/tests/GuestAio.cpp) target_include_directories(guest_aio_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_aio_tests PRIVATE libkernel libc) @@ -196,6 +210,16 @@ if(BUILD_TESTING) set_tests_properties(guest_cond_timedwait PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_cond_recursive_mutex_tests ${LIBS_INCLUDE_DIR}/tests/GuestCondRecursiveMutex.cpp) + target_include_directories(guest_cond_recursive_mutex_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_cond_recursive_mutex_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_cond_recursive_mutex_tests) + add_test(NAME guest_cond_recursive_mutex COMMAND guest_cond_recursive_mutex_tests) + set_tests_properties(guest_cond_recursive_mutex PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_cond_recursive_mutex PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_cpu_time_clocks_tests ${LIBS_INCLUDE_DIR}/tests/GuestCpuTimeClocks.cpp) target_include_directories(guest_cpu_time_clocks_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_cpu_time_clocks_tests PRIVATE libkernel libc) @@ -293,6 +317,18 @@ if(BUILD_TESTING) set_tests_properties(guest_raise_exception PROPERTIES TIMEOUT 30 ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + if(WIN32) + add_library(guest_unwind_module_info_fixture SHARED ${LIBS_INCLUDE_DIR}/tests/GuestUnwindModuleInfoFixture.cpp) + target_include_directories(guest_unwind_module_info_fixture PRIVATE ${LIBS_INCLUDE_DIR}) + target_compile_options(guest_unwind_module_info_fixture PRIVATE -fno-exceptions) + add_test_executable(guest_unwind_module_info_tests ${LIBS_INCLUDE_DIR}/tests/GuestUnwindModuleInfo.cpp) + target_include_directories(guest_unwind_module_info_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_unwind_module_info_tests PRIVATE libkernel libc guest_unwind_module_info_fixture) + configure_windows_unwind(guest_unwind_module_info_tests) + add_test(NAME guest_unwind_module_info COMMAND guest_unwind_module_info_tests) + set_tests_properties(guest_unwind_module_info PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + endif() add_test_executable(guest_sync_on_address_tests ${LIBS_INCLUDE_DIR}/tests/GuestSyncOnAddress.cpp) target_include_directories(guest_sync_on_address_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_sync_on_address_tests PRIVATE libkernel libc) @@ -313,6 +349,16 @@ if(BUILD_TESTING) set_tests_properties(guest_available_cpumask PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_current_cpu_tests ${LIBS_INCLUDE_DIR}/tests/GuestCurrentCpu.cpp) + target_include_directories(guest_current_cpu_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_current_cpu_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_current_cpu_tests) + add_test(NAME guest_current_cpu COMMAND guest_current_cpu_tests) + set_tests_properties(guest_current_cpu PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_current_cpu PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_getrusage_tests ${LIBS_INCLUDE_DIR}/tests/GuestGetrusage.cpp) target_include_directories(guest_getrusage_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_getrusage_tests PRIVATE libkernel libc) @@ -362,6 +408,16 @@ if(BUILD_TESTING) set_tests_properties(guest_udp PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_poll_tests ${LIBS_INCLUDE_DIR}/tests/GuestPoll.cpp) + target_include_directories(guest_poll_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_poll_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_poll_tests) + add_test(NAME guest_poll COMMAND guest_poll_tests) + set_tests_properties(guest_poll PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_poll PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_tcp_tests ${LIBS_INCLUDE_DIR}/tests/GuestTcp.cpp) target_include_directories(guest_tcp_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_tcp_tests PRIVATE libkernel libc) @@ -372,6 +428,16 @@ if(BUILD_TESTING) set_tests_properties(guest_tcp PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_socket_pair_tests ${LIBS_INCLUDE_DIR}/tests/GuestSocketPair.cpp) + target_include_directories(guest_socket_pair_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_socket_pair_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_socket_pair_tests) + add_test(NAME guest_socket_pair COMMAND guest_socket_pair_tests) + set_tests_properties(guest_socket_pair PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_socket_pair PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_elf_phdr_match_tests ${LIBS_INCLUDE_DIR}/tests/GuestElfPhdrMatch.cpp) target_include_directories(guest_elf_phdr_match_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_elf_phdr_match_tests PRIVATE libkernel libc) @@ -414,10 +480,6 @@ if(BUILD_TESTING) target_include_directories(guest_directory_entry_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_directory_entry_tests PRIVATE libkernel libc) add_test(NAME guest_directory_entries COMMAND guest_directory_entry_tests) - add_test_executable(guest_file_vector_tests ${LIBS_INCLUDE_DIR}/tests/GuestFileVector.cpp) - target_include_directories(guest_file_vector_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_file_vector_tests PRIVATE libkernel libc) - add_test(NAME guest_file_vector COMMAND guest_file_vector_tests) else() add_test_executable(guest_directory_close_tests ${LIBS_INCLUDE_DIR}/tests/GuestDirectoryClose.cpp) target_include_directories(guest_directory_close_tests PRIVATE ${LIBS_INCLUDE_DIR}) @@ -430,4 +492,13 @@ if(BUILD_TESTING) target_compile_options(guest_directory_close_tests PRIVATE -fasynchronous-unwind-tables) endif() endif() + add_test_executable(guest_file_vector_tests ${LIBS_INCLUDE_DIR}/tests/GuestFileVector.cpp) + target_include_directories(guest_file_vector_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_file_vector_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_file_vector_tests) + add_test(NAME guest_file_vector COMMAND guest_file_vector_tests) + if(WIN32) + set_tests_properties(guest_file_vector PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp index 297dd9d94..cd7bf920b 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp @@ -615,6 +615,30 @@ bool Reserved(const void* addr, size_t len) { return false; } +void UnmapRegistered(GuestAllocations::Mutation& mutation, void* addr, size_t len) { + mutation.Unmap(addr, len, [&](const void* piece, std::size_t pieceBytes, const void* allocation, bool last) { + auto* pieceAddress = const_cast(piece); + std::lock_guard lock(g_directLock); + EraseMappings(reinterpret_cast(piece), reinterpret_cast(piece) + pieceBytes); +#if defined(__linux__) + Unmap(pieceAddress, pieceBytes); +#else + if (KernelArena::Get().Contains(pieceAddress, pieceBytes)) munmap(pieceAddress, pieceBytes); + else if (last) munmap_release(const_cast(allocation)); + else munmap(pieceAddress, pieceBytes); +#endif + }); + EraseReservations(addr, len); + RecordProtection(addr, len, -1); +} + +void ReplaceFixedOverlap(GuestAllocations::Mutation& mutation, void* addr, size_t len, int flags) { + constexpr int GuestMapNoOverwrite = 0x80; + if (addr == nullptr || (flags & GuestMapFixedFlag) == 0) return; + if ((flags & GuestMapNoOverwrite) == 0 && mutation.Overlaps(addr, len)) UnmapRegistered(mutation, addr, len); + mutation.RequireAvailable(addr, len); +} + bool FixedNoOverwriteConflict(const GuestAllocations::Mutation& mutation, void* addr, size_t len, int flags) { constexpr int GuestMapNoOverwrite = 0x80; if (addr == nullptr || (flags & GuestMapFixedFlag) == 0 || (flags & GuestMapNoOverwrite) == 0) return false; @@ -634,7 +658,7 @@ int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart, RecordProtection(*addr, len, prot); return 0; } - if (*addr != nullptr && (flags & GuestMapFixedFlag) != 0) mutation.RequireAvailable(*addr, len); + ReplaceFixedOverlap(mutation, *addr, len, flags); std::lock_guard lock(g_directLock); ValidatePhysicalRange(static_cast(physStart), len); void* mapped = MapAligned(*addr, len, PROT_NONE, flags, alignment); @@ -663,7 +687,7 @@ int DoMapAnon(void** addr, size_t len, int prot, int flags) { RecordProtection(*addr, len, prot); return 0; } - if (*addr != nullptr && (flags & GuestMapFixedFlag) != 0) mutation.RequireAvailable(*addr, len); + ReplaceFixedOverlap(mutation, *addr, len, flags); void* mapped = MapAligned(*addr, len, LinuxProtFromSce(prot), flags, PS5_PAGE_SIZE); try { mutation.Add(mapped, len, (prot & 3) != 0, (prot & 2) != 0); @@ -738,20 +762,7 @@ int DoMunmap(void* addr, size_t len) { Trace("unmap %p+0x%zx", addr, len); if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0 || !addr) return SCE_KERNEL_ERROR_EINVAL; GuestAllocations::Mutation mutation; - mutation.Unmap(addr, len, [&](const void* piece, std::size_t pieceBytes, const void* allocation, bool last) { - auto* pieceAddress = const_cast(piece); - std::lock_guard lock(g_directLock); - EraseMappings(reinterpret_cast(piece), reinterpret_cast(piece) + pieceBytes); -#if defined(__linux__) - Unmap(pieceAddress, pieceBytes); -#else - if (KernelArena::Get().Contains(pieceAddress, pieceBytes)) munmap(pieceAddress, pieceBytes); - else if (last) munmap_release(const_cast(allocation)); - else munmap(pieceAddress, pieceBytes); -#endif - }); - EraseReservations(addr, len); - RecordProtection(addr, len, -1); + UnmapRegistered(mutation, addr, len); return 0; } diff --git a/core/libs/prx/libkernel/DirectMemory/Export.cpp b/core/libs/prx/libkernel/DirectMemory/Export.cpp index 0b20146d0..8d893ee6c 100644 --- a/core/libs/prx/libkernel/DirectMemory/Export.cpp +++ b/core/libs/prx/libkernel/DirectMemory/Export.cpp @@ -196,7 +196,8 @@ int32_t APS5_VABI sceKernelMapNamedFlexibleMemory(void** addr_in_out, size_t len } int32_t APS5_VABI sceKernelMapNamedFlexibleMemoryInternal(void** addr_in_out, size_t len, int prot, int flags, const char* name) { - return sceKernelMapNamedFlexibleMemory(addr_in_out, len, prot, flags, name); + constexpr int IgnoredInternalFlag = 0x8000; + return sceKernelMapNamedFlexibleMemory(addr_in_out, len, prot, flags & ~IgnoredInternalFlag, name); } int APS5_VABI sceKernelMprotect(const void* addr, size_t len, int prot) { diff --git a/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp b/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp index e3d2f1796..bac216bbc 100644 --- a/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp @@ -78,4 +78,23 @@ int APS5_VABI munmap_nid_postfix(void* address, std::size_t length) noexcept { } } +int APS5_VABI mprotect_nid_postfix(void* address, std::size_t length, int protection) noexcept { + const auto failed = [](int error) { + SetError(error); + return -1; + }; + const auto start = reinterpret_cast(address); + if (length == 0) return 0; + if (start == 0 || length > std::numeric_limits::max() - start) + return failed(GuestInvalid); + try { + if (DoMprotect(address, length, protection) != 0) return failed(GuestInvalid); + return 0; + } catch (const std::bad_alloc&) { + return failed(GuestNoMemory); + } catch (const std::exception&) { + return failed(GuestInvalid); + } +} + } diff --git a/core/libs/prx/libkernel/Equeue/Equeue.cpp b/core/libs/prx/libkernel/Equeue/Equeue.cpp index 7d4769d6e..b8d41b762 100644 --- a/core/libs/prx/libkernel/Equeue/Equeue.cpp +++ b/core/libs/prx/libkernel/Equeue/Equeue.cpp @@ -441,7 +441,7 @@ int APS5_VABI sceKernelAddAmprEvent(KernelEqueue eq, int id, void* udata) { } KernelEqueueEvent event{}; event.event.ident = static_cast(id); - event.event.filter = EVFILT_USER; + event.event.filter = EVFILT_AMPR; event.event.flags = EV_ADD | EV_CLEAR; event.event.udata = udata; event.filter.triggerFunc = [](KernelEqueueEvent* e, void* data) { @@ -473,7 +473,7 @@ int APS5_VABI sceKernelDeleteAmprEvent(KernelEqueue eq, int id) { if (eq == 0) { return EQUEUE_OK; } - EqueueDeleteEvent_nid_postfix(eq, static_cast(id), EVFILT_USER); + EqueueDeleteEvent_nid_postfix(eq, static_cast(id), EVFILT_AMPR); return EQUEUE_OK; } diff --git a/core/libs/prx/libkernel/Equeue/Equeue.hpp b/core/libs/prx/libkernel/Equeue/Equeue.hpp index c5d4f3e5d..dd3a47706 100644 --- a/core/libs/prx/libkernel/Equeue/Equeue.hpp +++ b/core/libs/prx/libkernel/Equeue/Equeue.hpp @@ -18,6 +18,7 @@ static constexpr int16_t EVFILT_TIMER = -7; static constexpr int16_t EVFILT_USER = -11; static constexpr int16_t EVFILT_VIDEO_OUT = -13; static constexpr int16_t EVFILT_HRTIMER = -15; +static constexpr int16_t EVFILT_AMPR = -25; static constexpr uint16_t EV_ADD = 0x0001; static constexpr uint16_t EV_ONESHOT = 0x0010; diff --git a/core/libs/prx/libkernel/File/src/Open.cpp b/core/libs/prx/libkernel/File/src/Open.cpp index 807fd4bba..2e323a8d9 100644 --- a/core/libs/prx/libkernel/File/src/Open.cpp +++ b/core/libs/prx/libkernel/File/src/Open.cpp @@ -116,6 +116,8 @@ int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode) { if (fd < 0) { return SceErrorFromErrno(errno); } + if ((flags & SCE_KERNEL_O_ACCMODE) != SCE_KERNEL_O_RDONLY || (flags & (SCE_KERNEL_O_CREAT | SCE_KERNEL_O_TRUNC))) + RecordWrittenPath_nid_no_patch(native); return fd; } @@ -189,6 +191,7 @@ int APS5_VABI sceKernelUnlink(const char* path) { if (NativeUnlink(native) != 0) { return SceErrorFromErrno(errno); } + RecordWrittenPath_nid_no_patch(native); return 0; } diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 7defbc421..109f729e7 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -13,10 +14,12 @@ #include #include #include +#include #include #include #include #include +#include struct KernelIovec { void* base; @@ -40,6 +43,7 @@ static int NativeMkdir(const std::filesystem::path& path, std::uint16_t mode) { return ::_wmkdir(path.wstring().c_str()); } static int NativeChmod(const std::filesystem::path& path, int mode) { + RecordWrittenPath_nid_no_patch(path); return ::_wchmod(path.wstring().c_str(), mode); } static std::optional NativeDescriptorPath(int descriptor) { @@ -70,6 +74,7 @@ static int NativeFtruncate(int descriptor, std::int64_t length) { return static_cast(::_chsize_s(descriptor, length)); } static int NativeUtimes(const std::filesystem::path& path, const KernelTimeval* times) { + RecordWrittenPath_nid_no_patch(path); if (times == nullptr) return ::_wutime(path.wstring().c_str(), nullptr); struct _utimbuf values{static_cast(times[0].tv_sec), static_cast(times[1].tv_sec)}; return ::_wutime(path.wstring().c_str(), &values); @@ -101,19 +106,38 @@ static std::int64_t NativePositioned(int descriptor, void* buf, std::size_t nbyt errno = EBADF; return -1; } + LARGE_INTEGER position{}; + if (::GetFileType(handle) != FILE_TYPE_DISK || !::SetFilePointerEx(handle, LARGE_INTEGER{}, &position, FILE_CURRENT)) { + errno = ESPIPE; + return -1; + } OVERLAPPED overlapped{}; overlapped.Offset = static_cast(offset); overlapped.OffsetHigh = static_cast(static_cast(offset) >> 32u); DWORD done = 0; const BOOL ok = write ? ::WriteFile(handle, buf, static_cast(nbytes), &done, &overlapped) : ::ReadFile(handle, buf, static_cast(nbytes), &done, &overlapped); + const DWORD error = ok ? ERROR_SUCCESS : ::GetLastError(); + if (!::SetFilePointerEx(handle, position, nullptr, FILE_BEGIN)) { + throw std::runtime_error("NativePositioned: cannot restore the file offset"); + } if (!ok) { - if (!write && ::GetLastError() == ERROR_HANDLE_EOF) return 0; + if (!write && error == ERROR_HANDLE_EOF) return 0; errno = EIO; return -1; } return done; } +static std::int64_t NativeTransfer(int descriptor, void* buf, std::size_t nbytes, bool write) { + if (nbytes > static_cast(std::numeric_limits::max())) { + throw std::runtime_error("NativeTransfer: nbytes exceeds platform limit"); + } + return write ? ::_write(descriptor, buf, static_cast(nbytes)) : ::_read(descriptor, buf, static_cast(nbytes)); +} +static bool NativeIsDisk(int descriptor) { + const auto handle = reinterpret_cast(::_get_osfhandle(descriptor)); + return handle != INVALID_HANDLE_VALUE && ::GetFileType(handle) == FILE_TYPE_DISK; +} static std::int64_t NativePread(int descriptor, void* buf, std::size_t nbytes, std::int64_t offset) { return NativePositioned(descriptor, buf, nbytes, offset, false); } @@ -128,8 +152,6 @@ static std::int64_t NativePwrite(int descriptor, const void* buf, std::size_t nb #include #include #include -#include -#include static int NativeRmdir(const std::filesystem::path& path) { return ::rmdir(path.c_str()); } @@ -291,13 +313,12 @@ int64_t APS5_VABI lseek_nid_postfix(int d, int64_t offset, int whence) { } int APS5_VABI mkdir_nid_postfix(const char* path, uint16_t mode) { - if (path == nullptr) { - APS5_INVALID_ARG_EX; - } + if (const int error = PathError(path)) return PosixFailure(error); auto native = ResolvePath_nid_no_patch(path); if (NativeMkdir(native, mode) != 0) { - throw std::runtime_error(std::string(__func__) + ": mkdir failed for " + native.string() + ", errno=" + std::to_string(errno)); + return PosixResult(SceErrorFromErrno(errno)); } + RecordWrittenPath_nid_no_patch(native); return 0; } @@ -405,6 +426,15 @@ int APS5_VABI sceKernelFsync(int fd) { #endif } +int APS5_VABI sceKernelWriteThrottlingStatus(std::uint64_t* status) { + if (status == nullptr) throw std::invalid_argument("sceKernelWriteThrottlingStatus: status is null"); + status[0] = std::numeric_limits::max(); + status[1] = 0; + status[2] = 0; + status[3] = 0; + return 0; +} + #ifdef _WIN32 int APS5_VABI sceKernelGetdirentries(int fd, char* buf, int nbytes, int64_t* basep) { @@ -474,9 +504,23 @@ int APS5_VABI sceKernelMkdir(const char* path, uint16_t mode) { if (std::filesystem::exists(native, error)) return SceErrorFromErrno(GUEST_EEXIST); if (!std::filesystem::exists(native.parent_path(), error)) return SceErrorFromErrno(GUEST_ENOENT); if (!std::filesystem::create_directory(native, error)) return SceErrorFromErrno(error.value() ? error.value() : GUEST_EIO); + RecordWrittenPath_nid_no_patch(native); return 0; } +static int CheckIovecs(const KernelIovec* iov, int iovcnt) { + if (iovcnt < 0 || iovcnt > KERNEL_IOV_MAX) return SceErrorFromErrno(GUEST_EINVAL); + if (iov == nullptr && iovcnt != 0) return SceErrorFromErrno(GUEST_EFAULT); + return 0; +} + +static bool OpenIovecs(const KernelIovec* iov, int iovcnt, std::deque& destinations) { + for (int i = 0; i < iovcnt; ++i) { + if (!destinations.emplace_back(iov[i].base, iov[i].length).Open()) return false; + } + return true; +} + #ifdef _WIN32 int64_t APS5_VABI sceKernelPread(int d, void* buf, size_t nbytes, int64_t offset) { @@ -487,20 +531,53 @@ int64_t APS5_VABI sceKernelPwrite(int d, const void* buf, size_t nbytes, int64_t return pwrite_nid_disambig1_nid_postfix(d, buf, nbytes, offset); } -int64_t APS5_VABI sceKernelReadv(int, const KernelIovec*, int) { - throw std::runtime_error(std::string(__func__) + ": not implemented on Windows"); +static std::int64_t TransferIovecs(int d, const KernelIovec* iov, int iovcnt, const std::int64_t* offset, bool write) { + if (const int error = CheckIovecs(iov, iovcnt)) return error; + if (offset != nullptr && *offset < 0) return SceErrorFromErrno(GUEST_EINVAL); + std::size_t total = 0; + for (int i = 0; i < iovcnt; ++i) { + if (iov[i].base == nullptr && iov[i].length != 0) return SceErrorFromErrno(GUEST_EFAULT); + if (iov[i].length > static_cast(std::numeric_limits::max()) - total) return SceErrorFromErrno(GUEST_EINVAL); + total += iov[i].length; + } + if (offset != nullptr && total > static_cast(std::numeric_limits::max() - *offset)) return SceErrorFromErrno(GUEST_EINVAL); + std::deque destinations; + if (!write && !OpenIovecs(iov, iovcnt, destinations)) return SceErrorFromErrno(GUEST_EFAULT); + if (total == 0) { + char none = 0; + const auto result = offset != nullptr ? NativePositioned(d, &none, 0, *offset, write) : NativeTransfer(d, &none, 0, write); + return result < 0 ? SceErrorFromErrno(errno) : 0; + } + const bool whole = write || offset != nullptr || NativeIsDisk(d); + std::int64_t done = 0; + for (int i = 0; i < iovcnt; ++i) { + auto* base = static_cast(iov[i].base); + for (std::size_t position = 0; position < iov[i].length;) { + const auto chunk = std::min(iov[i].length - position, std::numeric_limits::max()); + const auto result = offset != nullptr ? NativePositioned(d, base + position, chunk, *offset + done, write) : NativeTransfer(d, base + position, chunk, write); + if (result < 0) return done != 0 ? done : SceErrorFromErrno(errno); + done += result; + position += static_cast(result); + if (static_cast(result) < chunk || !whole) return done; + } + } + return done; } -int64_t APS5_VABI sceKernelWritev(int, const KernelIovec*, int) { - throw std::runtime_error(std::string(__func__) + ": not implemented on Windows"); +int64_t APS5_VABI sceKernelReadv(int d, const KernelIovec* iov, int iovcnt) { + return TransferIovecs(d, iov, iovcnt, nullptr, false); } -int64_t APS5_VABI sceKernelPreadv(int, const KernelIovec*, int, int64_t) { - throw std::runtime_error(std::string(__func__) + ": not implemented on Windows"); +int64_t APS5_VABI sceKernelWritev(int d, const KernelIovec* iov, int iovcnt) { + return TransferIovecs(d, iov, iovcnt, nullptr, true); } -int64_t APS5_VABI sceKernelPwritev(int, const KernelIovec*, int, int64_t) { - throw std::runtime_error(std::string(__func__) + ": not implemented on Windows"); +int64_t APS5_VABI sceKernelPreadv(int d, const KernelIovec* iov, int iovcnt, int64_t offset) { + return TransferIovecs(d, iov, iovcnt, &offset, false); +} + +int64_t APS5_VABI sceKernelPwritev(int d, const KernelIovec* iov, int iovcnt, int64_t offset) { + return TransferIovecs(d, iov, iovcnt, &offset, true); } #else @@ -521,19 +598,6 @@ int64_t APS5_VABI sceKernelPwrite(int d, const void* buf, size_t nbytes, int64_t return result < 0 ? SceErrorFromErrno(errno) : result; } -static int CheckIovecs(const KernelIovec* iov, int iovcnt) { - if (iovcnt < 0 || iovcnt > KERNEL_IOV_MAX) return SceErrorFromErrno(GUEST_EINVAL); - if (iov == nullptr && iovcnt != 0) return SceErrorFromErrno(GUEST_EFAULT); - return 0; -} - -static bool OpenIovecs(const KernelIovec* iov, int iovcnt, std::deque& destinations) { - for (int i = 0; i < iovcnt; ++i) { - if (!destinations.emplace_back(iov[i].base, iov[i].length).Open()) return false; - } - return true; -} - int64_t APS5_VABI sceKernelReadv(int d, const KernelIovec* iov, int iovcnt) { if (const int error = CheckIovecs(iov, iovcnt)) return error; std::deque destinations; @@ -571,8 +635,11 @@ int APS5_VABI sceKernelRename(const char* from, const char* to) { const auto source = ResolvePath_nid_no_patch(from); std::error_code error; if (!std::filesystem::exists(source, error)) return SceErrorFromErrno(GUEST_ENOENT); - std::filesystem::rename(source, ResolvePath_nid_no_patch(to), error); + const auto destination = ResolvePath_nid_no_patch(to); + std::filesystem::rename(source, destination, error); if (error) return SceErrorFromErrno(GUEST_EIO); + RecordWrittenPath_nid_no_patch(source); + RecordWrittenPath_nid_no_patch(destination); return 0; } @@ -583,6 +650,7 @@ int APS5_VABI sceKernelRmdir(const char* path) { if (!std::filesystem::is_directory(native, error)) return SceErrorFromErrno(std::filesystem::exists(native, error) ? GUEST_ENOTDIR : GUEST_ENOENT); if (!std::filesystem::is_empty(native, error)) return SceErrorFromErrno(GUEST_ENOTEMPTY); if (!std::filesystem::remove(native, error)) return SceErrorFromErrno(GUEST_EIO); + RecordWrittenPath_nid_no_patch(native); return 0; } @@ -617,6 +685,7 @@ int APS5_VABI sceKernelTruncate_nid_postfix(const char* path, std::int64_t lengt if (!std::filesystem::exists(native, error)) return SceErrorFromErrno(GUEST_ENOENT); std::filesystem::resize_file(native, static_cast(length), error); if (error) return SceErrorFromErrno(GUEST_EIO); + RecordWrittenPath_nid_no_patch(native); return 0; } diff --git a/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp b/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp index d2604c94d..509194a50 100644 --- a/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp +++ b/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp @@ -15,6 +15,7 @@ #include #else #include +#include #endif namespace { @@ -49,6 +50,39 @@ void* Symbol(Module& module, const char* name) { return ::dlsym(module.native, name); #endif } +#ifndef _WIN32 +std::uintptr_t MainProgramBase() { + static const std::uintptr_t base = [] { + struct Scan { + std::uintptr_t value = 0; + }; + Scan scan; + ::dl_iterate_phdr( + [](dl_phdr_info* info, size_t, void* data) { + auto* found = static_cast(data); + if (found->value == 0 && info->dlpi_name != nullptr && info->dlpi_name[0] == '\0') found->value = info->dlpi_addr; + return found->value == 0 ? 0 : 1; + }, + &scan); + return scan.value; + }(); + return base; +} + +bool FromGuestModule(void* address) { + Dl_info info{}; + if (address == nullptr || !::dladdr(address, &info) || info.dli_fname == nullptr) return false; + if (reinterpret_cast(info.dli_fbase) == MainProgramBase()) return true; + const std::string path(info.dli_fname); + return path.ends_with(".prx"); +} + +void* DefaultScopeSymbol(const char* symbolName) { + auto* result = ::dlsym(RTLD_DEFAULT, symbolName); + return FromGuestModule(result) ? result : nullptr; +} +#endif + void* FindSymbol(Module& module, const char* name) { if (auto* symbol = Symbol(module, name)) return symbol; const auto nid = Nid::ComputeNid(name, ""); @@ -116,7 +150,7 @@ void* APS5_VABI dlsym_nid_postfix(void* handle, const char* name) { std::vector> search; { std::lock_guard lock(modulesMutex); - if (handle == reinterpret_cast(static_cast(-2))) { + if (handle == nullptr || handle == reinterpret_cast(static_cast(-2))) { for (const auto& [key, module] : modules) if (module->global) search.push_back(module); } else { auto found = modules.find(reinterpret_cast(handle)); @@ -125,6 +159,13 @@ void* APS5_VABI dlsym_nid_postfix(void* handle, const char* name) { } } for (const auto& module : search) if (auto* result = FindSymbol(*module, name)) return result; +#ifndef _WIN32 + if (handle == nullptr || handle == reinterpret_cast(static_cast(-2))) { + if (auto* result = DefaultScopeSymbol(name)) return result; + const auto nid = Nid::ComputeNid(name, ""); + if (auto* result = DefaultScopeSymbol(nid.c_str())) return result; + } +#endif Error("dlsym: symbol not found in supported module scope"); return nullptr; } catch (const std::exception& error) { Error(error.what()); return nullptr; } diff --git a/core/libs/prx/libkernel/Module/src/Module.cpp b/core/libs/prx/libkernel/Module/src/Module.cpp index 1d52dc2ea..466329d8c 100644 --- a/core/libs/prx/libkernel/Module/src/Module.cpp +++ b/core/libs/prx/libkernel/Module/src/Module.cpp @@ -11,10 +11,67 @@ #ifdef _WIN32 #include #include +#elif defined(__APPLE__) +#include +#include +#include #else #include #endif +#ifdef _WIN32 +namespace { +std::uint64_t ReadEncoded(const std::uint8_t*& p, std::uint8_t encoding) { + const auto application = encoding & 0x70; + if ((encoding & 0x80) != 0 || (application != 0x00 && application != 0x10)) NotImplemented_nid_no_patch("sceKernelGetModuleInfoForUnwind eh_frame_ptr encoding other than absptr or pcrel"); + std::uint64_t value = 0; + const auto* at = p; + switch (encoding & 0x0f) { + case 0x03: { std::uint32_t v; std::memcpy(&v, p, 4); value = v; p += 4; break; } + case 0x0b: { std::int32_t v; std::memcpy(&v, p, 4); value = static_cast(static_cast(v)); p += 4; break; } + case 0x04: case 0x0c: std::memcpy(&value, p, 8); p += 8; break; + default: NotImplemented_nid_no_patch("sceKernelGetModuleInfoForUnwind eh_frame_ptr value format"); + } + if (application == 0x10) value += reinterpret_cast(at); + return value; +} + +void FillGuestUnwindInfo(const std::uint8_t* base, ModuleInfoForUnwind* info) { + const auto* dos = reinterpret_cast(base); + if (dos->e_magic != IMAGE_DOS_SIGNATURE) throw std::runtime_error("sceKernelGetModuleInfoForUnwind: image without a DOS header"); + const auto* nt = reinterpret_cast(base + dos->e_lfanew); + if (nt->Signature != IMAGE_NT_SIGNATURE) throw std::runtime_error("sceKernelGetModuleInfoForUnwind: image without an NT header"); + const auto* sections = IMAGE_FIRST_SECTION(nt); + for (unsigned i = 0; i < nt->FileHeader.NumberOfSections; ++i) { + if (std::memcmp(sections[i].Name, ".ehmeta", 8) != 0) continue; + std::uint32_t rva = 0; + std::memcpy(&rva, base + sections[i].VirtualAddress, 4); + const auto* header = base + rva; + if (header[0] != 1) NotImplemented_nid_no_patch("sceKernelGetModuleInfoForUnwind eh_frame_hdr version other than 1"); + const auto* p = header + 4; + const auto frames = ReadEncoded(p, header[1]); + const auto* record = reinterpret_cast(frames); + const auto* end = base + nt->OptionalHeader.SizeOfImage; + if (record < base || record >= end) throw std::runtime_error("sceKernelGetModuleInfoForUnwind: eh_frame_ptr outside the image"); + for (;;) { + if (record + 4 > end) throw std::runtime_error("sceKernelGetModuleInfoForUnwind: eh_frame has no terminator inside the image"); + std::uint32_t length = 0; + std::memcpy(&length, record, 4); + if (length == 0) break; + if (length == 0xffffffffu) NotImplemented_nid_no_patch("sceKernelGetModuleInfoForUnwind eh_frame record with a 64-bit length"); + record += 4 + length; + } + info->eh_frame_hdr_addr = reinterpret_cast(header); + info->eh_frame_addr = frames; + info->eh_frame_size = static_cast(record - reinterpret_cast(frames)); + info->seg0_addr = reinterpret_cast(base); + info->seg0_size = nt->OptionalHeader.SizeOfImage; + return; + } +} +} +#endif + extern "C" { void* APS5_VABI dlopen_nid_postfix(const char* path, int flags); void* APS5_VABI dlsym_nid_postfix(void* handle, const char* name); @@ -47,12 +104,58 @@ int APS5_VABI sceKernelGetModuleInfoForUnwind(uint64_t addr, int flags, ModuleIn info->eh_frame_size = 0; info->seg0_addr = reinterpret_cast(mbi.BaseAddress); info->seg0_size = mbi.RegionSize; + if (mbi.Type == MEM_IMAGE) FillGuestUnwindInfo(static_cast(mbi.AllocationBase), info); char path[4096] = {}; DWORD len = GetMappedFileNameA(GetCurrentProcess(), mbi.BaseAddress, path, sizeof(path) - 1); path[len] = '\0'; std::strncpy(info->name, path, sizeof(info->name) - 1); info->name[sizeof(info->name) - 1] = '\0'; return 0; +#elif defined(__APPLE__) + struct Search { + std::uint64_t address; + ModuleInfoForUnwind* info; + bool found; + } search {addr, info, false}; + dl_iterate_phdr([](dl_phdr_info* image, std::size_t, void* data) { + auto& search = *static_cast(data); + const Elf64_Phdr* first = nullptr; + const Elf64_Phdr* frames = nullptr; + bool contains = false; + for (std::uint16_t index = 0; index < image->dlpi_phnum; ++index) { + const auto& header = image->dlpi_phdr[index]; + const auto start = image->dlpi_addr + header.p_vaddr; + if (header.p_type == PT_LOAD && first == nullptr) first = &header; + if (header.p_type == PT_LOAD && search.address >= start && search.address - start < header.p_memsz) contains = true; + if (header.p_type == PT_GNU_EH_FRAME) frames = &header; + } + if (!contains) return 0; + auto* info = search.info; + info->st_size = sizeof(ModuleInfoForUnwind); + std::strncpy(info->name, image->dlpi_name != nullptr ? image->dlpi_name : "", sizeof(info->name) - 1); + info->name[sizeof(info->name) - 1] = '\0'; + info->eh_frame_hdr_addr = frames != nullptr ? image->dlpi_addr + frames->p_vaddr : 0; + info->eh_frame_addr = 0; + info->eh_frame_size = 0; + info->seg0_addr = image->dlpi_addr + first->p_vaddr; + info->seg0_size = first->p_memsz; + search.found = true; + return 1; + }, &search); + if (search.found) return 0; + Dl_info image {}; + if (!dladdr(reinterpret_cast(addr), &image) || image.dli_fbase == nullptr) return SCE_KERNEL_ERROR_ESRCH; + unsigned long textSize = 0; + getsegmentdata(static_cast(image.dli_fbase), "__TEXT", &textSize); + info->st_size = sizeof(ModuleInfoForUnwind); + std::strncpy(info->name, image.dli_fname != nullptr ? image.dli_fname : "", sizeof(info->name) - 1); + info->name[sizeof(info->name) - 1] = '\0'; + info->eh_frame_hdr_addr = 0; + info->eh_frame_addr = 0; + info->eh_frame_size = 0; + info->seg0_addr = reinterpret_cast(image.dli_fbase); + info->seg0_size = textSize; + return 0; #else std::ifstream maps("/proc/self/maps"); if (!maps) throw std::runtime_error("sceKernelGetModuleInfoForUnwind: failed to open /proc/self/maps"); diff --git a/core/libs/prx/libkernel/Pthread/Posix/Attr.cpp b/core/libs/prx/libkernel/Pthread/Posix/Attr.cpp index 5a027d874..a05f78402 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Attr.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Attr.cpp @@ -9,6 +9,7 @@ extern "C" { int APS5_VABI scePthreadAttrInit(PthreadAttr* attr); int APS5_VABI scePthreadAttrDestroy(PthreadAttr* attr); int APS5_VABI scePthreadAttrSetstacksize(PthreadAttr* attr, std::size_t stacksize); +int APS5_VABI scePthreadAttrSetstack(PthreadAttr* attr, void* addr, std::size_t size); } static bool Valid(const PthreadAttr* attr) { @@ -110,6 +111,11 @@ int APS5_VABI pthread_attr_setstacksize_nid_postfix(PthreadAttr* attr, size_t st return PosixThread::ToErrno(scePthreadAttrSetstacksize(attr, stack_size)); } +int APS5_VABI pthread_attr_setstack_nid_postfix(PthreadAttr* attr, void* stack_addr, size_t stack_size) { + if (!Valid(attr) || !stack_addr || stack_size < 16384) return PosixThread::GUEST_EINVAL; + return PosixThread::ToErrno(scePthreadAttrSetstack(attr, stack_addr, stack_size)); +} + int APS5_VABI pthread_attr_setsolosched_np_nid_postfix() { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp index 2f3fb16eb..f9db57ed3 100644 --- a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp +++ b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp @@ -5,6 +5,7 @@ #include "SceTypes.hpp" #include "prx/libkernel/Time/include/TimedWait.hpp" #include +#include #include #include #include @@ -63,6 +64,13 @@ struct PthreadSemPrivate { static constexpr KernelCpumask DEFAULT_THREAD_AFFINITY = 0x1FFF; static constexpr int DEFAULT_THREAD_PRIORITY = 700; +inline int GuestCpuFromHost(unsigned hostCpu, KernelCpumask threadAffinity) { + KernelCpumask mask = threadAffinity & DEFAULT_THREAD_AFFINITY; + if (mask == 0) mask = DEFAULT_THREAD_AFFINITY; + for (auto skip = hostCpu % static_cast(std::popcount(mask)); skip != 0; --skip) mask &= mask - 1; + return std::countr_zero(mask); +} + struct PthreadAttrPrivate { void* stackAddress = nullptr; std::size_t _stacksize; diff --git a/core/libs/prx/libkernel/Pthread/src/Attr.cpp b/core/libs/prx/libkernel/Pthread/src/Attr.cpp index 604a3af32..b31c92b74 100644 --- a/core/libs/prx/libkernel/Pthread/src/Attr.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Attr.cpp @@ -7,6 +7,7 @@ static constexpr int SCE_OK = 0; static constexpr std::size_t DEFAULT_STACK_SIZE = 1u << 20; +static constexpr std::size_t MIN_STACK_SIZE = 16384; static constexpr int DETACH_JOINABLE = 0; static constexpr int DETACH_DETACHED = 1; static constexpr int SCHED_FIFO_PS5 = 1; @@ -55,7 +56,7 @@ int APS5_VABI scePthreadAttrSetschedparam(PthreadAttr* attr, const KernelSchedPa int APS5_VABI scePthreadAttrSetstacksize(PthreadAttr* attr, std::size_t stacksize) { if (!attr || !*attr) throw std::runtime_error(std::string(__func__) + ": null attr"); - if (stacksize < 16384) throw std::runtime_error(std::string(__func__) + ": too small"); + if (stacksize < MIN_STACK_SIZE) return SCE_KERNEL_ERROR_EINVAL; #ifdef _WIN32 if (stacksize > std::numeric_limits::max() - 0xffffu) throw std::runtime_error(std::string(__func__) + ": stack size exceeds the Windows limit"); diff --git a/core/libs/prx/libkernel/Pthread/src/Cond.cpp b/core/libs/prx/libkernel/Pthread/src/Cond.cpp index 9eb2c9449..c18ad3d53 100644 --- a/core/libs/prx/libkernel/Pthread/src/Cond.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Cond.cpp @@ -61,8 +61,12 @@ int waitUntil(PthreadCond* cond, PthreadMutex* mutex, std::optional_count; m->_count = 0; m->_owner.store(std::thread::id{}, std::memory_order_release); + for (int level = 1; level < previousCount; ++level) + m->_rmtx.unlock(); if (deadlineNanos) timedOut = !c->_cv.WaitUntil(lock, *deadlineNanos); else c->_cv.Wait(lock); + for (int level = 1; level < previousCount; ++level) + m->_rmtx.lock(); m->_owner.store(std::this_thread::get_id(), std::memory_order_release); m->_count = previousCount; lock.release(); diff --git a/core/libs/prx/libkernel/Socket/include/SocketPoll.hpp b/core/libs/prx/libkernel/Socket/include/SocketPoll.hpp new file mode 100644 index 000000000..71d5f62e7 --- /dev/null +++ b/core/libs/prx/libkernel/Socket/include/SocketPoll.hpp @@ -0,0 +1,21 @@ +#pragma once +#include + +namespace KernelSocketPoll { +constexpr short Readable = 0x1; +constexpr short Urgent = 0x2; +constexpr short Writable = 0x4; +constexpr short Error = 0x8; +constexpr short HangUp = 0x10; +constexpr short Unknown = 0x20; + +struct Entry { + int descriptor; + short events; + short revents; +}; + +using Poller = int (*)(Entry* entries, int count, int timeoutMilliseconds); +} + +extern "C" void KernelSetSocketPoller_nid_no_patch(KernelSocketPoll::Poller poller); diff --git a/core/libs/prx/libkernel/Socket/src/Socket.cpp b/core/libs/prx/libkernel/Socket/src/Socket.cpp index 4e3025326..d2f176af1 100644 --- a/core/libs/prx/libkernel/Socket/src/Socket.cpp +++ b/core/libs/prx/libkernel/Socket/src/Socket.cpp @@ -2,6 +2,35 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libkernel/Socket/include/SocketPoll.hpp" + +#include +#include +#include +#include +#include + +namespace { + +constexpr int GuestEinval = 22; +constexpr int SelectDescriptorLimit = 1024; + +struct GuestTimeval { + std::int64_t seconds; + std::int64_t microseconds; +}; + +std::atomic g_socketPoller{nullptr}; + +bool DescriptorSet(const void* set, int descriptor) { + return set != nullptr && ((static_cast(set)[descriptor / 64] >> (descriptor % 64)) & 1u) != 0; +} + +void ClearDescriptor(void* set, int descriptor) { + if (set != nullptr) static_cast(set)[descriptor / 64] &= ~(std::uint64_t{1} << (descriptor % 64)); +} + +} extern "C" { @@ -16,14 +45,72 @@ int APS5_VABI inet_pton_nid_postfix(int af, const char* src, void* dst) { return __inet_pton_nid_postfix(af, src, dst); } +int* APS5_VABI __error_nid_postfix(); + +void KernelSetSocketPoller_nid_no_patch(KernelSocketPoll::Poller poller) { + g_socketPoller.store(poller); +} + int APS5_VABI select_nid_postfix(int nfds, void* readfds, void* writefds, void* exceptfds, const void* timeout) { - (void)nfds; - (void)readfds; - (void)writefds; - (void)exceptfds; - (void)timeout; - NotImplemented_nid_no_patch(__func__); - return 0; + const auto* limit = static_cast(timeout); + if (nfds < 0 || nfds > SelectDescriptorLimit || (limit != nullptr && (limit->seconds < 0 || limit->microseconds < 0 || limit->microseconds >= 1000000))) { + *__error_nid_postfix() = GuestEinval; + return -1; + } + std::vector entries; + for (int descriptor = 0; descriptor < nfds; ++descriptor) { + short events = 0; + if (DescriptorSet(readfds, descriptor)) events |= KernelSocketPoll::Readable; + if (DescriptorSet(writefds, descriptor)) events |= KernelSocketPoll::Writable; + if (DescriptorSet(exceptfds, descriptor)) events |= KernelSocketPoll::Urgent; + if (events != 0) entries.push_back({descriptor, events, 0}); + } + const auto start = std::chrono::steady_clock::now(); + const auto deadline = limit == nullptr ? std::chrono::steady_clock::time_point::max() + : start + std::chrono::seconds(limit->seconds) + std::chrono::microseconds(limit->microseconds); + for (;;) { + int waitMilliseconds = 100; + if (limit != nullptr) { + const auto remaining = std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()).count(); + waitMilliseconds = static_cast(std::clamp((remaining + 999) / 1000, 0, 100)); + } + if (!entries.empty()) { + const auto poller = g_socketPoller.load(); + if (poller == nullptr) NotImplemented_nid_no_patch("select on descriptors that are not libSceNet sockets"); + for (auto& entry : entries) entry.revents = 0; + const int result = poller(entries.data(), static_cast(entries.size()), waitMilliseconds); + if (result < 0) { + *__error_nid_postfix() = -result; + return -1; + } + int ready = 0; + for (const auto& entry : entries) { + if ((entry.revents & KernelSocketPoll::Unknown) != 0) NotImplemented_nid_no_patch("select on descriptors that are not libSceNet sockets"); + const bool readable = (entry.events & KernelSocketPoll::Readable) != 0 && (entry.revents & (KernelSocketPoll::Readable | KernelSocketPoll::HangUp | KernelSocketPoll::Error)) != 0; + const bool writable = (entry.events & KernelSocketPoll::Writable) != 0 && (entry.revents & (KernelSocketPoll::Writable | KernelSocketPoll::Error)) != 0; + const bool urgent = (entry.events & KernelSocketPoll::Urgent) != 0 && (entry.revents & KernelSocketPoll::Urgent) != 0; + ready += static_cast(readable) + static_cast(writable) + static_cast(urgent); + } + if (ready > 0) { + for (const auto& entry : entries) { + if ((entry.revents & (KernelSocketPoll::Readable | KernelSocketPoll::HangUp | KernelSocketPoll::Error)) == 0) ClearDescriptor(readfds, entry.descriptor); + if ((entry.revents & (KernelSocketPoll::Writable | KernelSocketPoll::Error)) == 0) ClearDescriptor(writefds, entry.descriptor); + if ((entry.revents & KernelSocketPoll::Urgent) == 0) ClearDescriptor(exceptfds, entry.descriptor); + } + return ready; + } + } else if (waitMilliseconds > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(waitMilliseconds)); + } + if (limit != nullptr && std::chrono::steady_clock::now() >= deadline) { + for (const auto& entry : entries) { + ClearDescriptor(readfds, entry.descriptor); + ClearDescriptor(writefds, entry.descriptor); + ClearDescriptor(exceptfds, entry.descriptor); + } + return 0; + } + } } } diff --git a/core/libs/prx/libkernel/Socket/src/Udp.cpp b/core/libs/prx/libkernel/Socket/src/Udp.cpp index af4f2f148..5b856cb03 100644 --- a/core/libs/prx/libkernel/Socket/src/Udp.cpp +++ b/core/libs/prx/libkernel/Socket/src/Udp.cpp @@ -6,12 +6,15 @@ #include #include #include +#include #include #include #endif +#include "prx/libc/include/GuestArena.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libkernel/Socket/include/SocketRuntime.hpp" #include +#include #include #include #include @@ -22,6 +25,8 @@ #include #include #include +#include +#include #include extern "C" int* APS5_VABI __error_nid_postfix(); @@ -34,6 +39,15 @@ using NativeSocket = int; constexpr auto Invalid = -1; #endif int Fail(int error) { *__error_nid_postfix() = error; return -1; } +constexpr int GuestNoSignal = 0x20000; +int NativeSendFlags(int flags) { +#ifdef _WIN32 + static_cast(flags); + return 0; +#else + return flags & GuestNoSignal ? MSG_NOSIGNAL : 0; +#endif +} int NativeError() { #ifdef _WIN32 switch (WSAGetLastError()) { @@ -73,6 +87,7 @@ int NativeError() { case ETIMEDOUT: return 60; case EINTR: return 4; case EINVAL: return 22; + case EPIPE: return 32; default: return 5; } #endif @@ -115,7 +130,7 @@ int Option(int guest) { bool Address(const void* input, std::uint32_t length, sockaddr_storage& native, socklen_t& size) { if (!input || length < 2) { Fail(14); return false; } const auto* bytes = static_cast(input); - if (bytes[1] == 2 && length >= 16 && bytes[0] == 16) { + if (bytes[1] == 2 && length >= 16) { auto& v4 = reinterpret_cast(native); v4.sin_family = AF_INET; std::memcpy(&v4.sin_port, bytes + 2, 2); @@ -123,7 +138,7 @@ bool Address(const void* input, std::uint32_t length, sockaddr_storage& native, size = sizeof(v4); return true; } - if (bytes[1] == 28 && length >= 28 && bytes[0] == 28) { + if (bytes[1] == 28 && length >= 28) { auto& v6 = reinterpret_cast(native); v6.sin6_family = AF_INET6; std::memcpy(&v6.sin6_port, bytes + 2, 2); @@ -166,6 +181,134 @@ bool GuestSockets::IsOpen(int descriptor) { return sockets.contains(descriptor); } +namespace { +#ifdef _WIN32 +using NativePollDescriptor = WSAPOLLFD; +#else +using NativePollDescriptor = pollfd; +#endif +struct GuestPollDescriptor { + int descriptor; + short events; + short revents; +}; +struct PollFlag { + short guest; + short native; +}; +constexpr short GuestPollPriority = 0x2; +constexpr short GuestPollInvalid = 0x20; +constexpr short GuestPollAccepted = 0x1 | 0x2 | 0x4 | 0x8 | 0x10 | 0x20 | 0x40 | 0x80 | 0x100; +constexpr PollFlag RequestFlags[] = { + {0x1, POLLIN}, {0x2, POLLPRI}, {0x4, POLLOUT}, {0x40, POLLRDNORM}, {0x80, POLLRDBAND}, {0x100, POLLWRBAND}}; +constexpr PollFlag StatusFlags[] = {{0x8, POLLERR}, {0x10, POLLHUP}, {GuestPollInvalid, POLLNVAL}}; +std::shared_ptr Find(int descriptor) { + std::lock_guard lock(socketsMutex); + const auto found = sockets.find(descriptor); + return found != sockets.end() ? found->second : nullptr; +} +} + +namespace { +constexpr int GuestUnix = 1; +constexpr int GuestCloseOnExec = 0x10000000; +constexpr int GuestNonBlocking = 0x20000000; +int PairError() { +#ifdef _WIN32 + switch (WSAGetLastError()) { + case WSAEMFILE: return 24; + default: return NativeError(); + } +#else + switch (errno) { + case EMFILE: return 24; + case ENFILE: return 23; + case ENOMEM: return 12; + default: return NativeError(); + } +#endif +} +int SetNonBlocking(Socket& socket) { +#ifdef _WIN32 + unsigned long value = 1; + if (ioctlsocket(socket.value, FIONBIO, &value)) return Fail(NativeError()); +#else + int value = 1; + if (::ioctl(socket.value, FIONBIO, &value)) return Fail(NativeError()); +#endif + socket.nonblocking = true; + return 0; +} +#ifdef _WIN32 +bool LoopbackName(NativeSocket socket, sockaddr_in& name) { + int size = sizeof(name); + return getsockname(socket, reinterpret_cast(&name), &size) == 0; +} +int NativePair(int type, NativeSocket (&pair)[2]) { + static const int startup = [] { WSADATA data{}; return WSAStartup(MAKEWORD(2, 2), &data); }(); + if (startup) return 5; + if (type == 5) throw std::runtime_error("socketpair: SOCK_SEQPACKET is not supported on Windows"); + sockaddr_in loopback{}; + loopback.sin_family = AF_INET; + loopback.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + const int nativeType = type == 1 ? SOCK_STREAM : SOCK_DGRAM; + Socket first(::socket(AF_INET, nativeType, 0), 2, type); + if (first.value == Invalid) return PairError(); + const int exclusive = 1; + setsockopt(first.value, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, reinterpret_cast(&exclusive), sizeof(exclusive)); + if (bind(first.value, reinterpret_cast(&loopback), sizeof(loopback))) return PairError(); + sockaddr_in firstName{}; + if (!LoopbackName(first.value, firstName)) return PairError(); + Socket second(::socket(AF_INET, nativeType, 0), 2, type); + if (second.value == Invalid) return PairError(); + if (type == 1) { + if (listen(first.value, 1)) return PairError(); + if (connect(second.value, reinterpret_cast(&firstName), sizeof(firstName))) return PairError(); + sockaddr_in peer{}; + int peerSize = sizeof(peer); + Socket accepted(accept(first.value, reinterpret_cast(&peer), &peerSize), 2, type); + if (accepted.value == Invalid) return PairError(); + sockaddr_in secondName{}; + if (!LoopbackName(second.value, secondName)) return PairError(); + if (peer.sin_port != secondName.sin_port) return 53; + const int noDelay = 1; + setsockopt(second.value, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast(&noDelay), sizeof(noDelay)); + setsockopt(accepted.value, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast(&noDelay), sizeof(noDelay)); + pair[0] = second.value; + pair[1] = accepted.value; + second.value = Invalid; + accepted.value = Invalid; + return 0; + } + if (bind(second.value, reinterpret_cast(&loopback), sizeof(loopback))) return PairError(); + sockaddr_in secondName{}; + if (!LoopbackName(second.value, secondName)) return PairError(); + if (connect(first.value, reinterpret_cast(&secondName), sizeof(secondName))) return PairError(); + if (connect(second.value, reinterpret_cast(&firstName), sizeof(firstName))) return PairError(); + pair[0] = first.value; + pair[1] = second.value; + first.value = Invalid; + second.value = Invalid; + return 0; +} +#else +int NativePair(int type, NativeSocket (&pair)[2]) { + const int nativeType = type == 1 ? SOCK_STREAM : type == 2 ? SOCK_DGRAM : SOCK_SEQPACKET; + int native[2]; + if (::socketpair(AF_UNIX, nativeType, 0, native)) return PairError(); + pair[0] = native[0]; + pair[1] = native[1]; + return 0; +} +#endif +int UnnamedAddress(void* address, std::uint32_t* length) { + const unsigned char bytes[16] = {16, GuestUnix}; + *length = std::min(*length, sizeof(bytes)); + std::memcpy(address, bytes, *length); + return 0; +} +} + extern "C" { int APS5_VABI fcntl_nid_postfix(int descriptor, int command, ...) { const auto socket = Lookup(descriptor); @@ -244,6 +387,48 @@ int APS5_VABI socket_nid_postfix(int family, int type, int protocol) { return Fail(12); } } +int APS5_VABI socketpair_nid_postfix(int family, int type, int protocol, int* descriptors) { + const bool nonblocking = (type & GuestNonBlocking) != 0; + const int plainType = type & ~(GuestCloseOnExec | GuestNonBlocking); + if (family != GuestUnix && family != 2 && family != 28) return Fail(47); + if (plainType == 0) return Fail(43); + if (family != GuestUnix) { + if (plainType != 1 && plainType != 2) return Fail(41); + return Fail(protocol == 0 || protocol == (plainType == 1 ? 6 : 17) ? 45 : 43); + } + if (protocol != 0) return Fail(43); + if (plainType != 1 && plainType != 2 && plainType != 5) return Fail(41); + if (!descriptors) return Fail(14); + const GuestArena::HostWrite destination(descriptors, 2 * sizeof(int)); + if (!destination.Open()) return Fail(14); + NativeSocket native[2]; + if (const int error = NativePair(plainType, native)) return Fail(error); + Socket firstGuard(native[0], GuestUnix, plainType); + Socket secondGuard(native[1], GuestUnix, plainType); + try { + auto first = std::make_shared(native[0], GuestUnix, plainType); + firstGuard.value = Invalid; + auto second = std::make_shared(native[1], GuestUnix, plainType); + secondGuard.value = Invalid; + if (nonblocking && (SetNonBlocking(*first) || SetNonBlocking(*second))) return -1; + std::lock_guard lock(socketsMutex); + if (nextDescriptor >= INT_MAX - 1) return Fail(24); + const int firstDescriptor = nextDescriptor++; + const int secondDescriptor = nextDescriptor++; + sockets.emplace(firstDescriptor, std::move(first)); + try { + sockets.emplace(secondDescriptor, std::move(second)); + } catch (const std::bad_alloc&) { + sockets.erase(firstDescriptor); + return Fail(12); + } + descriptors[0] = firstDescriptor; + descriptors[1] = secondDescriptor; + return 0; + } catch (const std::bad_alloc&) { + return Fail(12); + } +} int APS5_VABI connect_nid_postfix(int descriptor, const void* address, std::uint32_t length) { const auto socket = Lookup(descriptor); if (!socket) return -1; @@ -285,10 +470,10 @@ int APS5_VABI accept_nid_postfix(int descriptor, void* address, std::uint32_t* l std::int64_t APS5_VABI send_nid_postfix(int descriptor, const void* buffer, std::uint64_t length, int flags) { const auto socket = Lookup(descriptor); if (!socket) return -1; - if (flags != 0) return Fail(45); + if ((flags & ~GuestNoSignal) != 0) return Fail(45); if (length > INT_MAX) return Fail(40); if (!buffer && length) return Fail(14); - const auto result = ::send(socket->value, static_cast(buffer), static_cast(length), 0); + const auto result = ::send(socket->value, static_cast(buffer), static_cast(length), NativeSendFlags(flags)); return result < 0 ? Fail(NativeError()) : result; } std::int64_t APS5_VABI recv_nid_postfix(int descriptor, void* buffer, std::uint64_t length, int flags) { @@ -319,6 +504,7 @@ int APS5_VABI getsockname_nid_postfix(int descriptor, void* address, std::uint32 const auto socket = Lookup(descriptor); if (!socket) return -1; if (!address || !length) return Fail(14); + if (socket->family == GuestUnix) return UnnamedAddress(address, length); sockaddr_storage native{}; socklen_t size = sizeof(native); if (::getsockname(socket->value, reinterpret_cast(&native), &size)) return Fail(NativeError()); @@ -329,6 +515,7 @@ int APS5_VABI getpeername_nid_postfix(int descriptor, void* address, std::uint32 const auto socket = Lookup(descriptor); if (!socket) return -1; if (!address || !length) return Fail(14); + if (socket->family == GuestUnix) return UnnamedAddress(address, length); sockaddr_storage native{}; socklen_t size = sizeof(native); if (::getpeername(socket->value, reinterpret_cast(&native), &size)) return Fail(NativeError()); @@ -359,19 +546,19 @@ std::int64_t APS5_VABI sendto_nid_postfix(int descriptor, const void* buffer, st int flags, const void* address, std::uint32_t addressLength) { const auto socket = Lookup(descriptor); if (!socket) return -1; - if (flags != 0) return Fail(45); + if ((flags & ~GuestNoSignal) != 0) return Fail(45); if (length > INT_MAX) return Fail(40); if (!buffer && length) return Fail(14); int result; if (!address) { if (addressLength != 0) return Fail(22); - result = ::send(socket->value, static_cast(buffer), static_cast(length), 0); + result = ::send(socket->value, static_cast(buffer), static_cast(length), NativeSendFlags(flags)); } else { sockaddr_storage native{}; socklen_t size; if (!Address(address, addressLength, native, size)) return -1; if (native.ss_family != (socket->family == 2 ? AF_INET : AF_INET6)) return Fail(47); - result = static_cast(::sendto(socket->value, static_cast(buffer), static_cast(length), 0, + result = static_cast(::sendto(socket->value, static_cast(buffer), static_cast(length), NativeSendFlags(flags), reinterpret_cast(&native), size)); } return result < 0 ? Fail(NativeError()) : result; @@ -388,6 +575,10 @@ std::int64_t APS5_VABI recvfrom_nid_postfix(int descriptor, void* buffer, std::u const auto result = ::recvfrom(socket->value, static_cast(buffer), static_cast(length), flags & 2 ? MSG_PEEK : 0, reinterpret_cast(&native), &size); if (result < 0) return Fail(NativeError()); + if (address && socket->family == GuestUnix) { + if (socket->type == 1) *addressLength = 0; else UnnamedAddress(address, addressLength); + return result; + } if (address) GuestAddress(native, address, addressLength); return result; } @@ -471,7 +662,9 @@ std::int64_t APS5_VABI recvmsg_nid_postfix(int descriptor, GuestMsghdr* message, std::memcpy(message->iov[i].base, buffer.data() + offset, count); offset += count; } - if (message->name) { + if (message->name && socket->family == GuestUnix) { + if (socket->type == 1) message->nameLength = 0; else UnnamedAddress(message->name, &message->nameLength); + } else if (message->name) { if (size > 0 && (native.ss_family == AF_INET || native.ss_family == AF_INET6)) { GuestAddress(native, message->name, &message->nameLength); } else { @@ -481,4 +674,60 @@ std::int64_t APS5_VABI recvmsg_nid_postfix(int descriptor, GuestMsghdr* message, message->controlLength = 0; return received; } +int APS5_VABI poll_nid_postfix(GuestPollDescriptor* descriptors, std::uint32_t count, int timeout) { + if (timeout < -1) return Fail(22); + if (count && !descriptors) return Fail(14); + std::vector> held; + std::vector native; + std::vector owners; + int ready = 0; + for (std::uint32_t i = 0; i < count; ++i) { + auto& entry = descriptors[i]; + entry.revents = 0; + if (entry.descriptor < 0) continue; + if ((entry.events & ~GuestPollAccepted) != 0) + throw std::runtime_error("poll: unsupported event flags " + std::to_string(entry.events)); + if (entry.descriptor < GuestSockets::FirstDescriptor) + throw std::runtime_error("poll: descriptor " + std::to_string(entry.descriptor) + " is not a socket"); + auto socket = Find(entry.descriptor); + if (!socket) { + entry.revents = GuestPollInvalid; + ++ready; + continue; + } +#ifdef _WIN32 + if (entry.events & GuestPollPriority) throw std::runtime_error("poll: POLLPRI is not supported by WSAPoll"); +#endif + NativePollDescriptor request{}; + request.fd = socket->value; + for (const auto& flag : RequestFlags) + if (entry.events & flag.guest) request.events = static_cast(request.events | flag.native); + native.push_back(request); + owners.push_back(i); + held.push_back(std::move(socket)); + } + if (native.empty()) { + if (ready || timeout == 0) return ready; + if (timeout < 0) throw std::runtime_error("poll: an infinite wait without sockets never returns"); + std::this_thread::sleep_for(std::chrono::milliseconds(timeout)); + return 0; + } + const int wait = ready ? 0 : timeout; +#ifdef _WIN32 + const int result = WSAPoll(native.data(), static_cast(native.size()), wait); +#else + const int result = ::poll(native.data(), static_cast(native.size()), wait); +#endif + if (result < 0) return Fail(NativeError()); + for (std::size_t i = 0; i < native.size(); ++i) { + auto& entry = descriptors[owners[i]]; + for (const auto& flag : RequestFlags) + if ((entry.events & flag.guest) && (native[i].revents & flag.native)) + entry.revents = static_cast(entry.revents | flag.guest); + for (const auto& flag : StatusFlags) + if (native[i].revents & flag.native) entry.revents = static_cast(entry.revents | flag.guest); + if (entry.revents) ++ready; + } + return ready; +} } diff --git a/core/libs/prx/libkernel/System/src/Coredump.cpp b/core/libs/prx/libkernel/System/src/Coredump.cpp index 84b33fec8..767371123 100644 --- a/core/libs/prx/libkernel/System/src/Coredump.cpp +++ b/core/libs/prx/libkernel/System/src/Coredump.cpp @@ -36,9 +36,8 @@ int APS5_VABI sceCoredumpAttachUserFile(void) { return 0; } -APS5_EXPORT("Jrs7UUkGOFo", sceCoredumpUnknown01); -int APS5_VABI sceCoredumpUnknown01(void) { - NotImplemented_nid_no_patch("Jrs7UUkGOFo"); +int APS5_VABI sceCoredumpGetStopInfoGpu_Agc(void) { + NotImplemented_nid_no_patch(__func__); return 0; } @@ -62,6 +61,11 @@ int APS5_VABI sceCoredumpGetStopInfoCpu(void) { return 0; } +int APS5_VABI sceCoredumpWriteUserString() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + int APS5_VABI sceCoredumpAttachUserMemoryFile(void) { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/prx/libkernel/System/src/Exception.cpp b/core/libs/prx/libkernel/System/src/Exception.cpp index b40e31546..718e4119d 100644 --- a/core/libs/prx/libkernel/System/src/Exception.cpp +++ b/core/libs/prx/libkernel/System/src/Exception.cpp @@ -17,6 +17,7 @@ extern "C" Pthread APS5_VABI scePthreadSelf(); #ifdef _WIN32 extern "C" void Aps5RedirectedEntryStub(); +extern "C" [[noreturn]] void Aps5RedirectedExit(CONTEXT* context, const void* vectors, void (*restore)(CONTEXT*, EXCEPTION_RECORD*)); #endif namespace { @@ -103,13 +104,21 @@ GuestExceptionHandler Handler(int signum) { constexpr std::size_t RedZone = 128; constexpr std::size_t HomeArea = 32; constexpr auto RetryLimit = std::chrono::seconds(1); +constexpr std::size_t VectorBytes = 16 * 32; struct Delivery { GuestExceptionHandler handler; int signum; CONTEXT context; + std::uint8_t vectors[VectorBytes]; + std::uint64_t saveVectors; }; +const bool SaveVectors = [] { + __builtin_cpu_init(); + return __builtin_cpu_supports("avx") != 0; +}(); + void Deliver(GuestExceptionHandler handler, int signum, CONTEXT& context) { GuestUcontext ucontext{}; auto& m = ucontext.mcontext; @@ -161,8 +170,7 @@ void Deliver(GuestExceptionHandler handler, int signum, CONTEXT& context) { [[noreturn]] void RedirectedEntry(Delivery* delivery) { CONTEXT context = delivery->context; Deliver(delivery->handler, delivery->signum, context); - RtlRestoreContext(&context, nullptr); - std::abort(); + Aps5RedirectedExit(&context, delivery->saveVectors != 0 ? delivery->vectors : nullptr, RtlRestoreContext); } bool StackWritable(DWORD64 low, DWORD64 high) { @@ -187,6 +195,7 @@ void CALLBACK WaitingEntry(ULONG_PTR parameter) { static_assert(HomeArea + 8 == 40, "Aps5RedirectedEntryStub finds the delivery 40 bytes above its stack pointer"); static_assert(offsetof(Delivery, context) == 16 && offsetof(CONTEXT, Rax) == 0x78 && offsetof(CONTEXT, Rbp) == 0xa0 && offsetof(CONTEXT, R15) == 0xf0, "Aps5RedirectedEntryStub stores the live registers into the delivery's context"); +static_assert(offsetof(Delivery, vectors) == 1248 && offsetof(Delivery, saveVectors) == 1760, "Aps5RedirectedEntryStub stores ymm0-ymm15 into the delivery when saveVectors is set"); bool Exited(HANDLE native) { return WaitForSingleObject(native, 0) == WAIT_OBJECT_0; @@ -216,8 +225,8 @@ bool RaiseOn(Pthread thread, GuestExceptionHandler handler, int signum) { return true; } const auto native = static_cast(thread->nativeHandle); - auto queued = std::make_unique(Delivery{handler, signum, {}}); - alignas(16) Delivery delivery{handler, signum, {}}; + auto queued = std::make_unique(Delivery{handler, signum, {}, {}, 0}); + alignas(16) Delivery delivery{handler, signum, {}, {}, SaveVectors ? 1u : 0u}; const auto retryDeadline = std::chrono::steady_clock::now() + RetryLimit; for (;;) { if (SuspendThread(native) == static_cast(-1)) { @@ -287,8 +296,50 @@ asm(".text\n" " movq %r13, 280(%rsp)\n" " movq %r14, 288(%rsp)\n" " movq %r15, 296(%rsp)\n" + " cmpq $0, 1800(%rsp)\n" + " je 1f\n" + " vmovdqu %ymm0, 1288(%rsp)\n" + " vmovdqu %ymm1, 1320(%rsp)\n" + " vmovdqu %ymm2, 1352(%rsp)\n" + " vmovdqu %ymm3, 1384(%rsp)\n" + " vmovdqu %ymm4, 1416(%rsp)\n" + " vmovdqu %ymm5, 1448(%rsp)\n" + " vmovdqu %ymm6, 1480(%rsp)\n" + " vmovdqu %ymm7, 1512(%rsp)\n" + " vmovdqu %ymm8, 1544(%rsp)\n" + " vmovdqu %ymm9, 1576(%rsp)\n" + " vmovdqu %ymm10, 1608(%rsp)\n" + " vmovdqu %ymm11, 1640(%rsp)\n" + " vmovdqu %ymm12, 1672(%rsp)\n" + " vmovdqu %ymm13, 1704(%rsp)\n" + " vmovdqu %ymm14, 1736(%rsp)\n" + " vmovdqu %ymm15, 1768(%rsp)\n" + "1:\n" " leaq 40(%rsp), %rcx\n" - " jmp Aps5RedirectedEntry\n"); + " jmp Aps5RedirectedEntry\n" + ".globl Aps5RedirectedExit\n" + "Aps5RedirectedExit:\n" + " testq %rdx, %rdx\n" + " je 1f\n" + " vmovdqu 0(%rdx), %ymm0\n" + " vmovdqu 32(%rdx), %ymm1\n" + " vmovdqu 64(%rdx), %ymm2\n" + " vmovdqu 96(%rdx), %ymm3\n" + " vmovdqu 128(%rdx), %ymm4\n" + " vmovdqu 160(%rdx), %ymm5\n" + " vmovdqu 192(%rdx), %ymm6\n" + " vmovdqu 224(%rdx), %ymm7\n" + " vmovdqu 256(%rdx), %ymm8\n" + " vmovdqu 288(%rdx), %ymm9\n" + " vmovdqu 320(%rdx), %ymm10\n" + " vmovdqu 352(%rdx), %ymm11\n" + " vmovdqu 384(%rdx), %ymm12\n" + " vmovdqu 416(%rdx), %ymm13\n" + " vmovdqu 448(%rdx), %ymm14\n" + " vmovdqu 480(%rdx), %ymm15\n" + "1:\n" + " xorl %edx, %edx\n" + " jmp *%r8\n"); #endif extern "C" { diff --git a/core/libs/prx/libkernel/System/src/Process.cpp b/core/libs/prx/libkernel/System/src/Process.cpp index cc9e1301f..a7f0d139d 100644 --- a/core/libs/prx/libkernel/System/src/Process.cpp +++ b/core/libs/prx/libkernel/System/src/Process.cpp @@ -13,6 +13,7 @@ #include "prx/libc/include/ApplicationHeap.hpp" #include "prx/libc/include/Shutdown.hpp" #include "prx/libkernel/DirectMemory/DirectMemory.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" #include #include #include @@ -89,39 +90,6 @@ void validateSchedulingPolicy(int policy) { throw std::invalid_argument("Unsupported guest scheduling policy"); } -#ifdef _WIN32 -void syncVolumes() { - std::array name{}; - const auto first = FindFirstVolumeW(name.data(), static_cast(name.size())); - if (first == INVALID_HANDLE_VALUE) - throw std::system_error(GetLastError(), std::system_category(), "Enumerating volumes for sync"); - const std::unique_ptr search(first, &FindVolumeClose); - for (;;) { - DWORD flags = 0; - if (!GetVolumeInformationW(name.data(), nullptr, 0, nullptr, nullptr, &flags, nullptr, 0)) - throw std::system_error(GetLastError(), std::system_category(), "Reading volume properties for sync"); - if ((flags & FILE_READ_ONLY_VOLUME) == 0) { - std::wstring path(name.data()); - if (path.empty() || path.back() != L'\\') - throw std::runtime_error("Invalid volume path for sync"); - path.pop_back(); - const auto native = CreateFileW(path.c_str(), GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, 0, nullptr); - if (native == INVALID_HANDLE_VALUE) - throw std::system_error(GetLastError(), std::system_category(), "Opening volume for sync"); - const std::unique_ptr volume(native, &CloseHandle); - if (!FlushFileBuffers(volume.get())) - throw std::system_error(GetLastError(), std::system_category(), "Flushing volume"); - } - if (FindNextVolumeW(search.get(), name.data(), static_cast(name.size()))) - continue; - const auto error = GetLastError(); - if (error != ERROR_NO_MORE_FILES) - throw std::system_error(error, std::system_category(), "Enumerating volumes for sync"); - break; - } -} -#endif - } struct GuestResourceUsage { @@ -143,6 +111,8 @@ struct GuestResourceUsage { std::int64_t ru_nivcsw; }; +extern "C" Pthread APS5_VABI scePthreadSelf(); + extern "C" { // unknown data @@ -190,13 +160,13 @@ int APS5_VABI sceKernelGetCurrentCpu(void) { throw std::system_error(GetLastError(), std::system_category(), "Reading processor group size"); index += count; } - return static_cast(index); #else const int cpu = ::sched_getcpu(); if (cpu < 0) throw std::system_error(errno, std::generic_category(), "Reading current processor"); - return cpu; + const auto index = static_cast(cpu); #endif + return GuestCpuFromHost(index, scePthreadSelf()->affinity.load(std::memory_order_relaxed)); } std::uint64_t APS5_VABI sceKernelGetGPI(void) { @@ -232,11 +202,7 @@ int APS5_VABI sceKernelUuidCreate(std::uint32_t* uuid) { } void APS5_VABI sceKernelSync(void) { -#ifdef _WIN32 - syncVolumes(); -#else - ::sync(); -#endif + SyncWrittenPaths_nid_no_patch(); } int APS5_VABI sched_get_priority_max_nid_postfix(int policy) { diff --git a/core/libs/prx/libkernel/Time/TimedWait.cpp b/core/libs/prx/libkernel/Time/TimedWait.cpp index fd8813901..f5949d6da 100644 --- a/core/libs/prx/libkernel/Time/TimedWait.cpp +++ b/core/libs/prx/libkernel/Time/TimedWait.cpp @@ -94,9 +94,12 @@ struct Waiter { HANDLE timer = CreateHighResolutionTimer(); Waiter* previous = nullptr; Waiter* next = nullptr; + Waiter* nested = nullptr; bool queued = false; + bool busy = false; ~Waiter() { + delete nested; if (event) CloseHandle(event); if (timer) CloseHandle(timer); } @@ -118,6 +121,16 @@ Waiter* ThisThreadWaiter() { return waiter; } +Waiter* AcquireWaiter() { + Waiter* waiter = ThisThreadWaiter(); + while (waiter->busy) { + if (!waiter->nested) waiter->nested = new Waiter(); + waiter = waiter->nested; + } + waiter->busy = true; + return waiter; +} + bool EventSet(HANDLE event, DWORD milliseconds) { return AlertableWait(1, &event, milliseconds) == WAIT_OBJECT_0; } @@ -147,7 +160,7 @@ bool WaitEventUntil(Waiter* waiter, std::uint64_t deadlineNanos) { } // namespace Waiter* Condition::enqueue() { - Waiter* waiter = ThisThreadWaiter(); + Waiter* waiter = AcquireWaiter(); std::lock_guard lock(queueLock); waiter->queued = true; waiter->next = nullptr; @@ -170,17 +183,22 @@ void Condition::unlink(Waiter* waiter) { void Condition::waitSignal(Waiter* waiter) { AlertableWait(1, &waiter->event, INFINITE); + waiter->busy = false; } bool Condition::waitSignalUntil(Waiter* waiter, std::uint64_t deadlineNanos) { - if (WaitEventUntil(waiter, deadlineNanos)) return true; - std::lock_guard lock(queueLock); - if (waiter->queued) { - unlink(waiter); - return false; + bool signaled = WaitEventUntil(waiter, deadlineNanos); + if (!signaled) { + std::lock_guard lock(queueLock); + if (waiter->queued) { + unlink(waiter); + } else { + WaitForSingleObject(waiter->event, 0); + signaled = true; + } } - WaitForSingleObject(waiter->event, 0); - return true; + waiter->busy = false; + return signaled; } void Condition::NotifyOne() { diff --git a/core/libs/tests/ApplicationHeap.cpp b/core/libs/tests/ApplicationHeap.cpp index 8d882fdb9..7ea5cec5b 100644 --- a/core/libs/tests/ApplicationHeap.cpp +++ b/core/libs/tests/ApplicationHeap.cpp @@ -17,6 +17,7 @@ void* APS5_VABI _ZnamRKSt9nothrow_t_nid_postfix(std::size_t, const void*) noexce void APS5_VABI _ZdlPv_nid_postfix(void*); void APS5_VABI _ZdaPv_nid_postfix(void*); void APS5_VABI _ZdlPvSt11align_val_t_nid_postfix(void*, std::size_t); +void APS5_VABI _ZdlPvmSt11align_val_t_nid_postfix(void*, std::size_t, std::size_t); void* ApplicationHeapRealign_nid_no_patch(void*, std::size_t, std::size_t); char* APS5_VABI strdup_nid_postfix(const char*); char* APS5_VABI strndup_nid_postfix(const char*, std::size_t); @@ -201,6 +202,11 @@ int main(int argc, char** argv) { require(frees == 5); _ZdlPvSt11align_val_t_nid_postfix(nullptr, 64); require(frees == 5); + pointer = ApplicationHeapAlign_nid_no_patch(64, 37); + _ZdlPvmSt11align_val_t_nid_postfix(pointer, 37, 64); + require(frees == 6); + _ZdlPvmSt11align_val_t_nid_postfix(nullptr, 37, 64); + require(frees == 6); fail = true; require(_ZnwmRKSt9nothrow_t_nid_postfix(8, nullptr) == nullptr); require(_ZnamRKSt9nothrow_t_nid_postfix(8, nullptr) == nullptr); diff --git a/core/libs/tests/CpuTopology.cpp b/core/libs/tests/CpuTopology.cpp new file mode 100644 index 000000000..09ddd3568 --- /dev/null +++ b/core/libs/tests/CpuTopology.cpp @@ -0,0 +1,50 @@ +#include "prx/libc/include/CpuTopology.hpp" +#include +#include +#include +#include +#include + +static void Require(bool value, const char* what) { + if (value) return; + std::fprintf(stderr, "cpu topology test failed: %s\n", what); + std::exit(1); +} + +static std::uint64_t Parse(const char* text) { + const auto path = std::filesystem::temp_directory_path() / ("anyps5_cpu_topology_test_cpus-" + std::to_string(std::random_device{}())); + { + std::ofstream file(path, std::ios::trunc); + file << text; + } + const auto mask = CpuTopology::MaskFromCpuList(path.c_str()); + std::filesystem::remove(path); + return mask; +} + +int main() { + Require(Parse("0-15\n") == 0xffffull, "range"); + Require(Parse("16-31\n") == 0xffff0000ull, "high range"); + Require(Parse("0,2,4-5\n") == 0x35ull, "list of cpus and ranges"); + Require(Parse("3") == 0x8ull, "single cpu without a newline"); + Require(Parse("62-70\n") == 0xc000000000000000ull, "cpus past 63 are left out"); + Require(Parse("") == 0, "empty list"); + Require(Parse("4-2\n") == 0, "reversed range"); + Require(Parse("0-x\n") == 0, "malformed range"); + Require(Parse("0;1\n") == 0, "malformed separator"); + Require(CpuTopology::MaskFromCpuList("/nonexistent/anyps5/cpus") == 0, "missing file"); + + const auto& layout = CpuTopology::Get(); + Require(layout.process != 0, "process affinity read"); + Require(!layout.hybrid || (layout.performant & layout.efficient) == 0, "hybrid classes overlap"); + const std::uint64_t first = layout.process & (~layout.process + 1); + Require(CpuTopology::Pin(nullptr, first) == first, "pin to one cpu of the process"); + cpu_set_t set; + CPU_ZERO(&set); + Require(pthread_getaffinity_np(pthread_self(), sizeof(set), &set) == 0, "read thread affinity"); + Require(CPU_COUNT(&set) == 1 && CPU_ISSET(static_cast(__builtin_ctzll(first)), &set), "thread runs on the pinned cpu only"); + Require(CpuTopology::Pin(nullptr, ~layout.process) == 0, "mask outside the process is not applied"); + int dummy = 0; + Require(CpuTopology::Pin(&dummy, first) == 0, "other threads are not pinned by handle"); + return 0; +} diff --git a/core/libs/tests/ExceptionPersonality.cpp b/core/libs/tests/ExceptionPersonality.cpp new file mode 100644 index 000000000..75d8115a5 --- /dev/null +++ b/core/libs/tests/ExceptionPersonality.cpp @@ -0,0 +1,92 @@ +#include "prx/libc/include/exceptions/Unwind.hpp" +#include +#include + +namespace LibcUnwind { +_Unwind_Reason_Code CallPersonality(Word personality, _Unwind_Action actions, _Unwind_Exception* exception, _Unwind_Context* context); +} + +static void Require(bool value) { if (!value) std::abort(); } + +static const unsigned char cleanupTable[] = {0xff, 0xff, 0x01, 0x04, 0x10, 0x20, 0x40, 0x00}; +static const unsigned char noLandingTable[] = {0xff, 0xff, 0x01, 0x04, 0x10, 0x20, 0x00, 0x00}; + +constexpr std::uintptr_t region = 0x10000; + +struct Frame { + _Unwind_Context context {}; + _Unwind_Exception exception {}; + + Frame(const unsigned char* table, std::uintptr_t resume, bool signalFrame = false) { + context.region = region; + context.lsda = reinterpret_cast(table); + context.registers[16] = region + resume; + context.signalFrame = signalFrame; + } +}; + +static _Unwind_Reason_Code Direct(Frame& frame, _Unwind_Action actions, int version = 1) { + return __gcc_personality_v0_nid_postfix(version, actions, 0, &frame.exception, &frame.context); +} + +static _Unwind_Reason_Code Routed(Frame& frame, _Unwind_Action actions) { + return LibcUnwind::CallPersonality(reinterpret_cast(__gcc_personality_v0_nid_postfix), actions, &frame.exception, &frame.context); +} + +static void RequireInstalled(Frame& frame) { + Require(frame.context.registers[0] == reinterpret_cast(&frame.exception)); + Require(frame.context.registers[1] == 0); + Require(frame.context.registers[16] == region + 0x40); +} + +int main() { + { + Frame frame(cleanupTable, 0x18); + Require(Direct(frame, _UA_CLEANUP_PHASE) == _URC_INSTALL_CONTEXT); + RequireInstalled(frame); + } + { + Frame frame(cleanupTable, 0x18); + Require(Routed(frame, _UA_CLEANUP_PHASE) == _URC_INSTALL_CONTEXT); + RequireInstalled(frame); + } + { + Frame frame(cleanupTable, 0x18); + Require(Direct(frame, _UA_SEARCH_PHASE) == _URC_CONTINUE_UNWIND); + Require(frame.context.registers[16] == region + 0x18); + } + { + Frame frame(cleanupTable, 0x18); + Require(Routed(frame, _UA_SEARCH_PHASE) == _URC_CONTINUE_UNWIND); + } + { + Frame before(cleanupTable, 0x10); + Require(Direct(before, _UA_CLEANUP_PHASE) == _URC_CONTINUE_UNWIND); + Frame after(cleanupTable, 0x31); + Require(Direct(after, _UA_CLEANUP_PHASE) == _URC_CONTINUE_UNWIND); + } + { + Frame frame(cleanupTable, 0x10, true); + Require(Direct(frame, _UA_CLEANUP_PHASE) == _URC_INSTALL_CONTEXT); + RequireInstalled(frame); + } + { + Frame frame(noLandingTable, 0x18); + Require(Direct(frame, _UA_CLEANUP_PHASE) == _URC_CONTINUE_UNWIND); + } + { + Frame frame(nullptr, 0x18); + Require(Direct(frame, _UA_CLEANUP_PHASE) == _URC_CONTINUE_UNWIND); + } + { + Frame frame(cleanupTable, 0x18); + Require(Direct(frame, _UA_CLEANUP_PHASE, 2) == _URC_FATAL_PHASE1_ERROR); + } + { + Frame frame(cleanupTable, 0x18); + Require(LibcUnwind::CallPersonality(0x1000, _UA_SEARCH_PHASE, &frame.exception, &frame.context) == _URC_FATAL_PHASE1_ERROR); + Require(LibcUnwind::CallPersonality(0x1000, _UA_CLEANUP_PHASE, &frame.exception, &frame.context) == _URC_FATAL_PHASE2_ERROR); + } + std::puts("exception personality tests passed"); + return 0; +} diff --git a/core/libs/tests/ExceptionRuntime.cpp b/core/libs/tests/ExceptionRuntime.cpp index bb8df2c16..a6d0715d8 100644 --- a/core/libs/tests/ExceptionRuntime.cpp +++ b/core/libs/tests/ExceptionRuntime.cpp @@ -1,3 +1,4 @@ +#undef NDEBUG #include #include #include @@ -5,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -18,7 +21,9 @@ extern "C" void __cxa_rethrow_primary_exception_nid_postfix(void*); extern "C" [[noreturn]] void _ZSt14_Xout_of_rangePKc_nid_postfix(const char*); extern "C" [[noreturn]] void _ZSt13_Xrange_errorPKc_nid_postfix(const char*); +extern "C" [[noreturn]] void _ZNSt8__sce_v219_Xbad_function_callEv_nid_postfix(); extern "C" [[noreturn]] void __cxa_bad_cast_nid_postfix(); +extern "C" [[noreturn]] void _ZSt19_Throw_bad_weak_ptrv_nid_postfix(); extern "C" [[noreturn]] void _ZNKSt9exception6_RaiseEv_nid_postfix(const void*); extern "C" void* __cxa_vec_new3_nid_postfix(std::size_t, std::size_t, std::size_t, void(*)(void*), void(*)(void*), void*(*)(std::size_t), void(*)(void*, std::size_t)); extern "C" void __cxa_vec_delete3_nid_postfix(void*, std::size_t, std::size_t, void(*)(void*), void(*)(void*, std::size_t)); @@ -198,8 +203,14 @@ int main() { try { _ZSt13_Xrange_errorPKc_nid_postfix("not a logic error"); } catch (const std::logic_error&) { std::abort(); } catch (const std::range_error&) {} + bool badFunctionCall = false; + try { _ZNSt8__sce_v219_Xbad_function_callEv_nid_postfix(); } + catch (const std::bad_function_call&) { badFunctionCall = true; } + if (!badFunctionCall) std::abort(); try { __cxa_bad_cast_nid_postfix(); } catch (const std::exception& value) { assert(value.what() != nullptr); } + try { _ZSt19_Throw_bad_weak_ptrv_nid_postfix(); std::abort(); } + catch (const std::bad_weak_ptr& value) { if (std::strcmp(value.what(), "bad_weak_ptr") != 0) std::abort(); } try { _ZNKSt9exception6_RaiseEv_nid_postfix(nullptr); assert(false); } catch (const std::invalid_argument&) {} try { _ZNKSt9exception6_RaiseEv_nid_postfix(&object); assert(false); } diff --git a/core/libs/tests/Formatting.cpp b/core/libs/tests/Formatting.cpp index 12a89c00f..6cccc4a31 100644 --- a/core/libs/tests/Formatting.cpp +++ b/core/libs/tests/Formatting.cpp @@ -8,6 +8,7 @@ int APS5_VABI snprintf_nid_postfix(char*, size_t, const char*, ...); int APS5_VABI sprintf_nid_postfix(char*, const char*, ...); int APS5_VABI printf_nid_postfix(const char*, ...); int APS5_VABI libc_printf_nid_postfix(const char*, ...); +int APS5_VABI sscanf_nid_postfix(const char*, const char*, ...); int APS5_VABI vsnprintf_nid_postfix(char*, size_t, const char*, VaList*); int APS5_VABI vprintf_nid_postfix(const char*, VaList*); } @@ -38,6 +39,7 @@ static int APS5_VABI PrintList(const char* format, ...) { return result; } +#ifdef _WIN32 static bool CheckWidePrecision() { const char16_t input[] = u"A\u00e9\u20ac\U0001f600Z"; const char* expected[] = {"", "A", "A", "A\xc3\xa9", "A\xc3\xa9", "A\xc3\xa9", "A\xc3\xa9\xe2\x82\xac", @@ -65,6 +67,7 @@ static bool CheckWidePrecision() { Require(snprintf_nid_postfix(output, sizeof(output), "%.1s", "\xc3\xa9") == 1 && static_cast(output[0]) == 0xc3 && output[1] == 0); return correct; } +#endif __attribute__((noinline)) static void APS5_VABI RunChecks() { char buffer[1024]; @@ -101,6 +104,15 @@ __attribute__((noinline)) static void APS5_VABI RunChecks() { Require(snprintf_nid_postfix(buffer, 1, "%d", 123) == 3 && buffer[0] == 0); Require(sprintf_nid_postfix(buffer, "%hhd %hhu %hd %hu %%", 255, 257, 65535, 65537) == 11); Require(std::strcmp(buffer, "-1 1 -1 1 %") == 0); + Require(sprintf_nid_postfix(buffer, "%d %d %d %d %d %d", 1, 2, 3, 4, 5, 6) == 11); + Require(std::strcmp(buffer, "1 2 3 4 5 6") == 0); + int first = 0, second = 0, third = 0, fourth = 0, fifth = 0, sixth = 0; + char word[16] = {}; + double real = 0.0; + Require(sscanf_nid_postfix("1 2 3 4 5 6 seven 8.5", "%d %d %d %d %d %d %15s %lf", + &first, &second, &third, &fourth, &fifth, &sixth, word, &real) == 8); + Require(first == 1 && second == 2 && third == 3 && fourth == 4 && fifth == 5 && sixth == 6); + Require(std::strcmp(word, "seven") == 0 && real == 8.5); long long count = -1; int smallCount = -1; Require(snprintf_nid_postfix(buffer, 3, "abcd%lnEF%n", &count, &smallCount) == 6); @@ -109,8 +121,13 @@ __attribute__((noinline)) static void APS5_VABI RunChecks() { Require(buffer[0] == 'a' && buffer[1] == 0 && buffer[2] == 'b' && buffer[3] == 0); Require(printf_nid_postfix("printf: %d %.1f\n", 7, 2.5) == 14); Require(libc_printf_nid_postfix("libc_printf: %s\n", "OK") == 16); + Require(libc_printf_nid_postfix("%d %d %d %d %d %d %d\n", 1, 22, 333, 4444, 55555, 666666, 7777777) == 35); Require(PrintList("vprintf: %d\n", 42) == 12); std::puts("Formatting checks passed: 10000 iterations"); } +#ifdef _WIN32 int main() { RunChecks(); return CheckWidePrecision() ? 0 : 1; } +#else +int main() { RunChecks(); } +#endif diff --git a/core/libs/tests/GuestAio.cpp b/core/libs/tests/GuestAio.cpp index 7193da905..84d3f22cd 100644 --- a/core/libs/tests/GuestAio.cpp +++ b/core/libs/tests/GuestAio.cpp @@ -3,6 +3,7 @@ #include "prx/libkernel/File/include/FileFlags.hpp" #include "prx/libkernel/KernelErrors.hpp" #include +#include #include #include #include @@ -11,6 +12,8 @@ #include #include #include +#include +#include extern "C" { int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode); int APS5_VABI sceKernelClose(int d); @@ -178,5 +181,30 @@ int main() { Check(sceKernelAioDeleteRequests(badIds, 2, untouched) == SCE_KERNEL_ERROR_EINVAL); Check(untouched[0] == 7 && untouched[1] == 7); Check(sceKernelAioDeleteRequests(writeIds, 2, nullptr) == SCE_KERNEL_ERROR_EFAULT); + const auto shared = root / "shared.bin"; + std::string sharedContents; + for (char letter : {'A', 'B', 'C', 'D'}) sharedContents.append(1024, letter); + { std::ofstream stream(shared, std::ios::binary); stream << sharedContents; } + const int sharedFd = sceKernelOpen(shared.string().c_str(), SCE_KERNEL_O_RDONLY, 0); + Check(sharedFd >= 0); + std::atomic misread{false}; + std::vector readers; + for (std::int64_t offset = 0; offset < 4096; offset += 1024) { + readers.emplace_back([&, offset] { + for (int i = 0; i < 2000 && !misread; ++i) { + std::array bytes{}; + KernelAioResult result{-1, 0}; + KernelAioRwRequest request{offset, bytes.size(), bytes.data(), &result, sharedFd}; + std::int32_t requestId = 0; + if (sceKernelAioSubmitReadCommands(&request, 1, 0, &requestId) != 0 || result.state != 3 || + result.return_value != 16 || std::memcmp(bytes.data(), sharedContents.data() + offset, 16) != 0) { + misread = true; + } + } + }); + } + for (auto& reader : readers) reader.join(); + Check(!misread); + Check(sceKernelClose(sharedFd) == 0); Check(std::filesystem::remove_all(root) > 0); } diff --git a/core/libs/tests/GuestAjm.cpp b/core/libs/tests/GuestAjm.cpp index b2878b8a9..0c88f35d7 100644 --- a/core/libs/tests/GuestAjm.cpp +++ b/core/libs/tests/GuestAjm.cpp @@ -563,6 +563,85 @@ bool ControlRefused(std::uint32_t context, std::uint32_t instance, std::uint64_t return Refused(context, info); } +struct SyntheticAt9Channel { + std::uint8_t header; + std::uint8_t fill; + std::size_t blockBytes; +}; + +std::vector SyntheticAt9Block(const SyntheticAt9Channel& channel, std::uint32_t frame) { + std::vector block(channel.blockBytes, channel.fill); + block[0] = frame == 0 ? channel.header : static_cast(channel.header | 0x80u); + return block; +} + +std::vector DecodeAt9Superframe(std::uint32_t context, const std::uint8_t (&config)[4], const std::vector& superframe, std::size_t channels) { + std::uint32_t instance = 0; + Require(sceAjmInstanceCreate(context, 1, 0, &instance) == 0); + constexpr std::uint64_t runMultipleFrames = 1ull << 12; + constexpr std::uint64_t sidebandStream = 1ull << 47; + std::vector batch(4096); + std::vector pcm(1024 * channels); + std::int32_t initResult[2] = {-1, -1}; + struct { + DecodeSideband stream; + std::uint32_t frames; + std::uint32_t reserved; + } decoded{}; + AjmBatchInfo info{}; + Require(sceAjmBatchInitialize(batch.data(), batch.size(), &info) == 0); + Require(sceAjmBatchJobInitialize(&info, instance, config, sizeof(config), initResult) == 0); + Require(sceAjmBatchJobRun(&info, instance, sidebandStream | runMultipleFrames, superframe.data(), superframe.size(), pcm.data(), pcm.size() * sizeof(std::int16_t), &decoded, sizeof(decoded)) == 0); + Submit(context, info); + Require(initResult[0] == 0); + Require(decoded.stream.result == 0 && static_cast(decoded.stream.inputConsumed) == superframe.size()); + Require(static_cast(decoded.stream.outputWritten) == pcm.size() * sizeof(std::int16_t) && decoded.stream.totalDecodedSamples == 1024 && decoded.frames == 4); + Require(sceAjmInstanceDestroy(context, instance) == 0); + return pcm; +} + +void TestMultichannelAt9(std::uint32_t context) { + constexpr int invalidParameter = static_cast(0x80930005); + const std::uint8_t thirdOrder[4] = {0x30, 0x73, 0xC1, 0x7E}; + AjmDecAt9ConfigDataInfo parsed{}; + Require(sceAjmDecAt9ParseConfigData(thirdOrder, &parsed) == 0); + Require(parsed.channels == 16 && parsed.sample_rate == 48000 && parsed.frame_samples_per_channel == 256); + Require(parsed.superframe_samples_per_channel == 1024 && parsed.superframe_size == 6144); + const std::uint8_t validationBitSet[4] = {0x30, 0x73, 0xE1, 0x7E}; + Require(sceAjmDecAt9ParseConfigData(validationBitSet, &parsed) == invalidParameter); + + const SyntheticAt9Channel channels[2] = {{0x00, 0x07, 27}, {0x01, 0x0E, 22}}; + const std::uint8_t mono[4] = {0xFE, 0x70, 0x0B, 0xF0}; + std::vector reference[2]; + for (std::size_t channel = 0; channel < 2; ++channel) { + std::vector superframe; + for (std::uint32_t frame = 0; frame < 4; ++frame) { + const auto block = SyntheticAt9Block(channels[channel], frame); + superframe.insert(superframe.end(), block.begin(), block.end()); + } + superframe.resize(384, 0x01); + reference[channel] = DecodeAt9Superframe(context, mono, superframe, 1); + } + + const std::uint8_t stereoAmbisonic[4] = {0x30, 0x70, 0x41, 0x7E}; + std::vector superframe; + for (std::uint32_t frame = 0; frame < 4; ++frame) { + for (const auto& channel : channels) { + auto block = SyntheticAt9Block(channel, frame); + if (frame == 3) block.resize(384 - 3 * channel.blockBytes, 0x01); + superframe.insert(superframe.end(), block.begin(), block.end()); + } + } + Require(superframe.size() == 768); + const auto decoded = DecodeAt9Superframe(context, stereoAmbisonic, superframe, 2); + bool audible = false; + for (std::size_t sample = 0; sample < 1024; ++sample) { + Require(decoded[sample * 2] == reference[0][sample] && decoded[sample * 2 + 1] == reference[1][sample]); + audible = audible || reference[0][sample] != 0 || reference[1][sample] != 0; + } + Require(audible); +} + void TestControlAt9(std::uint32_t context) { std::uint32_t instance = 0; Require(sceAjmInstanceCreate(context, 1, 0, &instance) == 0); @@ -903,6 +982,7 @@ int main() { TestBatchWaitRelease(context); TestBatchCancel(context); TestControlAt9(context); + TestMultichannelAt9(context); TestControlMp3(context); TestControlOpus(context); TestResampleOpus(context); diff --git a/core/libs/tests/GuestAmpr.cpp b/core/libs/tests/GuestAmpr.cpp index c36095250..21fa65958 100644 --- a/core/libs/tests/GuestAmpr.cpp +++ b/core/libs/tests/GuestAmpr.cpp @@ -1,3 +1,4 @@ +#include "SceTypes.hpp" #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libkernel/Apr/include/AprCommandBuffer.hpp" #include @@ -15,6 +16,7 @@ extern "C" { int APS5_VABI sceAmprCommandBufferConstructor(Apr::CommandBufferObject*); int APS5_VABI sceAmprAprCommandBufferConstructor(Apr::CommandBufferObject*, std::uint64_t*, std::uint64_t*); int APS5_VABI sceAmprCommandBufferSetBuffer(Apr::CommandBufferObject*, void*, std::uint32_t); +void* APS5_VABI sceAmprCommandBufferClearBuffer(Apr::CommandBufferObject*); std::uint32_t APS5_VABI sceAmprCommandBufferGetCurrentOffset(const Apr::CommandBufferObject*); std::uint32_t APS5_VABI sceAmprCommandBufferGetNumCommands(const Apr::CommandBufferObject*); int APS5_VABI sceAmprCommandBufferWriteAddressOnCompletion(Apr::CommandBufferObject*, volatile std::uint64_t*, std::uint64_t); @@ -112,6 +114,15 @@ std::int64_t APS5_VABI sceAmprAmmMeasureAmmCommandSizeModifyProtect(std::uint64_ std::int64_t APS5_VABI sceAmprAmmMeasureAmmCommandSizeModifyProtectWithGpuMaskId(std::uint64_t, std::uint64_t, std::int32_t, std::int32_t, std::uint8_t); std::int64_t APS5_VABI sceAmprAmmMeasureAmmCommandSizeModifyMtypeProtect(std::uint64_t, std::uint64_t, std::int32_t, std::int32_t, std::int32_t); std::int64_t APS5_VABI sceAmprAmmMeasureAmmCommandSizeModifyMtypeProtectWithGpuMaskId(std::uint64_t, std::uint64_t, std::int32_t, std::int32_t, std::int32_t, std::uint8_t); +int APS5_VABI sceKernelCreateEqueue(KernelEqueue*, const char*); +int APS5_VABI sceKernelDeleteEqueue(KernelEqueue); +int APS5_VABI sceKernelWaitEqueue(KernelEqueue, KernelEvent*, int, int*, const KernelUseconds*); +int APS5_VABI sceKernelAddAmprEvent(KernelEqueue, int, void*); +int APS5_VABI sceKernelDeleteAmprEvent(KernelEqueue, int); +int APS5_VABI sceKernelGetEventFilter(const KernelEvent*); +std::uintptr_t APS5_VABI sceKernelGetEventId(const KernelEvent*); +std::intptr_t APS5_VABI sceKernelGetEventData(const KernelEvent*); +void* APS5_VABI sceKernelGetEventUserData(const KernelEvent*); } static void RequireAt(bool value, int line) { @@ -247,6 +258,24 @@ void TestSubmission() { Require(first == 0x1111 && second == 0x2222); } +void TestKernelEventQueue() { + KernelEqueue eq = 0; + int userData = 0; + Require(sceKernelCreateEqueue(&eq, "ampr") == 0); + Require(sceKernelAddAmprEvent(eq, 7, &userData) == 0); + Recorder recorder; + Require(sceAmprCommandBufferWriteKernelEventQueue_04_00(&recorder.buffer, static_cast(eq), 7, 0x1234, 0) == 0); + Require(sceKernelAprSubmitCommandBuffer(&recorder.buffer, 0) == 0); + KernelEvent event{}; + int count = 0; + const KernelUseconds poll = 0; + Require(sceKernelWaitEqueue(eq, &event, 1, &count, &poll) == 0 && count == 1); + Require(sceKernelGetEventFilter(&event) == -25); + Require(sceKernelGetEventId(&event) == 7 && sceKernelGetEventData(&event) == 0x1234 && sceKernelGetEventUserData(&event) == &userData); + Require(sceKernelDeleteAmprEvent(eq, 7) == 0); + Require(sceKernelDeleteEqueue(eq) == 0); +} + void TestWaits() { Recorder recorder; alignas(8) std::uint64_t value = 5; @@ -418,6 +447,15 @@ void TestVersionedCounters() { Require(sceAmprMeasureCommandSizeWriteCounter_04_00(0, size4, 0, 5) == rejected); } +void TestClearBuffer() { + Recorder recorder; + Require(sceAmprCommandBufferNop(&recorder.buffer, 1) == 0); + Require(recorder.Offset() != 0 && recorder.Commands() == 1); + Require(sceAmprCommandBufferClearBuffer(&recorder.buffer) == recorder.memory.data()); + Require(recorder.buffer.base == nullptr && recorder.buffer.size == 0 && recorder.Offset() == 0 && recorder.Commands() == 0); + Require(sceAmprCommandBufferClearBuffer(&recorder.buffer) == nullptr); +} + void TestConstructed() { Recorder recorder; const std::uint8_t payload[5] = {1, 2, 3, 4, 5}; @@ -902,12 +940,14 @@ int main() { TestRejectedArguments(); TestFullBuffer(); TestSubmission(); + TestKernelEventQueue(); TestWaits(); TestCounters(); TestRejectedWaitsAndCounters(); TestNops(); TestVersionedCommands(); TestVersionedCounters(); + TestClearBuffer(); TestConstructed(); TestGatherScatter(); TestAmm(); diff --git a/core/libs/tests/GuestAppContent.cpp b/core/libs/tests/GuestAppContent.cpp index 90ee7940c..6780ff814 100644 --- a/core/libs/tests/GuestAppContent.cpp +++ b/core/libs/tests/GuestAppContent.cpp @@ -1,27 +1,44 @@ #include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" +#include #include #include #include #include extern "C" { +int APS5_VABI sceAppContentInitialize(const AppContentInitParam*, AppContentBootParam*); int APS5_VABI sceAppContentAddcontMount(uint32_t, const NpUnifiedEntitlementLabel*, AppContentMountPoint*); int APS5_VABI sceAppContentAddcontUnmount(const AppContentMountPoint*); +int APS5_VABI sceAppContentTemporaryDataMount2(uint32_t, AppContentMountPoint*); +int APS5_VABI sceAppContentTemporaryDataUnmount(const AppContentMountPoint*); +int APS5_VABI sceAppContentTemporaryDataFormat(const AppContentMountPoint*); +int APS5_VABI sceAppContentTemporaryDataGetAvailableSpaceKb(const AppContentMountPoint*, size_t*); +int APS5_VABI access_nid_postfix(const char*, int); +int* APS5_VABI __error_nid_postfix(); int APS5_VABI sceAppContentGetAddcontInfo(uint32_t, const NpUnifiedEntitlementLabel*, void*); int APS5_VABI sceAppContentGetAddcontInfoList(uint32_t, void*, uint32_t, uint32_t*); int APS5_VABI sceAppContentDownloadDataGetAvailableSpaceKb(const AppContentMountPoint*, size_t*); +int APS5_VABI sceAppContentAppParamGetInt(uint32_t, int32_t*); } static constexpr int ErrorParameter = static_cast(0x80D90002); static constexpr int ErrorNotFound = static_cast(0x80D90005); static constexpr int ErrorDrmNoEntitlement = static_cast(0x80D90007); +static constexpr int ErrorBusy = static_cast(0x80D90003); +static constexpr int ErrorNotMounted = static_cast(0x80D90004); static void Require(bool value) { if (!value) std::abort(); } int main() { + const auto original = std::filesystem::current_path(); + const auto directory = original / ("appcontent-test-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + Require(std::filesystem::create_directory(directory)); + std::filesystem::current_path(directory); std::filesystem::create_directories("app0/sce_sys"); { std::ofstream param("app0/sce_sys/param.json", std::ios::binary); - param << R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}},"downloadDataSize":0})"; + param << R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}},"downloadDataSize":0,)" + R"("userDefinedParam1":23566,"userDefinedParam2":-7,"userDefinedParam3":0,"userDefinedParam4":2147483647})"; Require(static_cast(param)); } AppContentMountPoint download{}; @@ -31,6 +48,17 @@ int main() { Require(availableKb == 0); Require(sceAppContentDownloadDataGetAvailableSpaceKb(&download, nullptr) == ErrorParameter); + const int32_t expectedParams[] = {3, 23566, -7, 0, 2147483647}; + for (uint32_t id = 0; id < 5; ++id) { + int32_t value = 0x5a5a5a5a; + Require(sceAppContentAppParamGetInt(id, &value) == 0); + Require(value == expectedParams[id]); + } + int32_t paramValue = 0x5a5a5a5a; + Require(sceAppContentAppParamGetInt(5, ¶mValue) == ErrorParameter); + Require(paramValue == 0x5a5a5a5a); + Require(sceAppContentAppParamGetInt(0, nullptr) == ErrorParameter); + NpUnifiedEntitlementLabel label{}; std::memcpy(&label, "ADDCONT000000001", 16); AppContentMountPoint mountPoint{}; @@ -44,6 +72,49 @@ int main() { Require(sceAppContentAddcontUnmount(&mountPoint) == ErrorNotFound); Require(sceAppContentAddcontUnmount(nullptr) == ErrorParameter); + std::filesystem::create_directory(directory / "temp0"); + { std::ofstream file(directory / "temp0" / "old.txt"); file << "previous run"; } + AppContentBootParam boot{}; + Require(sceAppContentInitialize(nullptr, &boot) == 0); + Require(access_nid_postfix("/temp0/old.txt", 0) == -1); + Require(sceAppContentTemporaryDataUnmount(nullptr) == ErrorParameter); + Require(sceAppContentTemporaryDataUnmount(&mountPoint) == ErrorNotMounted); + Require(sceAppContentTemporaryDataMount2(0, nullptr) == ErrorParameter); + std::memset(&mountPoint, 0x5a, sizeof(mountPoint)); + Require(sceAppContentTemporaryDataMount2(2, &mountPoint) == ErrorParameter); + Require(std::memcmp(&mountPoint, &untouched, sizeof(mountPoint)) == 0); + Require(sceAppContentTemporaryDataMount2(0, &mountPoint) == 0); + Require(std::strcmp(mountPoint.data, "/temp0") == 0); + const auto backing = ResolvePath_nid_no_patch("/temp0"); + { std::ofstream file(backing / "keep.txt"); file << "retained"; } + { std::ofstream file(directory / "temp01"); file << "unrelated"; } + AppContentMountPoint second = untouched; + Require(sceAppContentTemporaryDataMount2(1, &second) == ErrorBusy); + Require(std::memcmp(&second, &untouched, sizeof(second)) == 0); + Require(std::filesystem::exists(backing / "keep.txt")); + Require(sceAppContentTemporaryDataUnmount(&second) == ErrorNotMounted); + Require(access_nid_postfix("/temp0/keep.txt", 0) == 0); + size_t available = 0; + Require(sceAppContentTemporaryDataGetAvailableSpaceKb(&mountPoint, &available) == 0 && available > 0); + Require(sceAppContentTemporaryDataUnmount(&mountPoint) == 0); + Require(std::filesystem::exists(backing / "keep.txt")); + Require(access_nid_postfix("/temp0/keep.txt", 0) == -1 && *__error_nid_postfix() == 2); + Require(access_nid_postfix("/temp0", 0) == -1 && *__error_nid_postfix() == 2); + Require(access_nid_postfix("/temp01", 0) == 0); + Require(sceAppContentTemporaryDataUnmount(&mountPoint) == ErrorNotMounted); + available = 123; + Require(sceAppContentTemporaryDataGetAvailableSpaceKb(&mountPoint, &available) == ErrorNotMounted && available == 123); + Require(sceAppContentTemporaryDataFormat(&mountPoint) == ErrorNotMounted); + Require(sceAppContentTemporaryDataMount2(0, &mountPoint) == 0); + Require(access_nid_postfix("/temp0/keep.txt", 0) == 0); + Require(sceAppContentTemporaryDataFormat(nullptr) == ErrorParameter); + Require(sceAppContentTemporaryDataFormat(&mountPoint) == 0); + Require(std::filesystem::is_empty(backing)); + { std::ofstream file(backing / "remove.txt"); file << "format"; } + Require(sceAppContentTemporaryDataUnmount(&mountPoint) == 0); + Require(sceAppContentTemporaryDataMount2(1, &mountPoint) == 0); + Require(std::filesystem::is_empty(backing)); + Require(sceAppContentTemporaryDataUnmount(&mountPoint) == 0); unsigned char info[24]; std::memset(info, 0x5a, sizeof(info)); unsigned char untouchedInfo[24]; @@ -70,4 +141,6 @@ int main() { Require(sceAppContentGetAddcontInfoList(0, nullptr, 0, nullptr) == ErrorParameter); Require(sceAppContentGetAddcontInfoList(0, info, 0, nullptr) == ErrorParameter); Require(sceAppContentGetAddcontInfoList(0, nullptr, 1, nullptr) == ErrorParameter); + std::filesystem::current_path(original); + std::filesystem::remove_all(directory); } diff --git a/core/libs/tests/GuestAudioIn.cpp b/core/libs/tests/GuestAudioIn.cpp index 27c717f5c..8786835e8 100644 --- a/core/libs/tests/GuestAudioIn.cpp +++ b/core/libs/tests/GuestAudioIn.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -43,7 +44,7 @@ void TestValidation() { } void TestCapture() { - const auto path = std::filesystem::temp_directory_path() / "anyps5_audio_in_capture.raw"; + const auto path = std::filesystem::temp_directory_path() / ("anyps5_audio_in_capture-" + std::to_string(std::random_device{}()) + ".raw"); std::vector recorded(256 * 2 * 4); for (std::size_t i = 0; i < recorded.size(); ++i) recorded[i] = static_cast(i * 7 - 3000); std::ofstream(path, std::ios::binary).write(reinterpret_cast(recorded.data()), static_cast(recorded.size() * 2)); diff --git a/core/libs/tests/GuestBluetoothHid.cpp b/core/libs/tests/GuestBluetoothHid.cpp new file mode 100644 index 000000000..f85b1d9ae --- /dev/null +++ b/core/libs/tests/GuestBluetoothHid.cpp @@ -0,0 +1,58 @@ +#include "prx/libc/include/general/VabiMacros.hpp" + +#include +#include +#include +#include + +extern "C" { +std::int32_t APS5_VABI sceBluetoothHidInit(); +std::int32_t APS5_VABI sceBluetoothHidRegisterCallback(void* callback, std::int32_t flags, void* param); +std::int32_t APS5_VABI sceBluetoothHidUnregisterCallback(); +std::int32_t APS5_VABI sceBluetoothHidRegisterDevice(std::uint16_t vendorId, std::uint16_t productId); +std::int32_t APS5_VABI sceBluetoothHidUnregisterDevice(std::uint16_t vendorId, std::uint16_t productId); +int APS5_VABI sceBluetoothHidGetInputReport(); +} + +namespace { + +int callbackCalls = 0; + +void APS5_VABI Callback() { + ++callbackCalls; +} + +void Require(bool condition, const char* message) { + if (!condition) { + std::fprintf(stderr, "BluetoothHid: %s\n", message); + std::abort(); + } +} + +} + +int main() { + Require(sceBluetoothHidInit() == 0, "initialization failed"); + std::uint8_t param[16]{}; + Require(sceBluetoothHidRegisterCallback(reinterpret_cast(&Callback), 0, param) == 0, "callback registration failed"); + Require(sceBluetoothHidRegisterDevice(0x44f, 0x1000) == 0, "device registration failed"); + Require(callbackCalls == 0, "a device connection was reported"); + bool threw = false; + try { + sceBluetoothHidRegisterCallback(nullptr, 0, param); + } catch (const std::invalid_argument&) { + threw = true; + } + Require(threw, "a null callback was accepted"); + threw = false; + try { + sceBluetoothHidGetInputReport(); + } catch (const std::runtime_error&) { + threw = true; + } + Require(threw, "an input report without a device did not throw"); + Require(sceBluetoothHidUnregisterDevice(0x44f, 0x1000) == 0, "device unregistration failed"); + Require(sceBluetoothHidUnregisterCallback() == 0, "callback unregistration failed"); + std::puts("BluetoothHid tests passed"); + return 0; +} diff --git a/core/libs/tests/GuestCondRecursiveMutex.cpp b/core/libs/tests/GuestCondRecursiveMutex.cpp new file mode 100644 index 000000000..92f716607 --- /dev/null +++ b/core/libs/tests/GuestCondRecursiveMutex.cpp @@ -0,0 +1,101 @@ +#include "SceTypes.hpp" +#include + +extern "C" { +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +int APS5_VABI scePthreadMutexattrInit(PthreadMutexattr* attr); +int APS5_VABI scePthreadMutexattrDestroy(PthreadMutexattr* attr); +int APS5_VABI scePthreadMutexattrSettype(PthreadMutexattr* attr, int type); +int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr, const char* name); +int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexLock(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex); +int APS5_VABI scePthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr, const char* name); +int APS5_VABI scePthreadCondDestroy(PthreadCond* cond); +int APS5_VABI scePthreadCondSignal(PthreadCond* cond); +int APS5_VABI scePthreadCondWait(PthreadCond* cond, PthreadMutex* mutex); +int APS5_VABI scePthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, unsigned int usec); +int APS5_VABI pthread_cond_wait_nid_postfix(PthreadCond* cond, PthreadMutex* mutex); +int APS5_VABI pthread_cond_timedwait_nid_postfix(PthreadCond* cond, PthreadMutex* mutex, const KernelTimespec* abstime); +int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* tp); +} + +static constexpr int SCE_OK = 0; +static constexpr int SCE_KERNEL_ERROR_EPERM = static_cast(0x80020001); +static constexpr int SCE_KERNEL_ERROR_ETIMEDOUT = static_cast(0x8002003C); +static constexpr int POSIX_ETIMEDOUT = 60; +static constexpr int MUTEX_TYPE_RECURSIVE = 2; + +static void Require(bool value) { if (!value) std::abort(); } + +struct Context { + PthreadMutex mutex = nullptr; + PthreadCond cond = nullptr; + bool signalled = false; +}; + +static void* APS5_VABI Signaller(void* arg) { + auto& context = *static_cast(arg); + Require(scePthreadMutexLock(&context.mutex) == SCE_OK); + context.signalled = true; + Require(scePthreadCondSignal(&context.cond) == SCE_OK); + Require(scePthreadMutexUnlock(&context.mutex) == SCE_OK); + return nullptr; +} + +static int SceWait(Context& context) { + return scePthreadCondWait(&context.cond, &context.mutex); +} + +static int SceTimedwait(Context& context) { + const int result = scePthreadCondTimedwait(&context.cond, &context.mutex, 1000); + return result == SCE_KERNEL_ERROR_ETIMEDOUT ? SCE_OK : result; +} + +static int PosixWait(Context& context) { + return pthread_cond_wait_nid_postfix(&context.cond, &context.mutex); +} + +static int PosixTimedwait(Context& context) { + KernelTimespec deadline{}; + Require(clock_gettime_nid_postfix(0, &deadline) == 0); + deadline.tv_nsec += 1000000; + if (deadline.tv_nsec >= 1000000000) { + deadline.tv_nsec -= 1000000000; + ++deadline.tv_sec; + } + const int result = pthread_cond_timedwait_nid_postfix(&context.cond, &context.mutex, &deadline); + return result == POSIX_ETIMEDOUT ? SCE_OK : result; +} + +static void WaitWithRecursiveMutex(int (*wait)(Context&)) { + Context context; + PthreadMutexattr attr = nullptr; + Require(scePthreadMutexattrInit(&attr) == SCE_OK); + Require(scePthreadMutexattrSettype(&attr, MUTEX_TYPE_RECURSIVE) == SCE_OK); + Require(scePthreadMutexInit(&context.mutex, &attr, nullptr) == SCE_OK); + Require(scePthreadMutexattrDestroy(&attr) == SCE_OK); + Require(scePthreadCondInit(&context.cond, nullptr, nullptr) == SCE_OK); + + Require(scePthreadMutexLock(&context.mutex) == SCE_OK); + Require(scePthreadMutexLock(&context.mutex) == SCE_OK); + Pthread thread = nullptr; + Require(scePthreadCreate(&thread, nullptr, Signaller, &context, nullptr) == SCE_OK); + while (!context.signalled) + Require(wait(context) == SCE_OK); + Require(scePthreadMutexUnlock(&context.mutex) == SCE_OK); + Require(scePthreadMutexUnlock(&context.mutex) == SCE_OK); + Require(scePthreadMutexUnlock(&context.mutex) == SCE_KERNEL_ERROR_EPERM); + Require(scePthreadJoin(thread, nullptr) == SCE_OK); + + Require(scePthreadCondDestroy(&context.cond) == SCE_OK); + Require(scePthreadMutexDestroy(&context.mutex) == SCE_OK); +} + +int main() { + WaitWithRecursiveMutex(SceWait); + WaitWithRecursiveMutex(SceTimedwait); + WaitWithRecursiveMutex(PosixWait); + WaitWithRecursiveMutex(PosixTimedwait); +} diff --git a/core/libs/tests/GuestCurrentCpu.cpp b/core/libs/tests/GuestCurrentCpu.cpp new file mode 100644 index 000000000..b7d7d75cc --- /dev/null +++ b/core/libs/tests/GuestCurrentCpu.cpp @@ -0,0 +1,66 @@ +#include "SceTypes.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" +#include +#include +#include + +extern "C" { +int APS5_VABI sceKernelGetCurrentCpu(void); +KernelCpumask APS5_VABI sceKernelGetAvailableCpumask(void); +Pthread APS5_VABI scePthreadSelf(); +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +int APS5_VABI scePthreadSetaffinity(Pthread thread, KernelCpumask mask); +} + +static constexpr int SCE_OK = 0; +static constexpr unsigned MAX_HOST_CPUS = 256; +static constexpr int CALLS_PER_THREAD = 64; + +static void Require(bool value) { if (!value) std::abort(); } + +static bool InMask(int cpu, KernelCpumask mask) { + return cpu >= 0 && cpu < 64 && (mask >> cpu & 1) != 0; +} + +struct PinnedRun { + KernelCpumask affinity; + bool inside; +}; + +static void* APS5_VABI ReportWhilePinned(void* arg) { + auto* run = static_cast(arg); + run->inside = scePthreadSetaffinity(scePthreadSelf(), run->affinity) == SCE_OK; + for (int call = 0; call < CALLS_PER_THREAD && run->inside; ++call) + run->inside = InMask(sceKernelGetCurrentCpu(), run->affinity); + return nullptr; +} + +int main() { + const KernelCpumask available = sceKernelGetAvailableCpumask(); + + const std::array affinities{available, 0x1FFB, 0x3, 0x3F, 0x1000, 0x1, 0xFFFF, 0x10000}; + for (const KernelCpumask affinity : affinities) { + const KernelCpumask expected = (affinity & available) != 0 ? affinity & available : available; + KernelCpumask reached = 0; + for (unsigned hostCpu = 0; hostCpu < MAX_HOST_CPUS; ++hostCpu) { + const int cpu = GuestCpuFromHost(hostCpu, affinity); + Require(InMask(cpu, expected)); + Require(GuestCpuFromHost(hostCpu, affinity) == cpu); + reached |= KernelCpumask{1} << cpu; + if (hostCpu + 1 == static_cast(std::popcount(expected))) Require(reached == expected); + } + Require(reached == expected); + } + + for (int call = 0; call < CALLS_PER_THREAD; ++call) Require(InMask(sceKernelGetCurrentCpu(), available)); + + for (KernelCpumask cpu = 1; cpu != 0; cpu <<= 1) { + if ((available & cpu) == 0) continue; + PinnedRun run{cpu, false}; + Pthread thread = nullptr; + Require(scePthreadCreate(&thread, nullptr, ReportWhilePinned, &run, nullptr) == SCE_OK); + Require(scePthreadJoin(thread, nullptr) == SCE_OK); + Require(run.inside); + } +} diff --git a/core/libs/tests/GuestDemangle.cpp b/core/libs/tests/GuestDemangle.cpp new file mode 100644 index 000000000..ea2f1100c --- /dev/null +++ b/core/libs/tests/GuestDemangle.cpp @@ -0,0 +1,306 @@ +#include "prx/libc/include/ApplicationHeap.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +char* APS5_VABI __cxa_demangle_nid_postfix(const char*, char*, std::size_t*, int*); +void* APS5_VABI malloc_nid_postfix(std::size_t); +void APS5_VABI free_nid_postfix(void*); +} + +namespace { + +constexpr char MangledName[] = "_ZN3foo3barEi"; +constexpr char DemangledName[] = "foo::bar(int)"; + +struct Allocation { + void* pointer = nullptr; + std::size_t size = 0; +}; + +std::array allocations{}; +std::size_t allocationCalls = 0; +std::size_t allocationFailures = 0; +std::size_t reallocationCalls = 0; +std::size_t reallocationFailures = 0; +std::size_t unexpectedFrees = 0; +std::size_t unexpectedReallocations = 0; +std::size_t failures = 0; +bool failNextAllocation = false; +bool failNextReallocation = false; +bool throwOnAllocation = false; +bool throwOnReallocation = false; + +std::size_t FindAllocation(const void* pointer) { + for (std::size_t index = 0; index < allocations.size(); ++index) { + if (allocations[index].pointer == pointer) return index; + } + return allocations.size(); +} + +bool Owns(const void* pointer) { + return FindAllocation(pointer) != allocations.size(); +} + +void Check(bool condition, const char* message) { + if (condition) return; + std::fprintf(stderr, "guest demangle: %s\n", message); + ++failures; +} + +void* Track(void* pointer, std::size_t size) { + if (pointer == nullptr) return nullptr; + for (auto& allocation : allocations) { + if (allocation.pointer == nullptr) { + allocation = {pointer, size}; + return pointer; + } + } + std::free(pointer); + return nullptr; +} + +void Forget(const void* pointer) { + const std::size_t index = FindAllocation(pointer); + if (index != allocations.size()) allocations[index] = {}; +} + +void ForgetIfMoved(void* original, void* result) { + if (original != nullptr && result != nullptr && result != original) Forget(original); +} + +void* APS5_VABI GuestAllocate(std::size_t size) { + ++allocationCalls; + if (throwOnAllocation) throw std::runtime_error("guest allocator callback failure"); + if (failNextAllocation) { + failNextAllocation = false; + ++allocationFailures; + return nullptr; + } + const std::size_t actualSize = size == 0 ? 1 : size; + return Track(std::malloc(actualSize), actualSize); +} + +void APS5_VABI GuestFree(void* pointer) { + if (pointer == nullptr) return; + const std::size_t index = FindAllocation(pointer); + if (index == allocations.size()) { + ++unexpectedFrees; + return; + } + std::free(pointer); + allocations[index] = {}; +} + +void* APS5_VABI GuestCalloc(std::size_t count, std::size_t size) { + if (size != 0 && count > std::numeric_limits::max() / size) return nullptr; + const std::size_t total = count * size; + void* pointer = GuestAllocate(total); + if (pointer != nullptr) std::memset(pointer, 0, total); + return pointer; +} + +void* APS5_VABI GuestReallocate(void* pointer, std::size_t size) { + ++reallocationCalls; + if (throwOnReallocation) throw std::runtime_error("guest reallocator callback failure"); + if (failNextReallocation) { + failNextReallocation = false; + ++reallocationFailures; + return nullptr; + } + if (pointer == nullptr) return GuestAllocate(size); + if (size == 0) { + GuestFree(pointer); + return nullptr; + } + const std::size_t index = FindAllocation(pointer); + if (index == allocations.size()) { + ++unexpectedReallocations; + return nullptr; + } + void* replacement = std::realloc(pointer, size); + if (replacement != nullptr) allocations[index] = {replacement, size}; + return replacement; +} + +void* APS5_VABI GuestAlignedAllocate(std::size_t, std::size_t size) { + return GuestAllocate(size); +} + +void* APS5_VABI GuestRealign(void* pointer, std::size_t size, std::size_t) { + return GuestReallocate(pointer, size); +} + +int APS5_VABI GuestPosixAlign(void** pointer, std::size_t, std::size_t size) { + if (pointer == nullptr) return 22; + *pointer = GuestAllocate(size); + return *pointer == nullptr ? 12 : 0; +} + +void RegisterAllocator() { + std::array api{}; + api[0] = reinterpret_cast(&GuestAllocate); + api[1] = reinterpret_cast(&GuestFree); + api[2] = reinterpret_cast(&GuestCalloc); + api[3] = reinterpret_cast(&GuestReallocate); + api[4] = reinterpret_cast(&GuestAlignedAllocate); + api[5] = reinterpret_cast(&GuestRealign); + api[6] = reinterpret_cast(&GuestPosixAlign); + ApplicationHeapRegister_nid_no_patch(api.data()); +} + +void ReleaseIfReturned(void* result, void* original = nullptr) { + ForgetIfMoved(original, result); + free_nid_postfix(result != nullptr ? result : original); +} + +void TestNullOutputBuffer() { + std::size_t length = 0; + int status = 99; + char* result = __cxa_demangle_nid_postfix(MangledName, nullptr, &length, &status); + Check(result != nullptr, "null output buffer did not demangle"); + Check(status == 0, "null output buffer returned the wrong status"); + Check(result != nullptr && std::strcmp(result, DemangledName) == 0, "null output buffer returned the wrong string"); + Check(length >= sizeof(DemangledName), "null output buffer returned an insufficient length"); + Check(result != nullptr && Owns(result), "null output buffer was not allocated by the guest allocator"); + ReleaseIfReturned(result); +} + +void TestOptionalStatus() { + char* result = __cxa_demangle_nid_postfix(MangledName, nullptr, nullptr, nullptr); + Check(result != nullptr && std::strcmp(result, DemangledName) == 0, "optional status output failed"); + Check(result != nullptr && Owns(result), "optional status output was not guest-owned"); + ReleaseIfReturned(result); +} + +void TestLargeBufferReuse() { + auto* buffer = static_cast(malloc_nid_postfix(128)); + const std::size_t originalLength = 128; + std::size_t length = originalLength; + int status = 99; + char* result = __cxa_demangle_nid_postfix(MangledName, buffer, &length, &status); + Check(result == buffer, "sufficient guest buffer was not reused"); + Check(status == 0, "sufficient guest buffer returned the wrong status"); + Check(result != nullptr && std::strcmp(result, DemangledName) == 0, "sufficient guest buffer returned the wrong string"); + Check(length == originalLength, "sufficient guest buffer length changed"); + ReleaseIfReturned(result, buffer); +} + +void TestSmallBufferGrowth() { + auto* buffer = static_cast(malloc_nid_postfix(1)); + std::size_t length = 1; + const std::size_t reallocationsBefore = reallocationCalls; + int status = 99; + char* result = __cxa_demangle_nid_postfix(MangledName, buffer, &length, &status); + Check(result != nullptr, "too-small guest buffer did not grow"); + Check(status == 0, "grown guest buffer returned the wrong status"); + Check(result != nullptr && std::strcmp(result, DemangledName) == 0, "grown guest buffer returned the wrong string"); + Check(length >= sizeof(DemangledName), "grown guest buffer returned an insufficient length"); + Check(reallocationCalls == reallocationsBefore + 1, "too-small guest buffer bypassed guest reallocation"); + Check(result != nullptr && Owns(result), "grown buffer was not owned by the guest allocator"); + ReleaseIfReturned(result, buffer); +} + +void TestInvalidInputs() { + int status = 99; + char* result = __cxa_demangle_nid_postfix("not_a_mangled_name", nullptr, nullptr, &status); + Check(result == nullptr && status == -2, "invalid mangled name did not return status -2"); + status = 99; + result = __cxa_demangle_nid_postfix(nullptr, nullptr, nullptr, &status); + Check(result == nullptr && status == -3, "null mangled name did not return status -3"); + + auto* buffer = static_cast(malloc_nid_postfix(32)); + std::memcpy(buffer, "unchanged", sizeof("unchanged")); + result = __cxa_demangle_nid_postfix(MangledName, buffer, nullptr, &status); + Check(result == nullptr && status == -3, "missing length for a supplied buffer did not return status -3"); + Check(std::strcmp(buffer, "unchanged") == 0, "invalid supplied-buffer arguments modified the buffer"); + free_nid_postfix(buffer); +} + +void TestAllocationFailure() { + const std::size_t failuresBefore = allocationFailures; + failNextAllocation = true; + std::size_t length = 0; + int status = 99; + char* result = __cxa_demangle_nid_postfix(MangledName, nullptr, &length, &status); + Check(result == nullptr, "guest allocation failure returned a buffer"); + Check(status == -1, "guest allocation failure did not return status -1"); + Check(allocationFailures == failuresBefore + 1, "demangler bypassed failed guest allocation"); + failNextAllocation = false; + ReleaseIfReturned(result); +} + +void TestReallocationFailure() { + auto* buffer = static_cast(malloc_nid_postfix(1)); + std::size_t length = 1; + const std::size_t failuresBefore = reallocationFailures; + failNextReallocation = true; + int status = 99; + char* result = __cxa_demangle_nid_postfix(MangledName, buffer, &length, &status); + Check(result == nullptr, "guest reallocation failure returned a buffer"); + Check(status == -1, "guest reallocation failure did not return status -1"); + Check(reallocationFailures == failuresBefore + 1, "demangler bypassed failed guest reallocation"); + Check(Owns(buffer), "failed guest reallocation did not preserve the input buffer"); + failNextReallocation = false; + ReleaseIfReturned(result, buffer); +} + +void TestAllocatorDiagnostics() { + throwOnAllocation = true; + bool allocationDiagnostic = false; + char* result = nullptr; + try { + result = __cxa_demangle_nid_postfix(MangledName, nullptr, nullptr, nullptr); + } catch (const std::runtime_error& error) { + allocationDiagnostic = std::strcmp(error.what(), "guest allocator callback failure") == 0; + } + throwOnAllocation = false; + Check(allocationDiagnostic, "allocator callback diagnostic was swallowed or changed"); + ReleaseIfReturned(result); + + auto* buffer = static_cast(malloc_nid_postfix(1)); + std::size_t length = 1; + throwOnReallocation = true; + bool reallocationDiagnostic = false; + result = nullptr; + try { + result = __cxa_demangle_nid_postfix(MangledName, buffer, &length, nullptr); + } catch (const std::runtime_error& error) { + reallocationDiagnostic = std::strcmp(error.what(), "guest reallocator callback failure") == 0; + } + throwOnReallocation = false; + Check(reallocationDiagnostic, "reallocator callback diagnostic was swallowed or changed"); + ReleaseIfReturned(result, buffer); +} + +void Cleanup() { + for (auto& allocation : allocations) { + if (allocation.pointer != nullptr) std::free(allocation.pointer); + allocation = {}; + } +} + +} + +int main() { + RegisterAllocator(); + TestNullOutputBuffer(); + TestOptionalStatus(); + TestLargeBufferReuse(); + TestSmallBufferGrowth(); + TestInvalidInputs(); + TestAllocationFailure(); + TestReallocationFailure(); + TestAllocatorDiagnostics(); + Check(unexpectedFrees == 0, "guest free received a pointer outside its allocator"); + Check(unexpectedReallocations == 0, "guest reallocation received a pointer outside its allocator"); + Cleanup(); + return failures == 0 ? 0 : 1; +} diff --git a/core/libs/tests/GuestDynamicLoader.cpp b/core/libs/tests/GuestDynamicLoader.cpp index 1b3172e67..74223730f 100644 --- a/core/libs/tests/GuestDynamicLoader.cpp +++ b/core/libs/tests/GuestDynamicLoader.cpp @@ -2,6 +2,9 @@ #include #include #include +#ifndef _WIN32 +#include +#endif #include #include extern "C" { @@ -13,6 +16,9 @@ int APS5_VABI _sceKernelRtldThreadAtexitIncrement_nid_postfix(const void*); int APS5_VABI _sceKernelRtldThreadAtexitDecrement_nid_postfix(const void*); } static void Require(bool value) { if (!value) std::abort(); } +#ifndef _WIN32 +extern "C" int APS5_VABI GuestDefaultScopeOnly(int a, int b) { return a * b; } +#endif template static bool ThrowsRuntimeError(TFunction function) { try { @@ -38,6 +44,7 @@ int main(int argc, char** argv) { auto add = reinterpret_cast(dlsym_nid_postfix(module, "GuestModuleAdd")); Require(add && add(17, 25) == 42); Require(dlsym_nid_postfix(reinterpret_cast(-2), "GuestModuleAdd") == reinterpret_cast(add)); + Require(dlsym_nid_postfix(nullptr, "GuestModuleAdd") == reinterpret_cast(add)); #ifndef _WIN32 auto mul = reinterpret_cast(dlsym_nid_postfix(module, "GuestModuleMul")); Require(mul && mul(6, 7) == 42); @@ -88,4 +95,20 @@ int main(int argc, char** argv) { Require(ThrowsRuntimeError([] { _sceKernelRtldThreadAtexitDecrement_nid_postfix(reinterpret_cast(&Require)); })); int local = 0; Require(ThrowsRuntimeError([&] { _sceKernelRtldThreadAtexitIncrement_nid_postfix(&local); })); +#ifndef _WIN32 + Require(dlsym_nid_postfix(nullptr, "malloc") == nullptr); + Require(dlsym_nid_postfix(nullptr, "dlopen") == nullptr); + Require(dlsym_nid_postfix(nullptr, "pthread_create") == nullptr); + Require(dlsym_nid_postfix(nullptr, "GuestDefaultScopeOnly") == reinterpret_cast(&GuestDefaultScopeOnly)); + Require(dlsym_nid_postfix(reinterpret_cast(-2), "GuestDefaultScopeOnly") == reinterpret_cast(&GuestDefaultScopeOnly)); + const auto hostLoaded = std::filesystem::absolute("anyps5-host-loaded-module-for-test.prx"); + std::filesystem::copy_file(argv[1], hostLoaded, std::filesystem::copy_options::overwrite_existing); + void* host = ::dlopen(hostLoaded.c_str(), RTLD_NOW | RTLD_GLOBAL); + Require(host != nullptr); + void* hostAdd = ::dlsym(host, "3qcG+yBRS+s"); + Require(hostAdd != nullptr); + Require(dlsym_nid_postfix(nullptr, "GuestModuleAdd") == hostAdd); + Require(dlsym_nid_postfix(nullptr, "malloc") == nullptr); + Require(::dlclose(host) == 0); +#endif } diff --git a/core/libs/tests/GuestErrors.cpp b/core/libs/tests/GuestErrors.cpp index 453dae221..52160d562 100644 --- a/core/libs/tests/GuestErrors.cpp +++ b/core/libs/tests/GuestErrors.cpp @@ -15,10 +15,12 @@ int main() { Require(*__error_nid_postfix() == 13); Require(std::strcmp(strerror_nid_postfix(78), "Function not implemented") == 0); char* parent = strerror_nid_postfix(22); + Require(*__error_nid_postfix() == 13); std::thread worker([] { Require(std::strcmp(strerror_nid_postfix(45), "Operation not supported") == 0); }); worker.join(); + *__error_nid_postfix() = 13; Require(std::strcmp(parent, "Invalid argument") == 0); char buffer[128]; for (int error = 0; error <= 96; ++error) { diff --git a/core/libs/tests/GuestFileVector.cpp b/core/libs/tests/GuestFileVector.cpp index a3cb50f35..1851fb20b 100644 --- a/core/libs/tests/GuestFileVector.cpp +++ b/core/libs/tests/GuestFileVector.cpp @@ -10,6 +10,16 @@ #include #include #include +#ifdef _WIN32 +#include +#include +static int MakePipe(int* ends) { return ::_pipe(ends, 64, _O_BINARY); } +static int ClosePipe(int end) { return ::_close(end); } +#else +#include +static int MakePipe(int* ends) { return ::pipe(ends); } +static int ClosePipe(int end) { return ::close(end); } +#endif struct GuestIovec { void* base; @@ -38,6 +48,7 @@ static void Check(bool value, int line) { static constexpr std::int64_t ErrorEbadf = static_cast(0x80020009u); static constexpr std::int64_t ErrorEfault = static_cast(0x8002000Eu); static constexpr std::int64_t ErrorEinval = static_cast(0x80020016u); +static constexpr std::int64_t ErrorEspipe = static_cast(0x8002001Du); static std::string Contents(const std::filesystem::path& path) { std::ifstream stream(path, std::ios::binary); @@ -81,6 +92,14 @@ int main() { Require(sceKernelLseek(file, 0, 1) == 13); Require(Contents(path) == "01ABCDE7ABCDE"); + GuestIovec gapped[3] = {{first, 2}, {nullptr, 0}, {second, 3}}; + Require(sceKernelPreadv(file, gapped, 3, 0) == 5); + Require(std::memcmp(first, "01", 2) == 0 && std::memcmp(second, "ABC", 3) == 0); + Require(sceKernelLseek(file, 2, 0) == 2); + Require(sceKernelReadv(file, gapped, 3) == 5); + Require(std::memcmp(first, "AB", 2) == 0 && std::memcmp(second, "CDE", 3) == 0); + Require(sceKernelLseek(file, 0, 1) == 7); + GuestIovec empty[1] = {{nullptr, 0}}; Require(sceKernelReadv(file, empty, 1) == 0); Require(sceKernelReadv(file, reads, 0) == 0); @@ -106,6 +125,17 @@ int main() { Require(sceKernelPreadv(file, reads, 2, 0) == ErrorEbadf); Require(sceKernelPwritev(file, writes, 2, 0) == ErrorEbadf); + int ends[2] = {}; + Require(MakePipe(ends) == 0); + char xyz[] = "xyz"; + GuestIovec message[1] = {{xyz, 3}}; + Require(sceKernelWritev(ends[1], message, 1) == 3); + Require(sceKernelReadv(ends[0], reads, 2) == 3); + Require(std::memcmp(first, "xyz", 3) == 0); + Require(sceKernelPreadv(ends[0], reads, 2, 0) == ErrorEspipe); + Require(sceKernelPwritev(ends[1], message, 1, 0) == ErrorEspipe); + Require(ClosePipe(ends[0]) == 0 && ClosePipe(ends[1]) == 0); + std::filesystem::remove_all(root); return 0; } diff --git a/core/libs/tests/GuestFilesystem.cpp b/core/libs/tests/GuestFilesystem.cpp index 2730e18f2..bd6a113c6 100644 --- a/core/libs/tests/GuestFilesystem.cpp +++ b/core/libs/tests/GuestFilesystem.cpp @@ -20,6 +20,7 @@ int APS5_VABI fchmod_nid_postfix(int, int); int APS5_VABI futimes_nid_postfix(int, const KernelTimeval*); int APS5_VABI socket_nid_postfix(int, int, int); int APS5_VABI sceKernelFsync(int); +int APS5_VABI sceKernelWriteThrottlingStatus(std::uint64_t*); int APS5_VABI sceKernelFtruncate(int, long long); int APS5_VABI sceKernelTruncate_nid_postfix(const char*, long long); int APS5_VABI sceKernelUtimes_nid_postfix(const char*, const void*); @@ -29,6 +30,7 @@ int APS5_VABI close_nid_postfix(int); int APS5_VABI stat_nid_postfix(const char*, FileStat*); int APS5_VABI unlink_nid_postfix(const char*); int APS5_VABI rmdir_nid_postfix(const char*); +int APS5_VABI mkdir_nid_postfix(const char*, unsigned short); int APS5_VABI sceKernelOpen(const char*, int, unsigned short); int APS5_VABI sceKernelClose(int); int APS5_VABI sceKernelStat(const char*, FileStat*); @@ -49,6 +51,9 @@ static void Check(bool value, int line) { #define Require(value) Check((value), __LINE__) int main() { Require(sceKernelDebugOutText(-1, "text") == static_cast(0x80020016u)); + std::uint64_t throttling[4] = {1, 2, 3, 4}; + Require(sceKernelWriteThrottlingStatus(throttling) == 0); + Require(throttling[0] == 0xffffffffu && throttling[1] == 0 && throttling[2] == 0 && throttling[3] == 0); Require(sceKernelDebugOutText(0, nullptr) == static_cast(0x8002000eu)); auto* captured = std::tmpfile(); Require(captured != nullptr); @@ -84,7 +89,10 @@ int main() { Require(close_nid_postfix(descriptors[0]) == 0); const auto root = std::filesystem::path("anyps5-filesystem-test-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); - Require(std::filesystem::create_directory(root)); + Require(mkdir_nid_postfix(root.string().c_str(), 0700) == 0); + Require(mkdir_nid_postfix(root.string().c_str(), 0700) == -1 && *__error_nid_postfix() == 17); + Require(mkdir_nid_postfix((root / "missing" / "child").string().c_str(), 0700) == -1 && *__error_nid_postfix() == 2); + Require(mkdir_nid_postfix(nullptr, 0700) == -1 && *__error_nid_postfix() == 14); const auto file = root / "file.txt"; { std::ofstream stream(file); stream << "retained until removal"; } Require(remove_nid_postfix(root.string().c_str()) == -1); diff --git a/core/libs/tests/GuestFont.cpp b/core/libs/tests/GuestFont.cpp index 7fd375b21..97e814da7 100644 --- a/core/libs/tests/GuestFont.cpp +++ b/core/libs/tests/GuestFont.cpp @@ -16,6 +16,9 @@ int APS5_VABI sceFontDestroyLibrary(FontLibrary*); int APS5_VABI sceFontCreateRenderer(const FontMemory*, const void*, FontRenderer*); int APS5_VABI sceFontDestroyRenderer(FontRenderer*); int APS5_VABI sceFontGetPixelResolution(FontLibrary, std::uint32_t*); +int APS5_VABI sceFontAttachDeviceCacheBuffer(FontLibrary, void*, std::uint32_t); +int APS5_VABI sceFontClearDeviceCache(FontLibrary); +int APS5_VABI sceFontDettachDeviceCacheBuffer(FontLibrary, void**, std::uint32_t*); int APS5_VABI sceFontSupportSystemFonts(FontLibrary); int APS5_VABI sceFontSupportExternalFonts(FontLibrary, std::uint32_t, std::uint32_t); int APS5_VABI sceFontOpenFontSet(FontLibrary, std::uint32_t, std::uint32_t, const FontOpenDetail*, FontHandle*); @@ -33,6 +36,9 @@ int APS5_VABI sceFontGetFontGlyphsCount(FontHandle, std::uint32_t*); int APS5_VABI sceFontGetCharGlyphCode(FontHandle, std::uint32_t, std::uint32_t*); int APS5_VABI sceFontGetFontResolution(FontHandle, std::uint32_t*, float*); int APS5_VABI sceFontGetRenderScaledKerning(FontHandle, std::uint32_t, std::uint32_t, FontKerning*); +int APS5_VABI sceFontGenerateCharGlyph(FontHandle, std::uint32_t, const FontGenerateGlyphDetail*, FontGlyph*); +int APS5_VABI sceFontGlyphDefineAttribute(FontGlyph, std::uint32_t, std::uint64_t); +int APS5_VABI sceFontDeleteGlyph(const FontMemory*, FontGlyph*); const void* APS5_VABI sceFontSelectLibraryFt(int); const void* APS5_VABI sceFontSelectRendererFt(int); } @@ -141,6 +147,32 @@ int main() { Require(sceFontGetPixelResolution(coarseLibrary, &subPixelCount) == SCE_FONT_ERROR_INVALID_LIBRARY && subPixelCount == 0); Require(sceFontDestroyLibrary(&coarseLibrary) == SCE_FONT_OK && coarseLibrary == nullptr); + void* cacheBuffer = nullptr; + std::uint32_t cacheSize = 0; + Require(sceFontDettachDeviceCacheBuffer(nullptr, &cacheBuffer, &cacheSize) == SCE_FONT_ERROR_INVALID_LIBRARY); + Require(sceFontDettachDeviceCacheBuffer(¬ALibrary, &cacheBuffer, &cacheSize) == SCE_FONT_ERROR_INVALID_LIBRARY); + Require(sceFontClearDeviceCache(library) == SCE_FONT_ERROR_NOT_ATTACHED_CACHE_BUFFER); + Require(sceFontDettachDeviceCacheBuffer(library, &cacheBuffer, &cacheSize) == SCE_FONT_ERROR_NOT_ATTACHED_CACHE_BUFFER); + + std::vector callerCache(0x2000); + Require(sceFontAttachDeviceCacheBuffer(library, callerCache.data(), 0x1000) == SCE_FONT_ERROR_INVALID_PARAMETER); + Require(sceFontAttachDeviceCacheBuffer(library, callerCache.data(), static_cast(callerCache.size())) == SCE_FONT_OK); + Require(sceFontAttachDeviceCacheBuffer(library, callerCache.data(), static_cast(callerCache.size())) == SCE_FONT_ERROR_ALREADY_ATTACHED); + Require(sceFontClearDeviceCache(library) == SCE_FONT_OK); + Require(sceFontDettachDeviceCacheBuffer(library, &cacheBuffer, &cacheSize) == SCE_FONT_OK && cacheBuffer == callerCache.data() && cacheSize == callerCache.size()); + Require(sceFontDettachDeviceCacheBuffer(library, &cacheBuffer, &cacheSize) == SCE_FONT_ERROR_NOT_ATTACHED_CACHE_BUFFER); + + Require(sceFontAttachDeviceCacheBuffer(library, nullptr, 0x2000) == SCE_FONT_OK); + cacheBuffer = nullptr; + cacheSize = 0; + Require(sceFontDettachDeviceCacheBuffer(library, &cacheBuffer, &cacheSize) == SCE_FONT_OK && cacheBuffer != nullptr && cacheSize == 0x2000); + Release(nullptr, cacheBuffer); + Require(sceFontDettachDeviceCacheBuffer(library, &cacheBuffer, &cacheSize) == SCE_FONT_ERROR_NOT_ATTACHED_CACHE_BUFFER); + + Require(sceFontAttachDeviceCacheBuffer(library, nullptr, 0x2000) == SCE_FONT_OK); + Require(sceFontDettachDeviceCacheBuffer(library, nullptr, nullptr) == SCE_FONT_OK); + Require(sceFontDettachDeviceCacheBuffer(library, nullptr, nullptr) == SCE_FONT_ERROR_NOT_ATTACHED_CACHE_BUFFER); + FontHandle font = reinterpret_cast(&memory); Require(sceFontOpenFontSet(library, SystemFontSet, 1, nullptr, &font) == SCE_FONT_ERROR_NO_SUPPORT_FUNCTION && font == nullptr); Require(sceFontSupportSystemFonts(library) == SCE_FONT_OK); @@ -217,6 +249,13 @@ int main() { Require(sceFontGetCharGlyphCode(font, 0, &glyphCode) == SCE_FONT_ERROR_NO_SUPPORT_CODE && glyphCode == 0); Require(sceFontSetResolutionDpi(font, 144, 144) == SCE_FONT_OK); Require(sceFontSetScalePixel(font, 100.0f, 100.0f) == SCE_FONT_OK); + FontGlyph glyph = nullptr; + Require(sceFontGenerateCharGlyph(font, 'A', nullptr, &glyph) == SCE_FONT_OK && glyph != nullptr); + Require(sceFontGlyphDefineAttribute(glyph, 0x11, 0) == SCE_FONT_OK); + Require(sceFontGlyphDefineAttribute(nullptr, 0x11, 0) == SCE_FONT_ERROR_INVALID_GLYPH); + FontGlyphOpaque notAGlyph{}; + Require(sceFontGlyphDefineAttribute(¬AGlyph, 0x11, 0) == SCE_FONT_ERROR_INVALID_GLYPH); + Require(sceFontDeleteGlyph(&memory, &glyph) == SCE_FONT_OK && glyph == nullptr); FontKerning kerning{1.0f, 1.0f, 1.0f, 1.0f}; Require(sceFontGetKerning(font, 'A', 'V', &kerning) == SCE_FONT_OK && KerningIs(kerning, -20.0f)); kerning = {1.0f, 1.0f, 1.0f, 1.0f}; diff --git a/core/libs/tests/GuestForceTlsDestructor.cpp b/core/libs/tests/GuestForceTlsDestructor.cpp index 991538f24..49264c929 100644 --- a/core/libs/tests/GuestForceTlsDestructor.cpp +++ b/core/libs/tests/GuestForceTlsDestructor.cpp @@ -1,6 +1,7 @@ #include "SceTypes.hpp" #include #include +#include #include #include @@ -68,7 +69,8 @@ bool ForceThrows(KernelModule handle) { int main() { self = ModuleOf(&dsoHandle); - void* otherDso = reinterpret_cast(&_sceLibcInternalThreadDtors_nid_postfix); + void* otherDso = dlsym(RTLD_NEXT, "_sceLibcInternalThreadDtors_nid_postfix"); + Require(otherDso != nullptr); const KernelModule other = ModuleOf(otherDso); Require(other != self && other != 0); int local = 0; diff --git a/core/libs/tests/GuestHeapRounding.cpp b/core/libs/tests/GuestHeapRounding.cpp new file mode 100644 index 000000000..1fd9527a6 --- /dev/null +++ b/core/libs/tests/GuestHeapRounding.cpp @@ -0,0 +1,172 @@ +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include "prx/libc/include/ApplicationHeap.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +void* APS5_VABI malloc_nid_postfix(std::size_t); +void APS5_VABI free_nid_postfix(void*); +void* APS5_VABI realloc_nid_postfix(void*, std::size_t); +} + +namespace { + +constexpr std::size_t HeaderBytes = 2 * sizeof(void*); +constexpr std::size_t MaximumSmallBlockBytes = 64u * 1024u; + +void registerDefaultHeap() { + std::array api{}; + ApplicationHeapRegister_nid_no_patch(api.data()); +} + +bool checkThreshold(std::size_t bytes, unsigned char value) { + auto* pointer = static_cast(malloc_nid_postfix(bytes)); + if (pointer == nullptr || reinterpret_cast(pointer) % 16 != 0) return false; + pointer[0] = value; + pointer[bytes - 1] = static_cast(value + 1); + const bool preserved = pointer[0] == value && pointer[bytes - 1] == static_cast(value + 1); + free_nid_postfix(pointer); + return preserved; +} + +DWORD runChild(const wchar_t* mode) { + std::array executable{}; + const DWORD length = GetModuleFileNameW(nullptr, executable.data(), static_cast(executable.size())); + if (length == 0 || length >= executable.size()) return 0xffffffffu; + std::wstring command = L"\"" + std::wstring(executable.data(), length) + L"\" " + mode; + std::vector commandLine(command.begin(), command.end()); + commandLine.push_back(L'\0'); + STARTUPINFOW startup{}; + startup.cb = sizeof(startup); + startup.dwFlags = STARTF_USESTDHANDLES; + startup.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + startup.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE); + startup.hStdError = GetStdHandle(STD_ERROR_HANDLE); + PROCESS_INFORMATION process{}; + if (!CreateProcessW(executable.data(), commandLine.data(), nullptr, nullptr, TRUE, CREATE_NO_WINDOW, + nullptr, nullptr, &startup, &process)) return 0xffffffffu; + const DWORD wait = WaitForSingleObject(process.hProcess, 30000); + if (wait != WAIT_OBJECT_0) { + const DWORD exitCode = wait == WAIT_TIMEOUT ? 0xfffffffeu : 0xffffffffu; + TerminateProcess(process.hProcess, exitCode); + WaitForSingleObject(process.hProcess, INFINITE); + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + return exitCode; + } + DWORD exitCode = 0xffffffffu; + if (!GetExitCodeProcess(process.hProcess, &exitCode)) exitCode = 0xffffffffu; + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + return exitCode; +} + +int runMallocOverflow() { + registerDefaultHeap(); + try { + auto* pointer = malloc_nid_postfix(std::numeric_limits::max() - HeaderBytes); + free_nid_postfix(pointer); + return 1; + } catch (const std::length_error& error) { + std::fprintf(stderr, "malloc overflow rejected: %s\n", error.what()); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "malloc failed with an unexpected exception: %s\n", error.what()); + return 2; + } catch (...) { + std::fputs("malloc failed with a non-standard exception\n", stderr); + return 3; + } +} + +int runReallocOverflow() { + registerDefaultHeap(); + constexpr std::size_t bytes = 16; + auto* original = static_cast(malloc_nid_postfix(bytes)); + std::memset(original, 0x5a, bytes); + bool rejectedWithExpectedError = false; + try { + auto* result = realloc_nid_postfix(original, std::numeric_limits::max() - HeaderBytes); + if (result != nullptr) { + free_nid_postfix(result); + return 4; + } + } catch (const std::length_error& error) { + rejectedWithExpectedError = true; + std::fprintf(stderr, "realloc overflow rejected: %s\n", error.what()); + } catch (const std::exception& error) { + std::fprintf(stderr, "realloc failed with an unexpected exception: %s\n", error.what()); + } catch (...) { + std::fputs("realloc failed with a non-standard exception\n", stderr); + } + bool originalPreserved = true; + for (std::size_t index = 0; index < bytes; ++index) { + if (original[index] != 0x5a) originalPreserved = false; + } + auto* replacement = static_cast(nullptr); + bool followUpAllocationThrew = false; + try { + replacement = static_cast(malloc_nid_postfix(bytes)); + } catch (const std::exception& error) { + followUpAllocationThrew = true; + std::fprintf(stderr, "follow-up allocation failed with an exception: %s\n", error.what()); + } catch (...) { + followUpAllocationThrew = true; + std::fputs("follow-up allocation failed with a non-standard exception\n", stderr); + } + unsigned result = 0; + if (!rejectedWithExpectedError) result |= 1; + if (!originalPreserved) result |= 2; + if (replacement == nullptr) result |= 4; + if (replacement == original) result |= 8; + if (followUpAllocationThrew) result |= 16; + if (result != 0) { + std::fprintf(stderr, "realloc checks failed: rejected=%d preserved=%d replacement=%d distinct=%d followUpThrew=%d\n", + rejectedWithExpectedError, originalPreserved, replacement != nullptr, replacement != original, followUpAllocationThrew); + } + if (result == 0) { + free_nid_postfix(replacement); + free_nid_postfix(original); + } + return static_cast(result); +} + +int runGuardedCases() { + bool success = true; + for (const auto& [mode, label] : std::array, 2>{ + std::pair{L"overflow-malloc", "overflow malloc"}, + std::pair{L"overflow-realloc", "overflow realloc"}}) { + const DWORD exitCode = runChild(mode); + if (exitCode != 0) { + std::fprintf(stderr, "%s child exited with 0x%08lx\n", label, static_cast(exitCode)); + success = false; + } + } + return success ? 0 : 1; +} + +} + +int main(int argc, char** argv) { + if (argc == 2) SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); + if (argc == 2 && std::strcmp(argv[1], "overflow-malloc") == 0) return runMallocOverflow(); + if (argc == 2 && std::strcmp(argv[1], "overflow-realloc") == 0) return runReallocOverflow(); + if (argc != 1) return 10; + registerDefaultHeap(); + if (!checkThreshold(MaximumSmallBlockBytes - HeaderBytes, 0x31) || + !checkThreshold(MaximumSmallBlockBytes - HeaderBytes + 1, 0x72)) return 11; + return runGuardedCases(); +} diff --git a/core/libs/tests/GuestHttp.cpp b/core/libs/tests/GuestHttp.cpp index 498b72e3c..2faab2957 100644 --- a/core/libs/tests/GuestHttp.cpp +++ b/core/libs/tests/GuestHttp.cpp @@ -15,6 +15,7 @@ int APS5_VABI sceHttpUriUnescape(char*, std::size_t*, std::size_t, const char*); int APS5_VABI sceHttpUriSweepPath(char*, const char*, std::size_t); int APS5_VABI sceHttpCreateEpoll(int, HttpEpollHandle*); int APS5_VABI sceHttpDestroyEpoll(int, HttpEpollHandle); +int APS5_VABI sceHttpWaitRequest(HttpEpollHandle, HttpNBEvent*, int, int); int APS5_VABI sceHttpReadData(int, void*, std::size_t); int APS5_VABI sceHttpCreateRequest2(int, const char*, const char*, std::uint64_t); int APS5_VABI sceHttpsEnableOption(int, std::uint32_t); @@ -23,6 +24,11 @@ int APS5_VABI sceHttpGetLastErrno(int, int*); int APS5_VABI sceHttpSetResponseHeaderMaxSize(int, std::uint64_t); int APS5_VABI sceHttpRedirectCacheFlush(int); int APS5_VABI sceHttpsUnloadCert(int); +int APS5_VABI sceHttpsSetSslVersion(int, int); +int APS5_VABI sceHttpsGetSslError(int, int*, std::uint32_t*); +int APS5_VABI sceHttpSetRedirectCallback(int, HttpRedirectCallback, void*); +int APS5_VABI sceHttpSetCookieRecvCallback(int, HttpCookieRecvCallback, void*); +int APS5_VABI sceHttpSetAuthInfoCallback(int, HttpAuthInfoCallback, void*); int APS5_VABI sceHttpParseStatusLine(const char*, std::size_t, std::int32_t*, std::int32_t*, std::int32_t*, const char**, std::size_t*); } @@ -30,6 +36,8 @@ static void Require(bool value) { if (!value) std::abort(); } static bool Equal(const char* left, const char* right) { return std::strcmp(left, right) == 0; } +static int AuthInfo(int, int, const char*, char*, char*, int, std::uint8_t**, std::uint64_t*, int*, void*) { std::abort(); } + int main() { constexpr int outOfMemory = static_cast(0x80431022); constexpr int invalidValue = static_cast(0x804311FE); @@ -181,6 +189,12 @@ int main() { HttpEpollHandle epoll = nullptr; Require(sceHttpCreateEpoll(1, nullptr) == invalidValue); Require(sceHttpCreateEpoll(1, &epoll) == 0 && epoll != nullptr); + HttpNBEvent events[2]{}; + Require(sceHttpWaitRequest(epoll, events, 2, 0) == 0); + Require(sceHttpWaitRequest(epoll, events, 2, 1000) == 0); + Require(sceHttpWaitRequest(epoll, nullptr, 2, 0) == invalidValue); + Require(sceHttpWaitRequest(epoll, events, 0, 0) == invalidValue); + Require(sceHttpWaitRequest(nullptr, events, 2, 0) == invalidValue); Require(sceHttpDestroyEpoll(1, epoll) == 0); char data[16]; @@ -196,10 +210,22 @@ int main() { Require(sceHttpsUnloadCert(1) == 0); Require(sceHttpSetResponseHeaderMaxSize(1, 8192) == 0); Require(sceHttpRedirectCacheFlush(1) == 0); + int authUserArg = 0; + Require(sceHttpSetAuthInfoCallback(1, AuthInfo, &authUserArg) == 0); + Require(sceHttpReadData(1, data, sizeof(data)) == network); int httpErrno = -1; Require(sceHttpGetLastErrno(1, &httpErrno) == 0); Require(httpErrno == 0); Require(sceHttpGetLastErrno(1, nullptr) == invalidValue); + Require(sceHttpsSetSslVersion(1, 0) == 0); + Require(sceHttpSetRedirectCallback(1, nullptr, nullptr) == 0); + Require(sceHttpSetCookieRecvCallback(1, nullptr, nullptr) == 0); + int sslError = -1; + std::uint32_t sslDetail = 0xFFFFFFFFu; + Require(sceHttpsGetSslError(1, &sslError, &sslDetail) == 0); + Require(sslError == 0 && sslDetail == 0); + Require(sceHttpsGetSslError(1, nullptr, &sslDetail) == invalidValue); + Require(sceHttpsGetSslError(1, &sslError, nullptr) == invalidValue); constexpr int parseInvalidResponse = static_cast(0x80432060); constexpr int parseInvalidValue = static_cast(0x804321FE); diff --git a/core/libs/tests/GuestHttp2.cpp b/core/libs/tests/GuestHttp2.cpp index 7be4800b5..dcc8347c3 100644 --- a/core/libs/tests/GuestHttp2.cpp +++ b/core/libs/tests/GuestHttp2.cpp @@ -1,6 +1,7 @@ #include "SceTypes.hpp" #include #include +#include extern "C" { int APS5_VABI sceHttp2Init(int, int, std::size_t, int); @@ -11,19 +12,55 @@ int APS5_VABI sceHttp2SetCookieBox(int, int); int APS5_VABI sceHttp2CookieFlush(int); int APS5_VABI sceHttp2SetRequestNoContentLength(int); int APS5_VABI sceHttp2SendRequest(int, const void*, std::size_t); +int APS5_VABI sceHttp2SendRequestAsync(int, const void*, std::size_t, Http2AsyncOption*, void*); +int APS5_VABI sceHttp2WaitAsync(int, Http2AsyncResult*, std::uint32_t*, void*); +int APS5_VABI sceHttp2DeleteRequest(int); int APS5_VABI sceHttp2Term(int); +int APS5_VABI sceKernelCreateEqueue(KernelEqueue* eq, const char* name); +int APS5_VABI sceKernelDeleteEqueue(KernelEqueue eq); +int APS5_VABI sceKernelWaitEqueue(KernelEqueue eq, KernelEvent* ev, int num, int* out, const KernelUseconds* timo); +int APS5_VABI sceKernelAddUserEventEdge(KernelEqueue eq, int id); +int APS5_VABI sceKernelDeleteUserEvent(KernelEqueue eq, int id); +uintptr_t APS5_VABI sceKernelGetEventId(const KernelEvent* ev); +void* APS5_VABI sceKernelGetEventUserData(const KernelEvent* ev); +} + +struct Http2MemoryPoolStats { + std::size_t pool_size; + std::size_t max_inuse_size; + std::size_t current_inuse_size; + std::int32_t reserved; +}; + +extern "C" { +int APS5_VABI sceHttp2GetMemoryPoolStats(int, Http2MemoryPoolStats*); +int APS5_VABI sceHttp2SetResolveRetry(int, std::int32_t); } static void Require(bool value) { if (!value) std::abort(); } +template +static bool Throws(TCall call) { + try { + call(); + } catch (const std::exception&) { + return true; + } + return false; +} + int main() { const int context = sceHttp2Init(1, 1, 0x10000, 4); Require(context > 0); + Http2MemoryPoolStats stats{1, 1, 1, 1}; + Require(sceHttp2GetMemoryPoolStats(context, &stats) == 0); + Require(stats.pool_size == 0x10000 && stats.max_inuse_size == 0 && stats.current_inuse_size == 0 && stats.reserved == 0); const int box = sceHttp2CreateCookieBox(context); const int other = sceHttp2CreateCookieBox(context); Require(box > 0 && other > 0 && box != other && box != context); const int tmpl = sceHttp2CreateTemplate(context, "agent", 2, 0); Require(tmpl > 0 && tmpl != box && tmpl != other); + Require(sceHttp2SetResolveRetry(tmpl, 3) == 0); Require(sceHttp2SetCookieBox(tmpl, box) == 0); Require(sceHttp2SetCookieBox(tmpl, 0) == 0); const int request = sceHttp2CreateRequestWithURL(tmpl, "POST", "https://example.com/", 16); @@ -32,5 +69,31 @@ int main() { Require(sceHttp2SetRequestNoContentLength(request) == 0); Require(sceHttp2CookieFlush(context) == 0); Require(sceHttp2SendRequest(request, nullptr, 0) == static_cast(0x80436063)); + + KernelEqueue eq = 0; + Require(sceKernelCreateEqueue(&eq, "http2") == 0); + Require(sceKernelAddUserEventEdge(eq, request) == 0); + int tag = 0; + Http2AsyncOption option{}; + option.equeue = eq; + option.user_event_id = request; + option.user_data = &tag; + Require(sceHttp2SendRequestAsync(request, nullptr, 0, &option, nullptr) == 0); + KernelEvent event{}; + int count = 0; + const KernelUseconds timeout = 1000000; + Require(sceKernelWaitEqueue(eq, &event, 1, &count, &timeout) == 0 && count == 1); + Require(sceKernelGetEventId(&event) == static_cast(request)); + Require(sceKernelGetEventUserData(&event) == &tag); + Http2AsyncResult result{}; + Require(sceHttp2WaitAsync(request, &result, nullptr, nullptr) == 0); + Require(result.req_id == request && result.result == static_cast(0x80436063)); + Require(Throws([&] { sceHttp2WaitAsync(request, &result, nullptr, nullptr); })); + Require(Throws([&] { sceHttp2SendRequestAsync(request, nullptr, 0, nullptr, nullptr); })); + Require(sceKernelDeleteUserEvent(eq, request) == 0); + Require(Throws([&] { sceHttp2SendRequestAsync(request, nullptr, 0, &option, nullptr); })); + Require(Throws([&] { sceHttp2WaitAsync(request, &result, nullptr, nullptr); })); + Require(sceHttp2DeleteRequest(request) == 0); + Require(sceKernelDeleteEqueue(eq) == 0); Require(sceHttp2Term(context) == 0); } diff --git a/core/libs/tests/GuestImageAnonymous.cpp b/core/libs/tests/GuestImageAnonymous.cpp new file mode 100644 index 000000000..f343bb149 --- /dev/null +++ b/core/libs/tests/GuestImageAnonymous.cpp @@ -0,0 +1,27 @@ +#include "prx/libc/include/GuestAllocations.hpp" +#include +#include +#include +#include +#include + +static void Require(bool value) { if (!value) std::abort(); } + +int initialized = 0x5a5a5a5a; + +int main() { + Require(GuestAllocations::GuestAllocationsGeneration_nid_postfix() != 0); + Require(initialized == 0x5a5a5a5a); + initialized = 1; + Require(initialized == 1); + const auto image = std::filesystem::read_symlink("/proc/self/exe").string(); + std::ifstream maps("/proc/self/maps"); + for (std::string line; std::getline(maps, line);) { + std::istringstream fields(line); + std::string span, permissions, offset, device, inode, path; + fields >> span >> permissions >> offset >> device >> inode >> std::ws; + std::getline(fields, path); + Require(permissions != "rw-p" || path != image); + } + return 0; +} diff --git a/core/libs/tests/GuestJson.cpp b/core/libs/tests/GuestJson.cpp index 7791c8404..8150bc249 100644 --- a/core/libs/tests/GuestJson.cpp +++ b/core/libs/tests/GuestJson.cpp @@ -20,6 +20,7 @@ struct Pair { Value value; }; using NullAccessCallback = const Value& (APS5_VABI*)(std::int32_t, const Value*, void*); +using SetNullAccessCallback = int (APS5_VABI*)(void*, NullAccessCallback, void*); extern "C" { int APS5_VABI _ZN3sce4Json11InitializerC1Ev(void*); @@ -27,6 +28,7 @@ int APS5_VABI _ZN3sce4Json11InitializerD1Ev(void*); int APS5_VABI _ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(void*, const void*); int APS5_VABI _ZN3sce4Json11Initializer9terminateEv(void*); int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(void*, NullAccessCallback, void*); +int APS5_VABI _ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(void*, NullAccessCallback, void*); void APS5_VABI _ZN3sce4Json6StringC1Ev(String*); void APS5_VABI _ZN3sce4Json6StringC1EPKc(String*, const char*); void APS5_VABI _ZN3sce4Json6StringD1Ev(String*); @@ -57,6 +59,7 @@ void APS5_VABI _ZN3sce4Json5ValueC1Eb(Value*, bool); void APS5_VABI _ZN3sce4Json5ValueC1Ed(Value*, double); void APS5_VABI _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(Value*, const Object*); void APS5_VABI _ZN3sce4Json5ValueD1Ev(Value*); +void APS5_VABI _ZN3sce4Json5Value5clearEv(Value*); int APS5_VABI _ZN3sce4Json5Value3setEl(Value*, std::int64_t); int APS5_VABI _ZN3sce4Json5Value3setEPKc(Value*, const char*); std::int32_t APS5_VABI _ZNK3sce4Json5Value7getTypeEv(const Value*); @@ -72,6 +75,7 @@ Value* APS5_VABI _ZN3sce4Json5Value10referValueERKNS0_6StringE(Value*, const Str const Value* APS5_VABI _ZNK3sce4Json5ValueixEm(const Value*, std::size_t); int APS5_VABI _ZN3sce4Json5Value9serializeERNS0_6StringE(Value*, String*); int APS5_VABI _ZN3sce4Json6Parser5parseERNS0_5ValueEPKcm(Value*, const char*, std::size_t); +bool APS5_VABI _ZNK3sce4Json6Object5emptyEv(const Object*); std::size_t APS5_VABI _ZNK3sce4Json6Object4sizeEv(const Object*); const Object* APS5_VABI _ZNK3sce4Json5Value9getObjectEv(const Value*); void APS5_VABI _ZN3sce4Json5ValueC1ENS0_9ValueTypeE(Value*, std::int32_t); @@ -219,7 +223,9 @@ static void ObjectsAndArrays() { String alpha{}, beta{}; _ZN3sce4Json6StringC1EPKc(&alpha, "alpha"); _ZN3sce4Json6StringC1EPKc(&beta, "beta"); + Require(_ZNK3sce4Json6Object5emptyEv(&object)); Value* first = _ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &alpha); + Require(!_ZNK3sce4Json6Object5emptyEv(&object)); Require(_ZNK3sce4Json5Value7getTypeEv(first) == TypeNull); Require(_ZN3sce4Json5Value3setEl(first, 1) == 0); Require(_ZN3sce4Json5Value3setEPKc(_ZN3sce4Json6ObjectixERKNS0_6StringE(&object, &beta), "b") == 0); @@ -265,13 +271,17 @@ static void ObjectsAndArrays() { _ZN3sce4Json5ArrayD1Ev(&array); } -static void NullAccess() { +static void NullAccess(SetNullAccessCallback setCallback) { + callbackCalls = 0; + lastRequested = -1; + lastParent = nullptr; + lastContext = nullptr; alignas(16) std::uint8_t initializer[16]{}; alignas(16) std::uint8_t parameter[64]{}; Require(_ZN3sce4Json11InitializerC1Ev(initializer) == 0); Require(_ZN3sce4Json11Initializer10initializeEPKNS0_13InitParameterE(initializer, parameter) == 0); int context = 0; - Require(_ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_(initializer, OnNullAccess, &context) == 0); + Require(setCallback(initializer, OnNullAccess, &context) == 0); _ZN3sce4Json5ValueC1Ev(&fallback); Require(_ZN3sce4Json5Value3setEl(&fallback, 99) == 0); @@ -351,10 +361,49 @@ static void ValueAccess() { Require(_ZN3sce4Json11InitializerD1Ev(initializer) == 0); } +static void ValueClear() { + Value value{}; + _ZN3sce4Json5ValueC1Ev(&value); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeNull); + _ZN3sce4Json5Value5clearEv(&value); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeNull); + + Require(_ZN3sce4Json5Value3setEl(&value, 12345) == 0); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeInteger); + _ZN3sce4Json5Value5clearEv(&value); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeNull); + + Require(_ZN3sce4Json5Value3setEPKc(&value, "test string") == 0); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeString); + _ZN3sce4Json5Value5clearEv(&value); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeNull); + + Object obj{}; + _ZN3sce4Json6ObjectC1Ev(&obj); + String key{}; + _ZN3sce4Json6StringC1EPKc(&key, "prop"); + Value* item = _ZN3sce4Json6ObjectixERKNS0_6StringE(&obj, &key); + _ZN3sce4Json5Value3setEl(item, 999); + _ZN3sce4Json5ValueC1ERKNS0_6ObjectE(&value, &obj); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeObject); + _ZN3sce4Json5Value5clearEv(&value); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeNull); + _ZN3sce4Json6StringD1Ev(&key); + _ZN3sce4Json6ObjectD1Ev(&obj); + + Require(_ZN3sce4Json5Value3setEl(&value, 777) == 0); + Require(_ZNK3sce4Json5Value7getTypeEv(&value) == TypeInteger); + Require(*_ZNK3sce4Json5Value10getIntegerEv(&value) == 777); + + _ZN3sce4Json5ValueD1Ev(&value); +} + int main() { ParseAndRoundTrip(); NestingDepth(); ObjectsAndArrays(); - NullAccess(); + NullAccess(_ZN3sce4Json11Initializer27setGlobalNullAccessCallBackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_); + NullAccess(_ZN3sce4Json11Initializer27setGlobalNullAccessCallbackEPFRKNS0_5ValueENS0_9ValueTypeEPS3_PvES7_); ValueAccess(); + ValueClear(); } diff --git a/core/libs/tests/GuestKernelSync.cpp b/core/libs/tests/GuestKernelSync.cpp new file mode 100644 index 000000000..bb227cb5e --- /dev/null +++ b/core/libs/tests/GuestKernelSync.cpp @@ -0,0 +1,73 @@ +#include "prx/libc/include/General.hpp" +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#define NOMINMAX +#define WIN32_LEAN_AND_MEAN +#include +#endif +extern "C" { +int APS5_VABI sceKernelOpen(const char*, int, unsigned short); +int APS5_VABI sceKernelClose(int); +std::int64_t APS5_VABI sceKernelWrite(int, const void*, std::size_t); +int APS5_VABI sceKernelMkdir(const char*, unsigned short); +int APS5_VABI sceKernelRename(const char*, const char*); +int APS5_VABI sceKernelUnlink(const char*); +int APS5_VABI sceKernelChmod_nid_postfix(const char*, std::uint16_t); +void APS5_VABI sceKernelSync(); +void* APS5_VABI fopen_nid_postfix(const char*, const char*); +int APS5_VABI fclose_nid_postfix(void*); +} +static void Check(bool value, int line) { + if (!value) { + std::fprintf(stderr, "Kernel sync check failed at line %d\n", line); + std::abort(); + } +} +#define Require(value) Check((value), __LINE__) +static bool Recorded(const char* guest) { + const auto paths = WrittenPaths_nid_no_patch(); + return std::find(paths.begin(), paths.end(), ResolvePath_nid_no_patch(guest).lexically_normal()) != paths.end(); +} +int main() { + std::filesystem::remove_all("kernel_sync_probe"); + Require(sceKernelMkdir("kernel_sync_probe", 0777) == 0); + Require(Recorded("kernel_sync_probe")); + std::ofstream("kernel_sync_probe/existing.bin") << "existing"; + const int reader = sceKernelOpen("kernel_sync_probe/existing.bin", 0x0, 0); + Require(reader >= 0 && sceKernelClose(reader) == 0); + void* stream = fopen_nid_postfix("kernel_sync_probe/existing.bin", "r"); + Require(stream != nullptr && fclose_nid_postfix(stream) == 0); + Require(!Recorded("kernel_sync_probe/existing.bin")); + stream = fopen_nid_postfix("kernel_sync_probe/existing.bin", "a"); + Require(stream != nullptr && fclose_nid_postfix(stream) == 0); + Require(Recorded("kernel_sync_probe/existing.bin")); + const int writer = sceKernelOpen("kernel_sync_probe/data.bin", 0x1 | 0x200 | 0x400, 0644); + Require(writer >= 0 && sceKernelWrite(writer, "data", 4) == 4 && sceKernelClose(writer) == 0); + Require(Recorded("kernel_sync_probe/data.bin")); + Require(sceKernelRename("kernel_sync_probe/data.bin", "kernel_sync_probe/moved.bin") == 0); + Require(Recorded("kernel_sync_probe/moved.bin")); + Require(sceKernelUnlink("kernel_sync_probe/moved.bin") == 0); + sceKernelSync(); + Require(!Recorded("kernel_sync_probe/data.bin") && !Recorded("kernel_sync_probe/moved.bin")); + sceKernelSync(); + const int locked = sceKernelOpen("kernel_sync_probe/read_only.bin", 0x1 | 0x200, 0644); + Require(locked >= 0 && sceKernelWrite(locked, "data", 4) == 4 && sceKernelClose(locked) == 0); + Require(sceKernelChmod_nid_postfix("kernel_sync_probe/read_only.bin", 0444) == 0); + Require(Recorded("kernel_sync_probe/read_only.bin")); + sceKernelSync(); + Require(sceKernelChmod_nid_postfix("kernel_sync_probe/read_only.bin", 0644) == 0); +#ifdef _WIN32 + const auto exclusive = CreateFileW(ResolvePath_nid_no_patch("kernel_sync_probe/read_only.bin").c_str(), GENERIC_READ, FILE_SHARE_READ, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + Require(exclusive != INVALID_HANDLE_VALUE); + sceKernelSync(); + CloseHandle(exclusive); +#endif + sceKernelSync(); + std::filesystem::remove_all("kernel_sync_probe"); +} diff --git a/core/libs/tests/GuestLifecycle.cpp b/core/libs/tests/GuestLifecycle.cpp index a78e20f06..11cb61e1e 100644 --- a/core/libs/tests/GuestLifecycle.cpp +++ b/core/libs/tests/GuestLifecycle.cpp @@ -19,7 +19,7 @@ struct InitEnvParams { extern "C" { int APS5_VABI cxa_atexit_nid_postfix(void (APS5_VABI *)(void*), void*, void*); void APS5_VABI cxa_finalize_nid_postfix(void*); -int APS5_VABI LibcInternalExtCxaThreadAtexit_nid_postfix(void (*)(void*), void*, void*); +int APS5_VABI LibcInternalExtCxaThreadAtexit_nid_postfix(void (APS5_VABI *)(void*), void*, void*); void APS5_VABI init_env_nid_postfix(const InitEnvParams*); } @@ -38,7 +38,12 @@ void APS5_VABI Second(void*) { secondAt = order++; } void APS5_VABI Other(void*) { otherAt = order++; } std::atomic threadCleanups{0}; -void ThreadDone(void*) { ++threadCleanups; } +std::atomic threadObjectSeen{nullptr}; +int threadObjectSentinel = 0; +void APS5_VABI ThreadDone(void* object) { + threadObjectSeen.store(object); + ++threadCleanups; +} } @@ -63,13 +68,14 @@ int main() { dlopen(nullptr, RTLD_NOW); #endif Require(image != nullptr); - Require(LibcInternalExtCxaThreadAtexit_nid_postfix(ThreadDone, nullptr, image) == 0); + Require(LibcInternalExtCxaThreadAtexit_nid_postfix(ThreadDone, &threadObjectSentinel, image) == 0); #ifndef _WIN32 dlclose(image); #endif }); worker.join(); Require(threadCleanups == 1); + Require(threadObjectSeen.load() == &threadObjectSentinel); try { init_env_nid_postfix(nullptr); diff --git a/core/libs/tests/GuestMath.cpp b/core/libs/tests/GuestMath.cpp index 259aeb3d6..ffdc06dff 100644 --- a/core/libs/tests/GuestMath.cpp +++ b/core/libs/tests/GuestMath.cpp @@ -1,4 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" +#include +#include #include #include #include @@ -22,6 +24,8 @@ float APS5_VABI fmodf_nid_postfix(float, float); float APS5_VABI asinf_nid_postfix(float); float APS5_VABI acosf_nid_postfix(float); float APS5_VABI atan2f_nid_postfix(float, float); +float APS5_VABI hypotf_nid_postfix(float, float); +double APS5_VABI hypot_nid_postfix(double, double); float APS5_VABI tanf_nid_postfix(float); float APS5_VABI log10f_nid_postfix(float); float APS5_VABI logbf_nid_postfix(float); @@ -177,6 +181,19 @@ int main() { Require(std::abs(acosf_nid_postfix(0.5f) - 1.0471976f) < 0.000001f); Require(std::abs(atan2f_nid_postfix(1.f, -1.f) - 2.3561945f) < 0.000001f); Require(tanf_nid_postfix(0.f) == 0.f); + Require(hypot_nid_postfix(3.0, 4.0) == 5.0 && hypot_nid_postfix(-3.0, -4.0) == 5.0 && hypotf_nid_postfix(3.f, -4.f) == 5.f); + Require(std::abs(hypot_nid_postfix(1e308, 1e308) / 1.4142135623730951e308 - 1.0) < 1e-15); + Require(std::abs(hypotf_nid_postfix(2e38f, 2e38f) / 2.8284271e38f - 1.f) < 1e-6f); + Require(std::isinf(hypot_nid_postfix(std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()))); + Require(std::isinf(hypotf_nid_postfix(std::numeric_limits::quiet_NaN(), -std::numeric_limits::infinity()))); + Require(std::isnan(hypot_nid_postfix(1.0, std::numeric_limits::quiet_NaN()))); + for (const bool signalingFirst : {true, false}) { + const float signaling = std::bit_cast(std::uint32_t{0x7f800001}); + const float infinity = std::numeric_limits::infinity(); + std::feclearexcept(FE_ALL_EXCEPT); + const float result = signalingFirst ? hypotf_nid_postfix(signaling, infinity) : hypotf_nid_postfix(infinity, signaling); + Require(std::isinf(result) && result > 0.f && std::fetestexcept(FE_INVALID) != 0); + } Require(log10f_nid_postfix(100.f) == 2.f); Require(logbf_nid_postfix(8.f) == 3.f && logbf_nid_postfix(-0.75f) == -1.f); Require(logbf_nid_postfix(std::numeric_limits::denorm_min()) == -149.f); diff --git a/core/libs/tests/GuestMbus.cpp b/core/libs/tests/GuestMbus.cpp new file mode 100644 index 000000000..bf57e1012 --- /dev/null +++ b/core/libs/tests/GuestMbus.cpp @@ -0,0 +1,44 @@ +#include "prx/libc/include/general/VabiMacros.hpp" + +#include +#include +#include +#include +#include + +extern "C" { +std::int32_t APS5_VABI sceDeviceServiceInitialize(std::int32_t mode, std::int32_t flags); +std::int32_t APS5_VABI sceDeviceServiceTerminate(); +std::int32_t APS5_VABI sceDeviceServiceGetEventState(std::int32_t deviceType); +std::int32_t APS5_VABI sceDeviceServiceQueryDeviceInfo_(std::int32_t deviceClass, std::int32_t arg1, std::int32_t arg2, void* infos, std::int32_t maxInfos, std::int32_t* count, std::int32_t* reserved, std::uint64_t infoSize); +} + +namespace { + +void Require(bool condition, const char* message) { + if (!condition) { + std::fprintf(stderr, "DeviceService: %s\n", message); + std::abort(); + } +} + +} + +int main() { + Require(sceDeviceServiceInitialize(3, 0) == 0, "initialization failed"); + Require(sceDeviceServiceGetEventState(1) == 0, "a device event was reported"); + std::array info{}; + std::int32_t count = 7; + std::int32_t reserved = 0; + Require(sceDeviceServiceQueryDeviceInfo_(0x7001, 0, 0, info.data(), 1, &count, &reserved, info.size()) == 0 && count == 0, "a device was listed"); + bool threw = false; + try { + sceDeviceServiceQueryDeviceInfo_(0x7001, 0, 0, info.data(), 1, nullptr, &reserved, info.size()); + } catch (const std::invalid_argument&) { + threw = true; + } + Require(threw, "a null count was accepted"); + Require(sceDeviceServiceTerminate() == 0, "termination failed"); + std::puts("DeviceService tests passed"); + return 0; +} diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index ff2cd1b95..892cefb40 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -36,6 +36,7 @@ extern "C" { void* APS5_VABI mmap_nid_postfix(void*, std::size_t, int, int, int, std::int64_t) noexcept; int APS5_VABI munmap_nid_postfix(void*, std::size_t) noexcept; +int APS5_VABI mprotect_nid_postfix(void*, std::size_t, int) noexcept; int* APS5_VABI __error_nid_postfix(); int APS5_VABI sceKernelMapNamedFlexibleMemory(void**, std::size_t, int, int, const char*); int APS5_VABI sceKernelMapNamedFlexibleMemoryInternal(void**, std::size_t, int, int, const char*); @@ -154,9 +155,15 @@ static void CheckInternalNamedFlexibleMapping() { Require(sceKernelAvailableFlexibleMemorySize(&available) == 0 && available == before - length); Require(sceKernelMunmap(mapped, length) == 0); Require(sceKernelAvailableFlexibleMemorySize(&available) == 0 && available == before); + void* flagged = nullptr; + Require(sceKernelMapNamedFlexibleMemoryInternal(&flagged, length, 3, 0x8000, "internal flag") == 0 && flagged != nullptr); + Require(std::strcmp(NameAt(flagged), "internal flag") == 0); + static_cast(flagged)[length - 1] = 1; + Require(sceKernelAvailableFlexibleMemorySize(&available) == 0 && available == before - length); + Require(sceKernelMunmap(flagged, length) == 0); bool rejected = false; void* unknown = nullptr; - try { sceKernelMapNamedFlexibleMemoryInternal(&unknown, length, 3, 0x8000, "internal mapping"); } catch (const std::exception&) { rejected = true; } + try { sceKernelMapNamedFlexibleMemoryInternal(&unknown, length, 3, 0x20000, "internal mapping"); } catch (const std::exception&) { rejected = true; } Require(rejected && unknown == nullptr); Require(sceKernelAvailableFlexibleMemorySize(&available) == 0 && available == before); } @@ -280,6 +287,27 @@ static void CheckReleaseDirectMemoryClearsMappings() { Require(sceKernelMunmap(mapped, page * 2) == 0); } +static void CheckFixedMappingReplacesPartialOverlap() { + constexpr std::size_t page = 0x4000; + std::int64_t phys = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * 3, 0, 0, &phys) == 0); + void* reserved = nullptr; + Require(sceKernelMapDirectMemory(&reserved, page * 3, 3, 0, phys, 0) == 0); + Require(sceKernelMunmap(reserved, page * 3) == 0); + void* head = reserved; + Require(sceKernelMapDirectMemory(&head, page, 3, 0x10, phys, 0) == 0 && head == reserved); + void* fixed = reserved; + Require(sceKernelMapDirectMemory(&fixed, page * 3, 3, 0x10, phys, 0) == 0); + Require(fixed == reserved); + static_cast(fixed)[page * 2 + 7] = 0x5c; + VirtualQueryInfo info{}; + Require(sceKernelVirtualQuery(fixed, 0, &info, sizeof(info)) == 0); + Require(info.is_direct && info.offset == static_cast(phys)); + Require(static_cast(fixed)[page * 2 + 7] == 0x5c); + Require(sceKernelMunmap(fixed, page * 3) == 0); + Require(sceKernelReleaseDirectMemory(phys, page * 3) == 0); +} + static void CheckGetDirectMemoryType() { constexpr std::size_t page = 0x4000; std::int64_t first = 0; @@ -1043,6 +1071,7 @@ int main() { CheckAudioCoprocessorProtection(); CheckDirectMemoryFollowsPhysicalPages(); CheckReleaseDirectMemoryClearsMappings(); + CheckFixedMappingReplacesPartialOverlap(); CheckDirectMemoryGpuProtBits(); CheckFixedVirtualReservation(); CheckReservedRangeIsNotCommitted(); @@ -1093,6 +1122,20 @@ int main() { const auto range = mutation.Find(memory); Require(range.bytes == page * 3 && range.readable && range.writable); } + Require(mprotect_nid_postfix(memory, 0, 1) == 0); + Require(mprotect_nid_postfix(nullptr, page, 1) == -1 && *__error_nid_postfix() == 22); + Require(mprotect_nid_postfix(memory + 1, 1, 1) == 0); + { + GuestAllocations::Mutation mutation; + const auto range = mutation.Find(memory); + Require(range.readable && !range.writable); + } + Require(memory[0] == 42); + Require(mprotect_nid_postfix(memory, page, 3) == 0); + { + GuestAllocations::Mutation mutation; + Require(mutation.Find(memory).writable); + } Require(munmap_nid_postfix(memory + 1, page) == -1 && *__error_nid_postfix() == 22); Require(munmap_nid_postfix(memory, 0) == -1 && *__error_nid_postfix() == 22); Require(memory[0] == 42); @@ -1102,6 +1145,11 @@ int main() { Require(memory[page * 2] == 73); Require(munmap_nid_postfix(memory + page * 2, page) == 0); Require(munmap_nid_postfix(memory, page) == -1); + *__error_nid_postfix() = 0; + Require(mprotect_nid_postfix(memory, page, 1) == -1 && *__error_nid_postfix() == 22); + *__error_nid_postfix() = 0; + Require(mprotect_nid_postfix(memory, std::numeric_limits::max(), 1) == -1 && + *__error_nid_postfix() == 22); for (int protection : {0, 1, 3, 5}) { void* mapped = mmap_nid_postfix(memory, 1, protection, 0x1002, -1, 0); Require(mapped != failed); diff --git a/core/libs/tests/GuestNpReachability.cpp b/core/libs/tests/GuestNpReachability.cpp new file mode 100644 index 000000000..541aa0d3e --- /dev/null +++ b/core/libs/tests/GuestNpReachability.cpp @@ -0,0 +1,32 @@ +#include "SceTypes.hpp" +#include + +extern "C" { +int APS5_VABI sceNpRegisterNpReachabilityStateCallback(void*, void*); +int APS5_VABI sceNpUnregisterNpReachabilityStateCallback(); +int APS5_VABI sceNpGetNpReachabilityState(int, uint32_t*); +int APS5_VABI sceNpCheckCallback(); +} + +static constexpr int ErrorArgument = static_cast(0x80550003); +static constexpr int ErrorAlreadyRegistered = static_cast(0x80550008); +static constexpr int ErrorNotRegistered = static_cast(0x80550009); +static void Require(bool value) { if (!value) std::abort(); } +static void APS5_VABI Callback(int, uint32_t, void*) { std::abort(); } + +int main() { + int userdata = 0; + auto* callback = reinterpret_cast(&Callback); + Require(sceNpUnregisterNpReachabilityStateCallback() == ErrorNotRegistered); + Require(sceNpRegisterNpReachabilityStateCallback(nullptr, &userdata) == ErrorArgument); + Require(sceNpRegisterNpReachabilityStateCallback(callback, &userdata) == 0); + Require(sceNpRegisterNpReachabilityStateCallback(callback, nullptr) == ErrorAlreadyRegistered); + Require(sceNpRegisterNpReachabilityStateCallback(nullptr, nullptr) == ErrorArgument); + uint32_t state = 99; + Require(sceNpGetNpReachabilityState(0, &state) == 0 && state == 0); + Require(sceNpCheckCallback() == 0); + Require(sceNpUnregisterNpReachabilityStateCallback() == 0); + Require(sceNpUnregisterNpReachabilityStateCallback() == ErrorNotRegistered); + Require(sceNpRegisterNpReachabilityStateCallback(callback, nullptr) == 0); + Require(sceNpUnregisterNpReachabilityStateCallback() == 0); +} diff --git a/core/libs/tests/GuestOfflineImports.cpp b/core/libs/tests/GuestOfflineImports.cpp new file mode 100644 index 000000000..99d3178f0 --- /dev/null +++ b/core/libs/tests/GuestOfflineImports.cpp @@ -0,0 +1,89 @@ +#include "SceTypes.hpp" + +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceHttpSetCookieEnabled(int, int); +int APS5_VABI sceHttpSendRequest(int, const void*, std::size_t); +int APS5_VABI sceNpEntitlementAccessGetEntitlementKey( + std::uint32_t, const NpUnifiedEntitlementLabel*, NpEntitlementAccessEntitlementKey*); +int APS5_VABI sceNpEntitlementAccessRequestUnifiedEntitlementInfoList(); +int APS5_VABI sceNpEntitlementAccessPollUnifiedEntitlementInfoList(); +int APS5_VABI sceNpEntitlementAccessRequestServiceEntitlementInfoList(); +int APS5_VABI sceNpEntitlementAccessPollServiceEntitlementInfoList(); +int APS5_VABI sceRudpInit_nid_postfix(void*, int); +int APS5_VABI sceRudpGetStatus(void*, std::size_t); +int APS5_VABI sceRudpTerminate(); +} + +static void Require(bool value) { if (!value) std::abort(); } + +int main() { + constexpr int invalidValue = static_cast(0x804311FE); + constexpr int network = static_cast(0x80431063); + Require(sceHttpSetCookieEnabled(1, 0) == 0); + Require(sceHttpSendRequest(1, nullptr, 0) == network); + bool cookieUnsupported = false; + try { + sceHttpSetCookieEnabled(1, 1); + } catch (const std::runtime_error& error) { + cookieUnsupported = std::string_view(error.what()) == "sceHttpSetCookieEnabled not implemented"; + } + Require(cookieUnsupported); + for (int enabled : {-1, 2, 0x100}) { + Require(sceHttpSetCookieEnabled(1, enabled) == invalidValue); + } + + constexpr int parameter = static_cast(0x817D0002); + constexpr int noEntitlement = static_cast(0x817D0007); + static_assert(sizeof(NpEntitlementAccessEntitlementKey) == 16); + NpUnifiedEntitlementLabel label{}; + std::memcpy(label.data, "unowned-addon", 14); + struct KeyBuffer { + std::uint64_t before; + NpEntitlementAccessEntitlementKey key; + std::uint64_t after; + } output; + std::memset(&output, 0xa5, sizeof(output)); + std::array original{}; + std::memcpy(original.data(), &output, sizeof(output)); + Require(sceNpEntitlementAccessGetEntitlementKey(0, nullptr, &output.key) == parameter); + Require(sceNpEntitlementAccessGetEntitlementKey(0, &label, nullptr) == parameter); + Require(sceNpEntitlementAccessGetEntitlementKey(0, nullptr, nullptr) == parameter); + for (std::uint32_t serviceLabel : {0u, 1u, 0xffffffffu}) { + Require(sceNpEntitlementAccessGetEntitlementKey(serviceLabel, &label, &output.key) == noEntitlement); + Require(std::memcmp(&output, original.data(), sizeof(output)) == 0); + } + + constexpr int signedOut = static_cast(0x80550006); + Require(sceNpEntitlementAccessRequestUnifiedEntitlementInfoList() == signedOut); + Require(sceNpEntitlementAccessPollUnifiedEntitlementInfoList() == signedOut); + Require(sceNpEntitlementAccessRequestServiceEntitlementInfoList() == signedOut); + Require(sceNpEntitlementAccessPollServiceEntitlementInfoList() == signedOut); + + std::array status; + status.fill(0x5a); + const auto originalStatus = status; + auto unsupported = [](void* data, std::size_t size) { + try { + sceRudpGetStatus(data, size); + } catch (const std::runtime_error& error) { + return std::string_view(error.what()) == "sceRudpGetStatus not implemented"; + } + return false; + }; + Require(unsupported(status.data(), status.size())); + Require(status == originalStatus); + Require(sceRudpInit_nid_postfix(nullptr, 0) == 0); + Require(unsupported(status.data(), status.size())); + Require(status == originalStatus); + Require(unsupported(nullptr, 0)); + Require(sceRudpTerminate() == 0); + return 0; +} diff --git a/core/libs/tests/GuestPad.cpp b/core/libs/tests/GuestPad.cpp index 866c44ba7..e8696ac01 100644 --- a/core/libs/tests/GuestPad.cpp +++ b/core/libs/tests/GuestPad.cpp @@ -8,15 +8,18 @@ extern "C" { int APS5_VABI scePadOpen_nid_postfix(int, int, int, const void*); +int APS5_VABI scePadOpenExt(int, int, int, const void*); int APS5_VABI scePadClose_nid_postfix(int); int APS5_VABI scePadGetHandle(int, int, int); int APS5_VABI scePadSetVibrationTriggerEffectWeakWhileEmbeddedMicInUse(bool); +int APS5_VABI scePadSetVibrationMode(int, int); int APS5_VABI scePadInit_nid_postfix(void); int APS5_VABI scePadRead_nid_postfix(int, PadData*, int); int APS5_VABI scePadReadState(int, PadData*); int APS5_VABI scePadSetTiltCorrectionState(int, bool); int APS5_VABI scePadResetOrientation(int); int APS5_VABI scePadSetAngularVelocityDeadbandState(int, bool); +int APS5_VABI scePadIsRemoteController(int, bool*); } static void Require(bool value) { if (!value) std::abort(); } @@ -67,6 +70,15 @@ static void CheckReadStateHandle(int handle) { Require(scePadReadState(handle, &data) == PAD_OK); } +static void CheckRemoteController(int handle) { + bool remote = true; + Require(scePadIsRemoteController(handle + 1, &remote) == PAD_ERROR_INVALID_HANDLE); + Require(scePadIsRemoteController(handle, nullptr) == PAD_ERROR_INVALID_ARG); + Require(remote); + Require(scePadIsRemoteController(handle, &remote) == PAD_OK); + Require(!remote); +} + int main() { constexpr int noHandle = static_cast(0x80920008); constexpr int user = 0x10000000; @@ -75,6 +87,11 @@ int main() { Require(scePadOpen_nid_postfix(user, 1, 0, nullptr) == PAD_ERROR_INVALID_ARG); Require(scePadOpen_nid_postfix(user, 0, 1, nullptr) == PAD_ERROR_INVALID_ARG); Require(scePadGetHandle(user, 0, 0) == noHandle); + const unsigned char wheel[16]{0xb7, 0x0e, 0x04, 0x0e, 0x08, 0x6e, 0x01}; + Require(scePadOpenExt(user, PAD_PORT_TYPE_SPECIAL, 0, wheel) == static_cast(0x80920007)); + Require(scePadOpenExt(user, PAD_PORT_TYPE_SPECIAL, 1, wheel) == PAD_ERROR_INVALID_ARG); + Require(scePadOpenExt(user, PAD_PORT_TYPE_SPECIAL, 0, nullptr) == PAD_ERROR_INVALID_ARG); + Require(scePadGetHandle(user, 0, 0) == noHandle); const int handle = scePadOpen_nid_postfix(user, 0, 0, nullptr); Require(handle > 0); Require(scePadGetHandle(user, 0, 0) == handle); @@ -82,6 +99,11 @@ int main() { CheckTiltCorrection(handle); CheckTouchContact(); CheckReadStateHandle(handle); + CheckRemoteController(handle); + Require(scePadSetVibrationMode(handle, 1) == 0); + Require(scePadSetVibrationMode(handle, 2) == 0); + Require(scePadSetVibrationMode(handle, 3) == PAD_ERROR_INVALID_ARG); + Require(scePadSetVibrationMode(handle, -1) == PAD_ERROR_INVALID_ARG); Require(scePadGetHandle(0xff, 16, 0) == handle); Require(scePadGetHandle(user, 16, 0) == noHandle); Require(scePadGetHandle(user, 0, 1) == noHandle); diff --git a/core/libs/tests/GuestPathCase.cpp b/core/libs/tests/GuestPathCase.cpp new file mode 100644 index 000000000..18b87e9c6 --- /dev/null +++ b/core/libs/tests/GuestPathCase.cpp @@ -0,0 +1,92 @@ +#include "prx/libc/include/General.hpp" +#include "prx/libc/include/FileStream.hpp" +#include "prx/libkernel/File/include/FileFlags.hpp" +#include "SceTypes.hpp" + +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceKernelOpen(const char*, int, std::uint16_t); +int APS5_VABI sceKernelClose(int); +std::int64_t APS5_VABI sceKernelRead(int, void*, std::size_t); +std::int64_t APS5_VABI sceKernelWrite(int, const void*, std::size_t); +int APS5_VABI sceKernelStat(const char*, FileStat*); +int APS5_VABI chdir_nid_postfix(const char*); +int APS5_VABI access_nid_postfix(const char*, int); +int APS5_VABI rename_nid_postfix(const char*, const char*); +int APS5_VABI remove_nid_postfix(const char*); +FileStream* APS5_VABI fopen_nid_postfix(const char*, const char*); +std::size_t APS5_VABI fread_nid_postfix(void*, std::size_t, std::size_t, FileStream*); +int APS5_VABI fclose_nid_postfix(FileStream*); +} + +static void Require(bool value) { if (!value) std::abort(); } + +static void CheckRead(const std::string& path, char expected) { + const int fd = sceKernelOpen(path.c_str(), SCE_KERNEL_O_RDONLY, 0); + Require(fd >= 0); + char value = 0; + Require(sceKernelRead(fd, &value, 1) == 1 && value == expected); + Require(sceKernelClose(fd) == 0); +} + +int main() { + const auto host = std::filesystem::canonical(std::filesystem::current_path()); + const auto name = "AnyPS5-Case-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()); + const auto directory = host / name; + Require(std::filesystem::create_directories(directory / "Data" / "Nested")); + { std::ofstream file(directory / "Data" / "Settings.ini"); file << 'x'; } + auto lower = name; + for (auto& c : lower) if (c >= 'A' && c <= 'Z') c += 'a' - 'A'; + CheckRead("/" + lower + "/dAtA/SETTINGS.INI", 'x'); + FileStat stat{}; + Require(sceKernelStat(("/" + lower + "/DATA/settings.INI").c_str(), &stat) == 0); + Require(chdir_nid_postfix(lower.c_str()) == 0); + Require(access_nid_postfix("DATA/SETTINGS.ini", 4) == 0); + auto* stream = fopen_nid_postfix("data\\SETTINGS.INI", "rb"); + char value = 0; + Require(stream && fread_nid_postfix(&value, 1, 1, stream) == 1 && value == 'x'); + Require(fclose_nid_postfix(stream) == 0); + Require(chdir_nid_postfix("DATA/../data") == 0); + CheckRead("settings.INI", 'x'); + Require(chdir_nid_postfix("..") == 0); + const int created = sceKernelOpen("data/nested/New.DAT", SCE_KERNEL_O_CREAT | SCE_KERNEL_O_EXCL | SCE_KERNEL_O_WRONLY, 0600); + Require(created >= 0 && sceKernelWrite(created, "y", 1) == 1 && sceKernelClose(created) == 0); + Require(std::filesystem::exists(directory / "Data" / "Nested" / "New.DAT")); + CheckRead("DATA/NESTED/new.dat", 'y'); + Require(sceKernelOpen("data/nested/NEW.dat", SCE_KERNEL_O_CREAT | SCE_KERNEL_O_EXCL | SCE_KERNEL_O_WRONLY, 0600) == static_cast(0x80020011u)); + Require(rename_nid_postfix("DATA/NESTED/new.dat", "data/nested/Moved.DAT") == 0); + Require(remove_nid_postfix("DATA/NESTED/moved.dat") == 0); + Require(sceKernelOpen("data/nested/missing", SCE_KERNEL_O_RDONLY, 0) == static_cast(0x80020002u)); + AddPathAlias_nid_no_patch("case-mount", (directory / "Data").string().c_str()); + CheckRead("/CASE-MOUNT/settings.INI", 'x'); + Require(access_nid_postfix("/case-mount-other/Settings.ini", 0) == -1); + BlockPathAlias_nid_no_patch("CASE-MOUNT"); + Require(access_nid_postfix("/case-mount/settings.ini", 0) == -1); + AddPathAlias_nid_no_patch("Case-Mount", (directory / "Data").string().c_str()); + CheckRead("/case-MOUNT/settings.INI", 'x'); + RemovePathAlias_nid_no_patch("CASE-mount"); + Require(access_nid_postfix("/case-mount/settings.ini", 0) == -1); +#ifndef _WIN32 + std::filesystem::create_directory_symlink("Data", directory / "Linked"); + CheckRead("linked/SETTINGS.INI", 'x'); + { std::ofstream file(directory / "Data" / "SETTINGS.INI"); file << 'z'; } + CheckRead("Data/Settings.ini", 'x'); + CheckRead("Data/SETTINGS.INI", 'z'); + bool ambiguous = false; + try { CheckRead("data/settings.ini", 'x'); } catch (const std::runtime_error&) { ambiguous = true; } + Require(ambiguous); + std::filesystem::remove(directory / "Data" / "SETTINGS.INI"); + std::filesystem::remove(directory / "Data" / "Settings.ini"); + { std::ofstream file(directory / "Data" / "SETTINGS.ini"); file << 'n'; } + CheckRead("data/settings.ini", 'n'); + std::filesystem::create_symlink("absent", directory / "Dangling"); + Require(sceKernelOpen("dangling", SCE_KERNEL_O_RDONLY, 0) == static_cast(0x80020002u)); +#endif + Require(chdir_nid_postfix("/") == 0); + Require(std::filesystem::current_path() == host); + std::filesystem::remove_all(directory); +} diff --git a/core/libs/tests/GuestPoll.cpp b/core/libs/tests/GuestPoll.cpp new file mode 100644 index 000000000..a32abd9a6 --- /dev/null +++ b/core/libs/tests/GuestPoll.cpp @@ -0,0 +1,66 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +struct PollDescriptor { + int descriptor; + short events; + short revents; +}; +extern "C" { +int APS5_VABI socket_nid_postfix(int, int, int); +int APS5_VABI bind_nid_postfix(int, const void*, std::uint32_t); +int APS5_VABI listen_nid_postfix(int, int); +int APS5_VABI accept_nid_postfix(int, void*, std::uint32_t*); +int APS5_VABI connect_nid_postfix(int, const void*, std::uint32_t); +int APS5_VABI getsockname_nid_postfix(int, void*, std::uint32_t*); +std::int64_t APS5_VABI send_nid_postfix(int, const void*, std::uint64_t, int); +std::int64_t APS5_VABI recv_nid_postfix(int, void*, std::uint64_t, int); +int APS5_VABI close_nid_postfix(int); +int APS5_VABI poll_nid_postfix(PollDescriptor*, std::uint32_t, int); +int* APS5_VABI __error_nid_postfix(); +} +static void Require(bool value) { if (!value) std::abort(); } +constexpr short In = 0x1; +constexpr short Out = 0x4; +constexpr short Hup = 0x10; +constexpr short Invalid = 0x20; +int main() { + const int listener = socket_nid_postfix(2, 1, 0); + Require(listener >= 0); + std::array address{16, 2, 0, 0, 127, 0, 0, 1}; + Require(bind_nid_postfix(listener, address.data(), address.size()) == 0); + std::uint32_t size = address.size(); + Require(getsockname_nid_postfix(listener, address.data(), &size) == 0); + Require(listen_nid_postfix(listener, 4) == 0); + PollDescriptor single{listener, In, -1}; + Require(poll_nid_postfix(&single, 1, 0) == 0 && single.revents == 0); + const auto start = std::chrono::steady_clock::now(); + Require(poll_nid_postfix(&single, 1, 50) == 0); + Require(std::chrono::steady_clock::now() - start >= std::chrono::milliseconds(40)); + const int client = socket_nid_postfix(2, 1, 0); + Require(client >= 0 && connect_nid_postfix(client, address.data(), address.size()) == 0); + Require(poll_nid_postfix(&single, 1, 1000) == 1 && single.revents == In); + const int server = accept_nid_postfix(listener, nullptr, nullptr); + Require(server >= 0); + PollDescriptor pair[3]{{server, In, 0}, {-1, In, 7}, {client, Out, 0}}; + Require(poll_nid_postfix(pair, 3, 1000) == 1); + Require(pair[0].revents == 0 && pair[1].revents == 0 && pair[2].revents == Out); + const char message[] = "poll"; + Require(send_nid_postfix(client, message, sizeof(message), 0) == sizeof(message)); + pair[2].events = In; + Require(poll_nid_postfix(pair, 3, 1000) == 1 && pair[0].revents == In && pair[2].revents == 0); + char buffer[8]{}; + Require(recv_nid_postfix(server, buffer, sizeof(buffer), 0) == sizeof(message)); + Require(close_nid_postfix(client) == 0); + Require(poll_nid_postfix(pair, 1, 1000) == 1 && (pair[0].revents & (In | Hup)) != 0); + Require(recv_nid_postfix(server, buffer, sizeof(buffer), 0) == 0); + PollDescriptor closed[2]{{client, In, 0}, {listener, In, 0}}; + Require(poll_nid_postfix(closed, 2, -1) == 1 && closed[0].revents == Invalid && closed[1].revents == 0); + Require(poll_nid_postfix(nullptr, 0, 10) == 0); + Require(poll_nid_postfix(nullptr, 1, 0) == -1 && *__error_nid_postfix() == 14); + Require(poll_nid_postfix(&single, 1, -2) == -1 && *__error_nid_postfix() == 22); + Require(close_nid_postfix(server) == 0); + Require(close_nid_postfix(listener) == 0); +} diff --git a/core/libs/tests/GuestPthreadAttr.cpp b/core/libs/tests/GuestPthreadAttr.cpp index 74368d79e..8588e2ae1 100644 --- a/core/libs/tests/GuestPthreadAttr.cpp +++ b/core/libs/tests/GuestPthreadAttr.cpp @@ -19,6 +19,7 @@ int APS5_VABI scePthreadAttrGetschedparam(const PthreadAttr* attr, KernelSchedPa int APS5_VABI scePthreadAttrGetaffinity(const PthreadAttr* attr, KernelCpumask* mask); int APS5_VABI scePthreadAttrGetstacksize(const PthreadAttr* attr, std::size_t* stackSize); int APS5_VABI scePthreadAttrGetdetachstate(const PthreadAttr* attr, int* state); +int APS5_VABI pthread_attr_setstacksize_nid_postfix(PthreadAttr* attr, std::size_t stackSize); } static constexpr int SCE_OK = 0; @@ -30,6 +31,8 @@ static constexpr int UPDATED_PRIORITY = 767; static constexpr KernelCpumask CREATION_AFFINITY = 0x3; static constexpr KernelCpumask UPDATED_AFFINITY = 0x1000; static constexpr std::size_t STACK_SIZE = 2u << 20; +static constexpr std::size_t MIN_STACK_SIZE = 16384; +static constexpr int GUEST_EINVAL = 22; static void Require(bool value) { if (!value) std::abort(); } @@ -40,6 +43,11 @@ struct ReportedAttributes { int detachState = -1; }; +extern "C" { +int APS5_VABI pthread_attr_setstack_nid_postfix(PthreadAttr* attr, void* addr, std::size_t size); +int APS5_VABI pthread_attr_getstack_nid_postfix(const PthreadAttr* attr, void** addr, std::size_t* size); +} + static ReportedAttributes Query(Pthread thread) { PthreadAttr attr = nullptr; Require(scePthreadAttrInit(&attr) == SCE_OK); @@ -55,12 +63,26 @@ static ReportedAttributes Query(Pthread thread) { return reported; } +static void CheckStackSizeLimit() { + PthreadAttr attr = nullptr; + Require(scePthreadAttrInit(&attr) == SCE_OK); + std::size_t stackSize = 0; + Require(scePthreadAttrSetstacksize(&attr, MIN_STACK_SIZE) == SCE_OK); + Require(scePthreadAttrGetstacksize(&attr, &stackSize) == SCE_OK && stackSize == MIN_STACK_SIZE); + Require(scePthreadAttrSetstacksize(&attr, MIN_STACK_SIZE - 1) == SCE_KERNEL_ERROR_EINVAL); + Require(scePthreadAttrSetstacksize(&attr, 0) == SCE_KERNEL_ERROR_EINVAL); + Require(pthread_attr_setstacksize_nid_postfix(&attr, MIN_STACK_SIZE - 1) == GUEST_EINVAL); + Require(scePthreadAttrGetstacksize(&attr, &stackSize) == SCE_OK && stackSize == MIN_STACK_SIZE); + Require(scePthreadAttrDestroy(&attr) == SCE_OK); +} + static void* APS5_VABI Worker(void* arg) { static_cast*>(arg)->get(); return nullptr; } int main() { + CheckStackSizeLimit(); PthreadAttr attr = nullptr; Require(scePthreadAttrInit(&attr) == SCE_OK); Require(scePthreadAttrSetinheritsched(&attr, EXPLICIT_SCHED) == SCE_OK); @@ -98,4 +120,19 @@ int main() { release.set_value(); Require(scePthreadJoin(thread, nullptr) == SCE_OK); + + PthreadAttr stackAttr = nullptr; + char stackMarker; + Require(pthread_attr_setstack_nid_postfix(nullptr, &stackMarker, STACK_SIZE) == 22); + Require(pthread_attr_setstack_nid_postfix(&stackAttr, &stackMarker, STACK_SIZE) == 22); + Require(scePthreadAttrInit(&stackAttr) == SCE_OK); + Require(pthread_attr_setstack_nid_postfix(&stackAttr, &stackMarker, STACK_SIZE) == 0); + Require(pthread_attr_setstack_nid_postfix(&stackAttr, nullptr, STACK_SIZE) == 22); + Require(pthread_attr_setstack_nid_postfix(&stackAttr, &stackMarker, 16383) == 22); + void* reportedAddress = nullptr; + std::size_t reportedSize = 0; + Require(pthread_attr_getstack_nid_postfix(&stackAttr, &reportedAddress, &reportedSize) == 0); + Require(reportedAddress == &stackMarker && reportedSize == STACK_SIZE); + Require(scePthreadAttrDestroy(&stackAttr) == SCE_OK); + Require(pthread_attr_setstack_nid_postfix(&stackAttr, &stackMarker, STACK_SIZE) == 22); } diff --git a/core/libs/tests/GuestRaiseException.cpp b/core/libs/tests/GuestRaiseException.cpp index ceed4647a..59d2bc455 100644 --- a/core/libs/tests/GuestRaiseException.cpp +++ b/core/libs/tests/GuestRaiseException.cpp @@ -20,10 +20,12 @@ int APS5_VABI sceKernelCreateSema(KernelSema* sem, const char* name, uint32_t at int APS5_VABI sceKernelDeleteSema(KernelSema sem); int APS5_VABI sceKernelSignalSema(KernelSema sem, int count); int APS5_VABI sceKernelWaitSema(KernelSema sem, int need, KernelUseconds* time); +int APS5_VABI sceKernelSyncOnAddressWait(std::uint32_t* address, std::uint32_t expected, const KernelUseconds* timeout, const char* name); } static constexpr int SCE_KERNEL_ERROR_EINVAL = static_cast(0x80020016); static constexpr int SCE_KERNEL_ERROR_ESRCH = static_cast(0x80020003); +static constexpr int SCE_KERNEL_ERROR_ETIMEDOUT = static_cast(0x8002003c); static constexpr int SIGUSR1 = 30; static constexpr int Repeats = 100; @@ -34,6 +36,7 @@ static std::atomic handlerThread; static std::atomic handlerRsp{0}; static std::atomic handlerFrame{0}; static std::atomic handlerSelf{nullptr}; +static std::atomic clobberVectors{false}; static void APS5_VABI Handler(int signum, void* context) { Require(signum == SIGUSR1); @@ -44,6 +47,7 @@ static void APS5_VABI Handler(int signum, void* context) { handlerRsp.store(rsp); handlerThread.store(std::this_thread::get_id()); handlerSelf.store(scePthreadSelf()); + if (clobberVectors.load()) asm volatile("vzeroall" ::: "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); calls.fetch_add(1); } @@ -156,6 +160,60 @@ static void* APS5_VABI Looping(void* arg) { return nullptr; } +static constexpr int VectorRounds = 100; +static constexpr std::size_t VectorBytes = 16 * 32; +alignas(32) static std::uint8_t vectorPattern[VectorBytes]; +alignas(32) static std::uint8_t vectorResult[VectorBytes]; +static volatile std::uint8_t vectorStop = 0; + +static void* APS5_VABI Vectors(void* arg) { + auto& worker = *static_cast(arg); + worker.id = std::this_thread::get_id(); + worker.started.store(true); + asm volatile( + "vmovdqu 0(%[pattern]), %%ymm0\n" + "vmovdqu 32(%[pattern]), %%ymm1\n" + "vmovdqu 64(%[pattern]), %%ymm2\n" + "vmovdqu 96(%[pattern]), %%ymm3\n" + "vmovdqu 128(%[pattern]), %%ymm4\n" + "vmovdqu 160(%[pattern]), %%ymm5\n" + "vmovdqu 192(%[pattern]), %%ymm6\n" + "vmovdqu 224(%[pattern]), %%ymm7\n" + "vmovdqu 256(%[pattern]), %%ymm8\n" + "vmovdqu 288(%[pattern]), %%ymm9\n" + "vmovdqu 320(%[pattern]), %%ymm10\n" + "vmovdqu 352(%[pattern]), %%ymm11\n" + "vmovdqu 384(%[pattern]), %%ymm12\n" + "vmovdqu 416(%[pattern]), %%ymm13\n" + "vmovdqu 448(%[pattern]), %%ymm14\n" + "vmovdqu 480(%[pattern]), %%ymm15\n" + "1:\n" + "pause\n" + "cmpb $0, (%[stop])\n" + "je 1b\n" + "vmovdqu %%ymm0, 0(%[result])\n" + "vmovdqu %%ymm1, 32(%[result])\n" + "vmovdqu %%ymm2, 64(%[result])\n" + "vmovdqu %%ymm3, 96(%[result])\n" + "vmovdqu %%ymm4, 128(%[result])\n" + "vmovdqu %%ymm5, 160(%[result])\n" + "vmovdqu %%ymm6, 192(%[result])\n" + "vmovdqu %%ymm7, 224(%[result])\n" + "vmovdqu %%ymm8, 256(%[result])\n" + "vmovdqu %%ymm9, 288(%[result])\n" + "vmovdqu %%ymm10, 320(%[result])\n" + "vmovdqu %%ymm11, 352(%[result])\n" + "vmovdqu %%ymm12, 384(%[result])\n" + "vmovdqu %%ymm13, 416(%[result])\n" + "vmovdqu %%ymm14, 448(%[result])\n" + "vmovdqu %%ymm15, 480(%[result])\n" + "vzeroupper\n" + : + : [pattern] "r"(vectorPattern), [result] "r"(vectorResult), [stop] "r"(&vectorStop) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); + return nullptr; +} + static std::atomic finishedReturned{false}; static void* APS5_VABI Finished(void*) { @@ -163,6 +221,94 @@ static void* APS5_VABI Finished(void*) { return nullptr; } +static std::uint32_t nestedWord = 0; +static KernelUseconds nestedTimeout = 0; +static std::atomic nestedEntered{false}; +static std::atomic nestedResult{0}; +static std::atomic nestedCalls{0}; + +static void APS5_VABI NestedHandler(int signum, void*) { + Require(signum == SIGUSR1); + nestedEntered.store(true); + KernelUseconds timeout = nestedTimeout; + nestedResult.store(sceKernelSyncOnAddressWait(&nestedWord, 0, &timeout, "nested")); + nestedCalls.fetch_add(1); +} + +struct SemaWaiter { + KernelSema sem = nullptr; + std::atomic started{false}; + std::atomic returned{false}; + int result = -1; +}; + +static void* APS5_VABI WaitSemaOnce(void* arg) { + auto& waiter = *static_cast(arg); + waiter.started.store(true); + waiter.result = sceKernelWaitSema(waiter.sem, 1, nullptr); + waiter.returned.store(true); + return nullptr; +} + +static void StartSemaWaiter(Pthread* thread, SemaWaiter& waiter, KernelSema sem, const char* name) { + waiter.sem = sem; + Require(scePthreadCreate(thread, nullptr, WaitSemaOnce, &waiter, name) == 0); + while (!waiter.started.load()) std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); +} + +static void ExpectReturned(SemaWaiter& waiter) { + for (int attempt = 0; attempt < 5000 && !waiter.returned.load(); ++attempt) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + Require(waiter.returned.load()); + Require(waiter.result == 0); +} + +static void ExpectNestedDone(int before) { + for (int attempt = 0; attempt < 5000 && nestedCalls.load() == before; ++attempt) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + Require(nestedCalls.load() == before + 1); + Require(nestedResult.load() == SCE_KERNEL_ERROR_ETIMEDOUT); +} + +static void NestedWaitKeepsLaterWaiters() { + KernelSema sem = nullptr; + Require(sceKernelCreateSema(&sem, "nested later", 0, 0, 2, nullptr) == 0); + SemaWaiter first; + SemaWaiter second; + Pthread firstThread = nullptr; + Pthread secondThread = nullptr; + StartSemaWaiter(&firstThread, first, sem, "nested first"); + StartSemaWaiter(&secondThread, second, sem, "nested second"); + nestedTimeout = 20000; + const int before = nestedCalls.load(); + Require(sceKernelRaiseException(firstThread, SIGUSR1) == 0); + ExpectNestedDone(before); + Require(sceKernelSignalSema(sem, 2) == 0); + ExpectReturned(first); + ExpectReturned(second); + Require(scePthreadJoin(firstThread, nullptr) == 0); + Require(scePthreadJoin(secondThread, nullptr) == 0); + Require(sceKernelDeleteSema(sem) == 0); +} + +static void NestedWaitKeepsOuterWake() { + KernelSema sem = nullptr; + Require(sceKernelCreateSema(&sem, "nested outer", 0, 0, 1, nullptr) == 0); + SemaWaiter waiter; + Pthread thread = nullptr; + StartSemaWaiter(&thread, waiter, sem, "nested outer"); + nestedTimeout = 1000000; + nestedEntered.store(false); + const int before = nestedCalls.load(); + Require(sceKernelRaiseException(thread, SIGUSR1) == 0); + while (!nestedEntered.load()) std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + Require(sceKernelSignalSema(sem, 1) == 0); + ExpectNestedDone(before); + ExpectReturned(waiter); + Require(scePthreadJoin(thread, nullptr) == 0); + Require(sceKernelDeleteSema(sem) == 0); +} + static void ExpectDelivery(int before, std::thread::id thread) { for (int attempt = 0; attempt < 5000 && calls.load() == before; ++attempt) std::this_thread::sleep_for(std::chrono::milliseconds(1)); Require(calls.load() == before + 1); @@ -298,6 +444,24 @@ int main() { CloseHandle(loopingHandle); Require(scePthreadJoin(loopingThread, nullptr) == 0); + if (__builtin_cpu_supports("avx")) { + for (std::size_t i = 0; i < VectorBytes; ++i) vectorPattern[i] = static_cast(i * 7 + 1); + clobberVectors.store(true); + Worker vectors; + Pthread vectorsThread = nullptr; + Require(scePthreadCreate(&vectorsThread, nullptr, Vectors, &vectors, "vectors") == 0); + while (!vectors.started.load()) std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + for (int raised = 0; raised < VectorRounds; ++raised) { + Require(sceKernelRaiseException(vectorsThread, SIGUSR1) == 0); + ExpectDelivery(1 + 2 * Repeats + HostRounds + LeavingRounds + ContinuingRounds + SelfingRounds + raised, vectors.id); + } + clobberVectors.store(false); + vectorStop = 1; + Require(scePthreadJoin(vectorsThread, nullptr) == 0); + Require(std::memcmp(vectorPattern, vectorResult, VectorBytes) == 0); + } + Pthread finishedThread = nullptr; Require(scePthreadCreate(&finishedThread, nullptr, Finished, nullptr, "finished") == 0); while (!finishedReturned.load()) std::this_thread::yield(); @@ -306,4 +470,9 @@ int main() { Require(scePthreadJoin(finishedThread, nullptr) == 0); Require(sceKernelRemoveExceptionHandler(SIGUSR1) == 0); + + Require(sceKernelInstallExceptionHandler(SIGUSR1, reinterpret_cast(&NestedHandler)) == 0); + NestedWaitKeepsLaterWaiters(); + NestedWaitKeepsOuterWake(); + Require(sceKernelRemoveExceptionHandler(SIGUSR1) == 0); } diff --git a/core/libs/tests/GuestRtc.cpp b/core/libs/tests/GuestRtc.cpp index 576dacbb8..bb64f3ece 100644 --- a/core/libs/tests/GuestRtc.cpp +++ b/core/libs/tests/GuestRtc.cpp @@ -120,9 +120,9 @@ int main() { const RtcDateTime dosBadMonth{2024, 13, 1, 0, 0, 0, 0}; Require(sceRtcGetDosTime(&dosBadMonth, &dosTime) == invalidMonth); const RtcDateTime dosEarly{1979, 12, 31, 0, 0, 0, 0}; - bool dosEarlyThrew = false; - try { sceRtcGetDosTime(&dosEarly, &dosTime); } catch (const std::exception&) { dosEarlyThrew = true; } - Require(dosEarlyThrew); + Require(sceRtcGetDosTime(&dosEarly, &dosTime) == invalidYear && dosTime == 0); + const RtcDateTime dosLate{2108, 1, 1, 0, 0, 0, 0}; + Require(sceRtcGetDosTime(&dosLate, &dosTime) == invalidYear && dosTime == 0xff9fbf7du); converted = RtcDateTime{1, 1, 1, 1, 1, 1, 1}; Require(sceRtcSetDosTime(&converted, 0x585d645cu) == 0 && Equal(converted, RtcDateTime{2024, 2, 29, 12, 34, 56, 0})); Require(sceRtcSetDosTime(&converted, 0x7f9fbf7du) == 0 && Equal(converted, RtcDateTime{2043, 12, 31, 23, 59, 58, 0})); @@ -157,22 +157,39 @@ int main() { Require(sceRtcParseRFC3339(&tick, "1970-01-01T00:00:00Z") == 0 && tick.tick == unixEpochTick); Require(sceRtcParseRFC3339(&tick, "2023-02-29T00:00:00Z") == invalidDay); Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:56") == badParse); - Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:56Zjunk") == badParse); - const char* invalidOffsets[] = { - "2024-02-29T12:34:56.789+00:99", - "2024-02-29T12:34:56.789-00:99", - "2024-02-29T12:34:56.789+00:60", - "2024-02-29T12:34:56.789-00:60", - "2024-02-29T12:34:56.789+24:00", - "2024-02-29T12:34:56.789-24:00", - "2024-02-29T12:34:56.789+99:59", - "2024-02-29T12:34:56.789-99:59", + const std::uint64_t secondTick = leapDayTick - 789000ull; + Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:56Zjunk") == 0 && tick.tick == secondTick); + Require(sceRtcParseRFC3339(&tick, "2024-02-29T14:34:56+02:00 trailing") == 0 && tick.tick == secondTick); + Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:56.Z") == 0 && tick.tick == secondTick); + Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:56.1234567Z") == 0 && tick.tick == secondTick + 123456ull); + const struct { + const char* text; + std::int64_t minutes; + } unrangedOffsets[] = { + {"2024-02-29T12:34:56.789+00:99", 99}, + {"2024-02-29T12:34:56.789-00:99", -99}, + {"2024-02-29T12:34:56.789+00:60", 60}, + {"2024-02-29T12:34:56.789-00:60", -60}, + {"2024-02-29T12:34:56.789+24:00", 1440}, + {"2024-02-29T12:34:56.789-24:00", -1440}, + {"2024-02-29T12:34:56.789+99:59", 5999}, + {"2024-02-29T12:34:56.789-99:99", -6039}, }; - for (const char* text : invalidOffsets) { + for (const auto& offset : unrangedOffsets) { + Require(sceRtcParseRFC3339(&tick, offset.text) == 0); + Require(tick.tick == leapDayTick - static_cast(offset.minutes * 60000000)); + } + for (const char* text : {"2024-02-29 12:34:56Z", "2024-02-29T12:34:56+0100", "2024-02-29T12:34:56+01", "2024-02-29T12:34:56 Z", " 2024-02-29T12:34:56Z", "2024-02-29T12:34:56..Z", "2024-2-29T12:34:56Z"}) { tick.tick = 123; Require(sceRtcParseRFC3339(&tick, text) == badParse); Require(tick.tick == 123); } + tick.tick = 123; + Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:60Z") == 0 && tick.tick == 123); + Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:60+01:00") == 0 && tick.tick == 123ull - 3600000000ull); + Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:61Z") == static_cast(0x80B5000D)); + Require(sceRtcParseRFC3339(&tick, "0000-13-40T99:99:99Z") == static_cast(0x80B50008)); + Require(sceRtcParseRFC3339(&tick, "0001-01-01T00:00:00+01:00") == 0 && tick.tick == 0xFFFFFFFF296C5C00ull); Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:56.789+00:00") == 0 && tick.tick == leapDayTick); Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:56.789-00:00") == 0 && tick.tick == leapDayTick); Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:56.789+00:59") == 0 && tick.tick == leapDayTick - 3540000000ull); @@ -181,45 +198,105 @@ int main() { Require(sceRtcParseRFC3339(&tick, "2024-02-29T12:34:56.789-23:59") == 0 && tick.tick == leapDayTick + 86340000000ull); Require(sceRtcParseRFC3339(nullptr, "1970-01-01T00:00:00Z") == invalidPointer); - Require(sceRtcParseDateTime(&tick, "2024-02-29T12:34:56.789") == 0 && tick.tick == leapDayTick); - Require(sceRtcParseDateTime(&tick, "2024-02-29 12:34:56") == 0 && tick.tick == leapDayTick - 789000ull); + for (const char* text : {"2024-02-29T12:34:56.789", "2024-02-29 12:34:56", "2024-02-29 12:34:56Z", "2024-02-29", "2024/02/29T12:34:56Z", "2024-02-29T12:34:56."}) { + tick.tick = 123; + Require(sceRtcParseDateTime(&tick, text) == badParse); + Require(tick.tick == 123); + } + Require(sceRtcParseDateTime(&tick, " \t 2024-02-29T12:34:56.789Z") == 0 && tick.tick == leapDayTick); + Require(sceRtcParseDateTime(&tick, "2024-02-29T12:34:56.789Zjunk") == 0 && tick.tick == leapDayTick); + Require(sceRtcParseDateTime(&tick, "2024-02-29T12:34:56.789+99:99") == 0 && tick.tick == leapDayTick - 6039ull * 60000000ull); + tick.tick = 123; + Require(sceRtcParseDateTime(&tick, "2024-02-29T12:34:60-01:00") == 0 && tick.tick == 123ull + 3600000000ull); + Require(sceRtcParseDateTime(&tick, "2024-13-01T00:00:00Z") == static_cast(0x80B50009)); Require(sceRtcParseDateTime(&tick, "2024-02-29T14:04:56.789+01:30") == 0 && tick.tick == leapDayTick); Require(sceRtcParseDateTime(&tick, "2024-02-29t12:34:56.789z") == 0 && tick.tick == leapDayTick); Require(sceRtcParseDateTime(&tick, "1970-01-01T00:00:00Z") == 0 && tick.tick == unixEpochTick); - Require(sceRtcParseDateTime(&tick, "2023-02-29T00:00:00") == invalidDay); + Require(sceRtcParseDateTime(&tick, "2023-02-29T00:00:00Z") == invalidDay); Require(sceRtcParseDateTime(nullptr, "1970-01-01T00:00:00Z") == invalidPointer); Require(sceRtcParseDateTime(&tick, "Thu, 29 Feb 2024 12:34:56") == 0 && tick.tick == leapDayTick - 789000ull); Require(sceRtcParseDateTime(&tick, "Thu, 29 Feb 2024 12:34:56 GMT") == 0 && tick.tick == leapDayTick - 789000ull); Require(sceRtcParseDateTime(&tick, "Thu, 29 Feb 2024 14:04:56 +0130") == 0 && tick.tick == leapDayTick - 789000ull); Require(sceRtcParseDateTime(&tick, "Thu, 29 Feb 2024 11:04:56 -0130") == 0 && tick.tick == leapDayTick - 789000ull); Require(sceRtcParseDateTime(&tick, "Thu, 01 Jan 1970 00:00:00 +0000") == 0 && tick.tick == unixEpochTick); - Require(sceRtcParseDateTime(&tick, "Thu, 30 Feb 2024 00:00:00") == invalidDay); Require(sceRtcParseDateTime(&tick, "Thu Feb 29 12:34:56 2024") == 0 && tick.tick == leapDayTick - 789000ull); Require(sceRtcParseDateTime(&tick, "Thu Jan 1 00:00:00 1970") == 0 && tick.tick == unixEpochTick); - Require(sceRtcParseDateTime(&tick, "Thu Feb 30 00:00:00 2024") == invalidDay); - const char* unparseable[] = { - "2024-02-29", - "2024/02/29T12:34:56", - "2024-02-29T12:34:56Zjunk", - "2024-02-29T12:34:56+99:99", - "2024-02-29T12:34:56.", - "Thu, 29 Feb 2024 12:34:56 +01:30", - "Thu, 29 Feb 2024 12:34:56 PST", - "Thu, 29 Xyz 2024 12:34:56", - "Xyz, 29 Feb 2024 12:34:56", - "Thu, 29 Feb 2024 12:34:56 +2400", - "Thu Feb 29 12:34:56 2024\n", - "Thu Feb 29 12:34:56 24", - "Thu Feb 29 12:34:56 2024", + const auto at = [](int year, int month, int day, int hour, int minute, int second) { + const RtcDateTime time{static_cast(year), static_cast(month), static_cast(day), + static_cast(hour), static_cast(minute), static_cast(second), 0}; + RtcTick result{}; + Require(sceRtcGetTick(&time, &result) == 0); + return result.tick; }; - for (const char* text : unparseable) { - bool thrown = false; - try { - sceRtcParseDateTime(&tick, text); - } catch (const std::exception&) { - thrown = true; - } - Require(thrown); + constexpr std::uint64_t untouched = 123; + const std::uint64_t march = at(2024, 3, 1, 12, 34, 56); + const struct { + const char* text; + int result; + std::uint64_t local; + std::int64_t minutes; + } dateTimes[] = { + {"thursday,29-feb-24 12:34:56", 0, leapDayTick - 789000ull, 0}, + {"Mon 29 February 2024 12:34:56 -0130", 0, leapDayTick - 789000ull, -90}, + {" \tFri, 01 Mar 2024 12:34:56 GMT", 0, march, 0}, + {"Thu, 1-Mar 2024 12:34:56 -9999", 0, march, -6039}, + {"Thu, 01 Mar 2024 12:34:56 +01:30", 0, march, 0}, + {"Thu, 01 Mar 2024 12:34:56 +0130x", 0, march, 90}, + {"Thu, 01 Mar 2024 12:34:56 PST8PDT", 0, march, -480}, + {"Thu, 01 Mar 2024 12:34:56 nzdt", 0, march, 780}, + {"Thu, 01 Mar 2024 12:34:56 KST", 0, march, 540}, + {"Thu, 01 Mar 2024 12:34:56 HST", 0, march, 420}, + {"Thu, 01 Mar 2024 12:34:56 Jt", 0, march, 450}, + {"Thu, 01 Mar 2024 12:34:56 ut", 0, march, -420}, + {"Thu, 01 Mar 2024 12:34:56 xT", 0, march, 0}, + {"Thu, 01 Mar 2024 12:34:56 Ux", 0, march, 0}, + {"Thu, 01 Mar 2024 12:34:56 B", 0, march, 60}, + {"Thu, 01 Mar 2024 12:34:56 m", 0, march, 720}, + {"Thu, 01 Mar 2024 12:34:56 N", 0, march, 0}, + {"Thu, 01 Mar 2024 12:34:56 y", 0, march, -660}, + {"Thu, 01 Mar 2024 12:34:56 Z", 0, march, 0}, + {"Thu, 01 Mar 2024 12:34:56 J", badParse, untouched, 0}, + {"Thu, 01 Mar 2024 12:34:56 +0100", badParse, untouched, 0}, + {"Thu, 01 Mar 2024 12:34:56 (UTC)", badParse, untouched, 0}, + {"Thu, 01 Mar 2024 12:34:56\tEST", 0, march, 0}, + {"Thu, 01 Mar 2024 12:34:56 ", 0, march, 0}, + {"Thu, 01 Mar 2024 12:34:567 EST", 0, march, 0}, + {"Thu, 01 Mar 2024 12:34 EST", 0, at(2024, 3, 1, 12, 34, 0), 0}, + {"Thu, 01 Mar 2024 12:34EST", badParse, untouched, 0}, + {"Thu, 01 Mar 2024 1:2:3 EST", 0, at(2024, 3, 1, 1, 2, 3), -300}, + {"Thu, 01 Mar 2024 26:00:00 GMT", badParse, untouched, 0}, + {"Thu, 01 Mar 2024 25:00:00 +0100", 0, untouched, 60}, + {"Thu, 29 Feb 2023 12:34:56 EST", 0, untouched, -300}, + {"Thu, 01 Mar 49 12:34:56 GMT", 0, at(2049, 3, 1, 12, 34, 56), 0}, + {"Thu, 01 Mar 50 12:34:56 GMT", 0, at(1950, 3, 1, 12, 34, 56), 0}, + {"Mon, 01 Jan 0001 00:00:00 +0100", 0, 0, 60}, + {"Thu, 01 Mar 024 12:34:56 GMT", badParse, untouched, 0}, + {"Thu, 01 Mar 10000 12:34:56 GMT", badParse, untouched, 0}, + {"Thu, 001 Mar 2024 12:34:56 GMT", badParse, untouched, 0}, + {"Thu, 01 Mar 2024 12:34:56 GMT", badParse, untouched, 0}, + {"Thu, 01 Mar 2024\t12:34:56 GMT", badParse, untouched, 0}, + {"Thu, 01 Sept 2024 12:34:56 GMT", badParse, untouched, 0}, + {"Thurs, 01 Mar 2024 12:34:56 GMT", badParse, untouched, 0}, + {"Thu , 01 Mar 2024 12:34:56 GMT", badParse, untouched, 0}, + {", 01 Mar 2024 12:34:56 GMT", badParse, untouched, 0}, + {"\nThu, 01 Mar 2024 12:34:56 GMT", badParse, untouched, 0}, + {"THURSDAY february 9 1:2:3 2024 +0100", 0, at(2024, 2, 9, 1, 2, 3), 0}, + {"ThuMar 1 12:34:56 10000", 0, at(1000, 3, 1, 12, 34, 56), 0}, + {"Thu,\tFeb 29 12:34:56 2024", 0, leapDayTick - 789000ull, 0}, + {"Thu Feb 29 99:99:99 2024", 0, untouched, 0}, + {"Thu Feb 29 12:34:56 2024", badParse, untouched, 0}, + {"Thu Feb 29 12:34 2024", badParse, untouched, 0}, + {"Thu Feb 29 12:34:56 24", badParse, untouched, 0}, + {"Thu Feb 29 12:34:56\t2024", badParse, untouched, 0}, + {"Feb 29 12:34:56 2024", badParse, untouched, 0}, + {"\n2024-02-29T12:34:56Z", badParse, untouched, 0}, + {"+2024-02-29T12:34:56Z", badParse, untouched, 0}, + {"", badParse, untouched, 0}, + }; + for (const auto& dateTime : dateTimes) { + tick.tick = untouched; + Require(sceRtcParseDateTime(&tick, dateTime.text) == dateTime.result); + Require(tick.tick == dateTime.local - static_cast(dateTime.minutes * 60000000)); } RtcTick source{leapDayTick}; diff --git a/core/libs/tests/GuestSaveDataDialog.cpp b/core/libs/tests/GuestSaveDataDialog.cpp new file mode 100644 index 000000000..5d8e06371 --- /dev/null +++ b/core/libs/tests/GuestSaveDataDialog.cpp @@ -0,0 +1,78 @@ +#include "prx/libSceSaveDataDialog.native/SaveDataDialog.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" + +#include +#include +#include + +extern "C" { +int APS5_VABI sceSaveDataDialogInitialize(); +int APS5_VABI sceSaveDataDialogOpen(const void* param); +int APS5_VABI sceSaveDataDialogGetStatus(); +int APS5_VABI sceSaveDataDialogUpdateStatus(); +int APS5_VABI sceSaveDataDialogIsReadyToDisplay(); +int APS5_VABI sceSaveDataDialogProgressBarInc(int target, std::uint32_t delta); +int APS5_VABI sceSaveDataDialogProgressBarSetValue(int target, std::uint32_t rate); +int APS5_VABI sceSaveDataDialogClose(const void* param); +int APS5_VABI sceSaveDataDialogGetResult(void* result); +int APS5_VABI sceSaveDataDialogTerminate(); +} + +static void Require(bool value, const char* message) { + if (!value) { + std::fprintf(stderr, "%s\n", message); + std::abort(); + } +} + +static void RequireRunning() { + Require(sceSaveDataDialogGetStatus() == SAVE_DATA_DIALOG_STATUS_RUNNING, + "save progress must remain running until closed"); + Require(sceSaveDataDialogUpdateStatus() == SAVE_DATA_DIALOG_STATUS_RUNNING, + "polling save progress must not finish the dialog"); + Require(sceSaveDataDialogIsReadyToDisplay() == 1, "save progress must be ready"); +} + +int main() { + Require(sceSaveDataDialogInitialize() == 0, "initialize"); + + SaveDataDirName names[2]{{}, {"SDU1"}}; + SaveDataDialogItems items{}; + items.dir_names = names; + items.dir_names_num = 2; + int context = 42; + SaveDataDialogParam param{}; + param.size = sizeof(param); + param.mode = SAVE_DATA_DIALOG_MODE_PROGRESS_BAR; + param.items = &items; + param.user_data = &context; + + for (int save = 0; save < 2; ++save) { + Require(sceSaveDataDialogOpen(¶m) == 0, "open save progress"); + for (int poll = 0; poll < 8; ++poll) RequireRunning(); + Require(sceSaveDataDialogOpen(¶m) == SAVE_DATA_DIALOG_ERROR_INVALID_STATE, + "opening another dialog must not interrupt an active save"); + Require(sceSaveDataDialogProgressBarSetValue(0, 25) == 0, "set progress"); + Require(sceSaveDataDialogProgressBarInc(0, 25) == 0, "increment progress"); + RequireRunning(); + Require(sceSaveDataDialogProgressBarSetValue(0, 100) == 0, "complete progress"); + RequireRunning(); + Require(sceSaveDataDialogClose(nullptr) == 0, "close save progress"); + Require(sceSaveDataDialogUpdateStatus() == SAVE_DATA_DIALOG_STATUS_FINISHED, + "closing save progress must finish it"); + SaveDataDirName selected{}; + SaveDataDialogResult result{}; + result.dir_name = &selected; + Require(sceSaveDataDialogGetResult(&result) == 0, "read completed result"); + Require(result.mode == param.mode && result.result == SAVE_DATA_DIALOG_RESULT_OK && + result.user_data == &context && std::strcmp(selected.data, "SDU1") == 0, + "save result must preserve its mode, directory and caller context"); + } + + param.mode = 1; + Require(sceSaveDataDialogOpen(¶m) == 0, "open selection dialog"); + Require(sceSaveDataDialogUpdateStatus() == SAVE_DATA_DIALOG_STATUS_FINISHED, + "selection dialog must retain the existing automatic response"); + Require(sceSaveDataDialogTerminate() == 0, "terminate"); + Require(sceSaveDataDialogGetStatus() == SAVE_DATA_DIALOG_STATUS_NONE, "terminated state"); +} diff --git a/core/libs/tests/GuestSaveDataMountStatus.cpp b/core/libs/tests/GuestSaveDataMountStatus.cpp new file mode 100644 index 000000000..c51a1cb5c --- /dev/null +++ b/core/libs/tests/GuestSaveDataMountStatus.cpp @@ -0,0 +1,62 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "SceTypes.hpp" +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceSaveDataInitialize3(const void*); +int APS5_VABI sceSaveDataMount3(const SaveDataMount3*, SaveDataMountResult*); +int APS5_VABI sceSaveDataUmount2(std::uint32_t, const SaveDataMountPoint*); +} + +namespace { + +constexpr std::uint32_t MountReadWrite = 2; +constexpr std::uint32_t MountCreate = 4; +constexpr std::uint32_t MountCreate2 = 32; + +int failures = 0; + +void Check(bool condition, const std::string& what) { + if (condition) return; + std::fprintf(stderr, "savedata mount status check failed: %s\n", what.c_str()); + ++failures; +} + +void MountAndCheck(const char* name, std::uint32_t mode, std::uint32_t expectedStatus, const std::string& what) { + SceSaveDataDirName dirName{}; + std::memcpy(dirName.data, name, std::strlen(name) + 1); + SaveDataMount3 mount{}; + mount.dir_name = &dirName; + mount.mount_mode = mode; + SaveDataMountResult result{}; + const int code = sceSaveDataMount3(&mount, &result); + Check(code == 0, what + ": mounts"); + if (code != 0) return; + Check(result.mount_status == expectedStatus, what + ": status is " + std::to_string(result.mount_status) + ", expected " + std::to_string(expectedStatus)); + Check(sceSaveDataUmount2(0, &result.mount_point) == 0, what + ": unmounts"); +} + +} + +int main() { + const auto root = std::filesystem::temp_directory_path() / ("anyps5-savedata-mount-status-" + std::to_string(std::random_device{}())); + std::filesystem::create_directories(root); + const auto previous = std::filesystem::current_path(); + std::filesystem::current_path(root); + Check(sceSaveDataInitialize3(nullptr) == 0, "SaveData initializes"); + MountAndCheck("fresh", MountReadWrite | MountCreate2, 1u, "CREATE2 of a missing save"); + MountAndCheck("fresh", MountReadWrite | MountCreate2, 0u, "CREATE2 of an existing save"); + MountAndCheck("made", MountReadWrite | MountCreate, 1u, "CREATE of a missing save"); + MountAndCheck("made", MountReadWrite, 0u, "open of an existing save"); + std::filesystem::current_path(previous); + std::error_code error; + std::filesystem::remove_all(root, error); + if (failures != 0) return 1; + std::printf("savedata mount status tests passed\n"); + return 0; +} diff --git a/core/libs/tests/GuestSaveDataSearch.cpp b/core/libs/tests/GuestSaveDataSearch.cpp new file mode 100644 index 000000000..cc24049af --- /dev/null +++ b/core/libs/tests/GuestSaveDataSearch.cpp @@ -0,0 +1,74 @@ +#include "SceTypes.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" + +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceSaveDataDirNameSearch(const SaveDataDirNameSearchCond*, SaveDataDirNameSearchResult*); +int APS5_VABI sceSaveDataMount3(const SaveDataMount3*, SaveDataMountResult*); +int APS5_VABI sceSaveDataUmount2(std::uint32_t, const SaveDataMountPoint*); +} + +static void Require(bool value, const char* message) { + if (!value) throw std::runtime_error(message); +} + +static void SearchAndMount() { + const std::array names{std::string(31, 'A'), std::string(31, 'B')}; + for (const auto& name : names) std::filesystem::create_directories(std::filesystem::path("_sd") / name); + + alignas(SceSaveDataDirName) std::array buffer; + buffer.fill('!'); + SaveDataDirNameSearchCond cond{}; + cond.user_id = 1; + SaveDataDirNameSearchResult result{}; + result.dir_names = reinterpret_cast(buffer.data()); + result.dir_names_num = 2; + Require(sceSaveDataDirNameSearch(&cond, &result) == 0, "search failed"); + Require(result.hit_num == 2 && result.set_num == 2, "search must return both saves"); + for (std::size_t i = 64; i < buffer.size(); ++i) { + Require(buffer[i] == '!', "directory search overran two 32-byte guest records"); + } + std::array found{}; + for (int i = 0; i < 2; ++i) { + const char* record = buffer.data() + i * 32; + Require(record[31] == '\0', "directory record must terminate within 32 bytes"); + const bool first = std::strcmp(record, names[0].c_str()) == 0; + Require(first || std::strcmp(record, names[1].c_str()) == 0, "returned directory name is corrupted"); + const int index = first ? 0 : 1; + Require(!found[index], "search returned a duplicate directory"); + found[index] = true; + + SaveDataMount3 mount{}; + mount.user_id = cond.user_id; + mount.dir_name = reinterpret_cast(record); + mount.mount_mode = 1; + SaveDataMountResult mounted{}; + Require(sceSaveDataMount3(&mount, &mounted) == 0, "returned directory must be mountable"); + Require(sceSaveDataUmount2(0, &mounted.mount_point) == 0, "unmount failed"); + } +} + +int main() { + const auto previous = std::filesystem::current_path(); + const auto root = std::filesystem::temp_directory_path() / + ("anyps5-savedata-search-" + std::to_string(std::random_device{}())); + std::filesystem::create_directories(root); + std::filesystem::current_path(root); + int status = 0; + try { + SearchAndMount(); + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + status = 1; + } + std::filesystem::current_path(previous); + std::filesystem::remove_all(root); + return status; +} diff --git a/core/libs/tests/GuestSceNet.cpp b/core/libs/tests/GuestSceNet.cpp index 4bbb122ce..4432ef709 100644 --- a/core/libs/tests/GuestSceNet.cpp +++ b/core/libs/tests/GuestSceNet.cpp @@ -1,5 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include "SceTypes.hpp" +#include #include #include #include @@ -9,6 +10,8 @@ #include extern "C" { +extern const std::uint8_t in6addr_any_nid_postfix[16]; +extern const std::uint8_t in6addr_loopback_nid_postfix[16]; int APS5_VABI sceNetInit_nid_postfix(void); int APS5_VABI sceNetSocket(const char*, int, int, int); int APS5_VABI sceNetBind_nid_postfix(int, const void*, std::uint32_t); @@ -27,11 +30,15 @@ int APS5_VABI sceNetEpollCreate(const char*, int); int APS5_VABI sceNetEpollControl(int, int, int, const NetEpollEvent*); int APS5_VABI sceNetEpollWait(int, NetEpollEvent*, int, int); int APS5_VABI sceNetEpollDestroy(int); +extern const std::uint32_t sce_net_in6addr_any[4]; int APS5_VABI sceNetResolverCreate(const char*, int, int); int APS5_VABI sceNetResolverStartNtoa(int, const char*, void*, int, int, int); +int APS5_VABI sceNetResolverStartNtoaMultipleRecordsEx(int, const char*, void*, int, int, int); int APS5_VABI sceNetResolverDestroy(int); int APS5_VABI sceNetResolverGetError(int, int*); int APS5_VABI sceNetCtlGetState(int*); +int APS5_VABI select_nid_postfix(int, void*, void*, void*, const void*); +int* APS5_VABI __error_nid_postfix(); int APS5_VABI sceNetInetPton(int, const char*, void*); const char* APS5_VABI sceNetInetNtop(int, const void*, char*, std::uint32_t); int* APS5_VABI sceNetErrnoLoc(void); @@ -52,9 +59,18 @@ struct NetMsghdr { int flags; }; +struct NetMemoryPoolStats { + std::size_t pool_size; + std::size_t max_inuse_size; + std::size_t current_inuse_size; +}; + extern "C" { std::int64_t APS5_VABI sceNetSendmsg(int, const NetMsghdr*, int); std::int64_t APS5_VABI sceNetRecvmsg(int, NetMsghdr*, int); +int APS5_VABI sceNetPoolCreate(const char*, int, int); +int APS5_VABI sceNetPoolDestroy(int); +int APS5_VABI sceNetGetMemoryPoolStats(int, NetMemoryPoolStats*); } static void Require(bool condition) { @@ -65,6 +81,17 @@ static bool Failed(std::int64_t result, int error) { return result == static_cast(0x80410100u | static_cast(error)) && *sceNetErrnoLoc() == error; } +static void CheckPoolStats() { + const int pool = sceNetPoolCreate("stats", 0x4000, 0); + Require(pool > 0); + NetMemoryPoolStats stats{1, 1, 1}; + Require(sceNetGetMemoryPoolStats(pool, &stats) == 0); + Require(stats.pool_size == 0x4000 && stats.max_inuse_size == 0 && stats.current_inuse_size == 0); + Require(Failed(sceNetGetMemoryPoolStats(pool, nullptr), 22)); + Require(sceNetPoolDestroy(pool) == 0); + Require(Failed(sceNetGetMemoryPoolStats(pool, &stats), 9)); +} + static void CheckMessages(int receiver, int sender, const std::array& address) { char head[] = "scatter "; char tail[] = "gather"; @@ -142,12 +169,52 @@ static void CheckAddressText(int family, const char* text) { Require(sceNetInetNtop(family, address.data(), nullptr, output.size()) == nullptr && *sceNetErrnoLoc() == 22); } +static void CheckUnspecifiedIpv6() { + std::array unspecified{}; + Require(sceNetInetPton(28, "::", unspecified.data()) == 1); + Require(std::memcmp(sce_net_in6addr_any, unspecified.data(), unspecified.size()) == 0); + char text[4] = {'x', 'x', 'x', 'x'}; + Require(sceNetInetNtop(28, sce_net_in6addr_any, text, 3) == text); + Require(std::strcmp(text, "::") == 0 && text[3] == 'x'); + + const int receiver = sceNetSocket("ipv6-any", 28, 2, 17); + const int sender = sceNetSocket("ipv6-loopback", 28, 2, 17); + Require(receiver >= 0 && sender >= 0); + std::array address{28, 28}; + std::memcpy(address.data() + 8, sce_net_in6addr_any, 16); + Require(sceNetBind_nid_postfix(receiver, address.data(), address.size()) == 0); + std::uint32_t size = address.size(); + address.fill(0xa5); + Require(sceNetGetsockname(receiver, address.data(), &size) == 0 && size == address.size()); + Require(address[0] == 28 && address[1] == 28 && (address[2] != 0 || address[3] != 0)); + Require(std::memcmp(address.data() + 8, unspecified.data(), unspecified.size()) == 0); + Require(sceNetInetPton(28, "::1", address.data() + 8) == 1); + + const char payload[] = "IPv6 wildcard receive"; + Require(sceNetSendto(sender, payload, sizeof(payload), 0, address.data(), address.size()) == sizeof(payload)); + char received[sizeof(payload)]{}; + std::array peer{}; + size = peer.size(); + Require(sceNetRecvfrom(receiver, received, sizeof(received), 0, peer.data(), &size) == sizeof(received)); + Require(std::memcmp(received, payload, sizeof(payload)) == 0 && size == peer.size() && peer[1] == 28); + Require(std::memcmp(peer.data() + 8, address.data() + 8, 16) == 0); + Require(std::memcmp(sce_net_in6addr_any, unspecified.data(), unspecified.size()) == 0); + Require(sceNetSocketClose(sender) == 0); + Require(sceNetSocketClose(receiver) == 0); +} + int main() { + for (int i = 0; i < 16; ++i) { + Require(in6addr_any_nid_postfix[i] == 0); + Require(in6addr_loopback_nid_postfix[i] == (i == 15 ? 1 : 0)); + } Require(sceNetInit_nid_postfix() == 0); + CheckPoolStats(); CheckAddressText(2, "127.0.0.1"); CheckAddressText(2, "255.255.255.255"); CheckAddressText(28, "::1"); CheckAddressText(28, "1234:5678:9abc:def0:1234:5678:9abc:def0"); + CheckUnspecifiedIpv6(); const int listener = sceNetSocket(nullptr, 2, 1, 6); Require(listener >= 0); @@ -165,10 +232,17 @@ int main() { address_size = peer.size(); const int accepted = sceNetAccept(listener, peer.data(), &address_size); Require(accepted >= 0 && address_size == 16); + Require(sceNetConnect(client, address.data(), address.size()) == static_cast(0x80410138) && *sceNetErrnoLoc() == 56); const int nonblocking = 1; Require(sceNetSetsockopt(accepted, 0xffff, 0x1200, &nonblocking, sizeof(nonblocking)) == 0); char pending = 0; Require(sceNetRecv(accepted, &pending, sizeof(pending), 0) == static_cast(0x80410123) && *sceNetErrnoLoc() == 35); + const int connecting = sceNetSocket(nullptr, 2, 1, 6); + Require(connecting >= 0); + Require(sceNetSetsockopt(connecting, 0xffff, 0x1200, &nonblocking, sizeof(nonblocking)) == 0); + const int started = sceNetConnect(connecting, address.data(), address.size()); + Require(started == 0 || (started == static_cast(0x80410124) && *sceNetErrnoLoc() == 36)); + Require(sceNetSocketClose(connecting) == 0); const int epoll = sceNetEpollCreate("guest-sce-net", 0); Require(epoll >= 0); @@ -222,6 +296,20 @@ int main() { Require(sceNetRecvfrom(udp_receiver, datagram_result, sizeof(datagram_result), 0, source.data(), &address_size) == sizeof(datagram_result)); Require(std::strcmp(datagram, datagram_result) == 0 && source[1] == 2); + std::uint64_t readable[16]{}; + readable[udp_receiver / 64] |= std::uint64_t{1} << (udp_receiver % 64); + const std::int64_t poll_now[2]{0, 0}; + Require(select_nid_postfix(udp_receiver + 1, readable, nullptr, nullptr, poll_now) == 0); + Require(readable[udp_receiver / 64] == 0); + Require(sceNetSendto(udp_sender, datagram, sizeof(datagram), 0, address.data(), address.size()) == sizeof(datagram)); + readable[udp_receiver / 64] |= std::uint64_t{1} << (udp_receiver % 64); + const std::int64_t wait_second[2]{1, 0}; + Require(select_nid_postfix(udp_receiver + 1, readable, nullptr, nullptr, wait_second) == 1); + Require(((readable[udp_receiver / 64] >> (udp_receiver % 64)) & 1u) != 0); + Require(sceNetRecvfrom(udp_receiver, datagram_result, sizeof(datagram_result), 0, nullptr, nullptr) == sizeof(datagram_result)); + const std::int64_t bad_timeout[2]{0, 1000000}; + Require(select_nid_postfix(udp_receiver + 1, readable, nullptr, nullptr, bad_timeout) == -1 && *__error_nid_postfix() == 22); + Require(select_nid_postfix(1025, nullptr, nullptr, nullptr, poll_now) == -1 && *__error_nid_postfix() == 22); CheckMessages(udp_receiver, udp_sender, address); Require(sceNetSocketClose(udp_sender) == 0); Require(sceNetSocketClose(udp_receiver) == 0); @@ -241,11 +329,38 @@ int main() { Require(sceNetResolverStartNtoa(resolver, "localhost", ipv4.data(), 5000000, 1, 0) == 0); Require(ipv4[0] == 127); Require(sceNetResolverGetError(resolver, &resolver_error) == 0 && resolver_error == 0); + std::array records{}; + records.fill(0xA5); + Require(sceNetResolverStartNtoaMultipleRecordsEx(resolver, "localhost", records.data(), 5000000, 1, 0) == 0); + std::int32_t record_family = 0; + std::int32_t record_count = 0; + std::int32_t record_count4 = 0; + std::memcpy(&record_family, records.data() + 16, sizeof(record_family)); + std::memcpy(&record_count, records.data() + 320, sizeof(record_count)); + std::memcpy(&record_count4, records.data() + 324, sizeof(record_count4)); + Require(records[0] == 127 && record_family == 2 && record_count >= 1 && record_count <= 10 && record_count4 == record_count); + Require(std::all_of(records.begin() + 32 * record_count, records.begin() + 320, [](std::uint8_t byte) { return byte == 0; })); + Require(std::all_of(records.begin() + 328, records.begin() + 384, [](std::uint8_t byte) { return byte == 0; })); + Require(std::all_of(records.begin() + 384, records.end(), [](std::uint8_t byte) { return byte == 0xA5; })); + for (int first = 0; first < record_count; ++first) { + for (int second = first + 1; second < record_count; ++second) { + Require(std::memcmp(records.data() + 32 * first, records.data() + 32 * second, 4) != 0); + } + } + const auto resolved = records; + Require(sceNetResolverStartNtoaMultipleRecordsEx(resolver, nullptr, records.data(), 5000000, 1, 0) == + static_cast(0x80410116) && *sceNetErrnoLoc() == 22 && records == resolved); + Require(sceNetResolverStartNtoaMultipleRecordsEx(resolver, "localhost", nullptr, 5000000, 1, 0) == + static_cast(0x80410116) && *sceNetErrnoLoc() == 22); + Require(sceNetResolverStartNtoaMultipleRecordsEx(resolver, "guest-sce-net.invalid", records.data(), 5000000, 1, 0) == + static_cast(0x804101E1) && records == resolved); Require(sceNetResolverGetError(resolver, nullptr) == static_cast(0x80410116) && *sceNetErrnoLoc() == 22); Require(sceNetResolverDestroy(resolver) == 0); resolver_error = -1; Require(sceNetResolverGetError(resolver, &resolver_error) == static_cast(0x80410109) && *sceNetErrnoLoc() == 9 && resolver_error == -1); + Require(sceNetResolverStartNtoaMultipleRecordsEx(resolver, "localhost", records.data(), 5000000, 1, 0) == + static_cast(0x80410109) && *sceNetErrnoLoc() == 9 && records == resolved); std::array ipv6{}; Require(sceNetInetPton(28, "::1", ipv6.data()) == 1); diff --git a/core/libs/tests/GuestShaderLink.cpp b/core/libs/tests/GuestShaderLink.cpp index 5e8f3b31a..20ba2f0cd 100644 --- a/core/libs/tests/GuestShaderLink.cpp +++ b/core/libs/tests/GuestShaderLink.cpp @@ -5,10 +5,31 @@ #include #include #include +#include extern "C" int APS5_VABI sceAgcLinkShaders(ShaderRegister*, ShaderRegister*, const void*, const Shader*, const Shader*, std::uint32_t); extern "C" void* APS5_VABI sceAgcGetRegisterDefaults(); extern "C" void* APS5_VABI sceAgcGetRegisterDefaults2(std::uint32_t); static void Require(bool value) { if (!value) std::abort(); } +static unsigned preparations = 0; +static unsigned mappings = 0; +static unsigned links = 0; +static const Shader* mappedPixel = nullptr; + +extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span stages, std::span context, std::span primitive) { + Require(!stages.empty() && stages[0] != nullptr && context.size() == 2u && primitive.size() == 3u); + ++preparations; +} + +extern "C" void AgcDriverResolveShaderAbi_nid_postfix(const Shader* shader, std::span context, std::span primitive) { + Require(shader != nullptr && context.size() == 32u && primitive.empty()); + mappedPixel = shader; + ++mappings; +} + +extern "C" void AgcDriverResolveGraphicsAbi_nid_postfix(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType) { + Require(vertex != nullptr && (pixel == nullptr || pixel == mappedPixel) && (primitiveType == 0u || primitiveType == 2u || primitiveType == 4u)); + ++links; +} int main() { using namespace ShaderRegs; auto* defaults = static_cast(sceAgcGetRegisterDefaults()); @@ -83,4 +104,5 @@ int main() { input.default_value = 2; Require(sceAgcLinkShaders(context.data(), primitive.data(), nullptr, &vertex, &pixel, 4) == 0); Require(context[2].value == 0x220); + Require(preparations == 4u && mappings == 3u && links == 7u); } diff --git a/core/libs/tests/GuestSocketPair.cpp b/core/libs/tests/GuestSocketPair.cpp new file mode 100644 index 000000000..f6a29a846 --- /dev/null +++ b/core/libs/tests/GuestSocketPair.cpp @@ -0,0 +1,151 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include + +struct PollDescriptor { + int descriptor; + short events; + short revents; +}; + +extern "C" { +int APS5_VABI socketpair_nid_postfix(int, int, int, int*); +std::int64_t APS5_VABI send_nid_postfix(int, const void*, std::uint64_t, int); +std::int64_t APS5_VABI recv_nid_postfix(int, void*, std::uint64_t, int); +std::int64_t APS5_VABI recvfrom_nid_postfix(int, void*, std::uint64_t, int, void*, std::uint32_t*); +int APS5_VABI getsockname_nid_postfix(int, void*, std::uint32_t*); +int APS5_VABI getpeername_nid_postfix(int, void*, std::uint32_t*); +int APS5_VABI shutdown_nid_postfix(int, int); +int APS5_VABI fcntl_nid_postfix(int, int, ...); +int APS5_VABI poll_nid_postfix(PollDescriptor*, std::uint32_t, int); +int APS5_VABI close_nid_postfix(int); +int* APS5_VABI __error_nid_postfix(); +} + +static void Require(bool value) { if (!value) std::abort(); } + +constexpr int Unix = 1; +constexpr int Stream = 1; +constexpr int Datagram = 2; +constexpr int SequencedPacket = 5; +constexpr int CloseOnExec = 0x10000000; +constexpr int NonBlocking = 0x20000000; +constexpr int NoSignal = 0x20000; +constexpr int Peek = 2; +constexpr int FirstSocket = 0x10000000; +constexpr short In = 0x1; +constexpr short Out = 0x4; +constexpr short Hup = 0x10; + +static bool Fails(int domain, int type, int protocol, int error) { + int pair[2] = {-7, -7}; + *__error_nid_postfix() = 0; + const int result = socketpair_nid_postfix(domain, type, protocol, pair); + return result == -1 && *__error_nid_postfix() == error && pair[0] == -7 && pair[1] == -7; +} + +static bool Readable(int descriptor, short accepted = In) { + PollDescriptor waited{descriptor, In, 0}; + return poll_nid_postfix(&waited, 1, 2000) == 1 && (waited.revents & accepted) != 0; +} + +static bool Exchange(int from, int to, const char* text) { + char received[32]{}; + const auto length = static_cast(std::strlen(text) + 1); + if (send_nid_postfix(from, text, length, NoSignal) != static_cast(length)) return false; + if (!Readable(to)) return false; + if (recv_nid_postfix(to, received, sizeof(received), 0) != static_cast(length)) return false; + return std::strcmp(received, text) == 0; +} + +static bool Unnamed(int descriptor, int (APS5_VABI *query)(int, void*, std::uint32_t*)) { + std::uint8_t name[28]; + std::memset(name, 0xaa, sizeof(name)); + std::uint32_t length = sizeof(name); + if (query(descriptor, name, &length) != 0 || length != 16 || name[0] != 16 || name[1] != 1 || name[16] != 0xaa) return false; + length = 4; + std::memset(name, 0xaa, sizeof(name)); + return query(descriptor, name, &length) == 0 && length == 4 && name[0] == 16 && name[1] == 1 && name[4] == 0xaa; +} + +int main() { + int pair[2] = {-1, -1}; + Require(socketpair_nid_postfix(Unix, Stream, 0, pair) == 0); + Require(pair[0] >= FirstSocket && pair[1] >= FirstSocket && pair[0] != pair[1]); + Require(Exchange(pair[0], pair[1], "one way")); + Require(Exchange(pair[1], pair[0], "other way")); + Require(Unnamed(pair[0], getsockname_nid_postfix) && Unnamed(pair[1], getpeername_nid_postfix)); + + PollDescriptor polled[2] = {{pair[1], In, 0}, {pair[0], Out, 0}}; + Require(poll_nid_postfix(polled, 2, 0) == 1 && polled[0].revents == 0 && (polled[1].revents & Out) != 0); + const char ping[] = "ping"; + Require(send_nid_postfix(pair[0], ping, sizeof(ping), 0) == sizeof(ping)); + polled[0].revents = 0; + Require(poll_nid_postfix(polled, 1, 1000) == 1 && (polled[0].revents & In) != 0); + char peeked[8]{}; + Require(recv_nid_postfix(pair[1], peeked, sizeof(peeked), Peek) == sizeof(ping) && std::strcmp(peeked, ping) == 0); + char received[8]{}; + std::uint8_t from[28]; + std::uint32_t fromLength = sizeof(from); + Require(recvfrom_nid_postfix(pair[1], received, sizeof(received), 0, from, &fromLength) == sizeof(ping) && fromLength == 0); + Require(std::strcmp(received, ping) == 0); + + Require(fcntl_nid_postfix(pair[1], 3) == 2); + Require(fcntl_nid_postfix(pair[1], 4, 4) == 0); + Require(fcntl_nid_postfix(pair[1], 3) == 6); + Require(recv_nid_postfix(pair[1], received, sizeof(received), 0) == -1 && *__error_nid_postfix() == 35); + + Require(shutdown_nid_postfix(pair[0], 1) == 0); + Require(Readable(pair[1], In | Hup) && recv_nid_postfix(pair[1], received, sizeof(received), 0) == 0); + Require(Exchange(pair[1], pair[0], "still open")); + Require(close_nid_postfix(pair[0]) == 0); + Require(Readable(pair[1], In | Hup) && recv_nid_postfix(pair[1], received, sizeof(received), 0) == 0); +#ifndef _WIN32 + Require(send_nid_postfix(pair[1], ping, sizeof(ping), NoSignal) == -1 && *__error_nid_postfix() == 32); +#endif + Require(close_nid_postfix(pair[1]) == 0); + Require(close_nid_postfix(pair[1]) == -1 && *__error_nid_postfix() == 9); + + Require(socketpair_nid_postfix(Unix, Stream | NonBlocking | CloseOnExec, 0, pair) == 0); + Require(fcntl_nid_postfix(pair[0], 3) == 6 && fcntl_nid_postfix(pair[1], 3) == 6); + Require(recv_nid_postfix(pair[0], received, sizeof(received), 0) == -1 && *__error_nid_postfix() == 35); + Require(Exchange(pair[1], pair[0], "wake up")); + Require(close_nid_postfix(pair[0]) == 0 && close_nid_postfix(pair[1]) == 0); + + Require(socketpair_nid_postfix(Unix, Datagram, 0, pair) == 0); + Require(fcntl_nid_postfix(pair[0], 3) == 2); + Require(send_nid_postfix(pair[0], "a", 1, 0) == 1); + Require(send_nid_postfix(pair[0], "bb", 2, 0) == 2); + char datagram[16]{}; + std::uint8_t sender[28]; + std::uint32_t senderLength = sizeof(sender); + Require(Readable(pair[1])); + Require(recvfrom_nid_postfix(pair[1], datagram, sizeof(datagram), 0, sender, &senderLength) == 1 && datagram[0] == 'a'); + Require(senderLength == 16 && sender[0] == 16 && sender[1] == 1); + Require(recv_nid_postfix(pair[1], datagram, sizeof(datagram), 0) == 2 && datagram[0] == 'b' && datagram[1] == 'b'); + Require(Exchange(pair[1], pair[0], "back")); + Require(close_nid_postfix(pair[0]) == 0 && close_nid_postfix(pair[1]) == 0); + +#ifndef _WIN32 + Require(socketpair_nid_postfix(Unix, SequencedPacket, 0, pair) == 0); + Require(send_nid_postfix(pair[0], "a", 1, 0) == 1); + Require(send_nid_postfix(pair[0], "bb", 2, 0) == 2); + Require(recv_nid_postfix(pair[1], datagram, sizeof(datagram), 0) == 1); + Require(recv_nid_postfix(pair[1], datagram, sizeof(datagram), 0) == 2); + Require(close_nid_postfix(pair[0]) == 0 && close_nid_postfix(pair[1]) == 0); +#endif + + Require(Fails(2, Stream, 0, 45)); + Require(Fails(28, Datagram, 0, 45)); + Require(Fails(3, Stream, 0, 47)); + Require(Fails(Unix, 0, 0, 43)); + Require(Fails(Unix, 3, 0, 41)); + Require(Fails(Unix, 3, 6, 43)); + Require(Fails(Unix, Stream | 0x01000000, 0, 41)); + Require(Fails(Unix, Stream, 6, 43)); + *__error_nid_postfix() = 0; + Require(socketpair_nid_postfix(Unix, Stream, 0, nullptr) == -1 && *__error_nid_postfix() == 14); + return 0; +} diff --git a/core/libs/tests/GuestSsl.cpp b/core/libs/tests/GuestSsl.cpp index ed5ea97fe..327e6b0b5 100644 --- a/core/libs/tests/GuestSsl.cpp +++ b/core/libs/tests/GuestSsl.cpp @@ -1,5 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include +#include #include extern "C" { @@ -8,6 +9,15 @@ int APS5_VABI sceSslGetCaCerts(int, void*); int APS5_VABI sceSslFreeCaCerts(int, void*); } +struct SslMemoryPoolStats { + std::size_t pool_size; + std::size_t max_inuse_size; + std::size_t current_inuse_size; + std::int32_t reserved; +}; + +extern "C" int APS5_VABI sceSslGetMemoryPoolStats(int, SslMemoryPoolStats*); + static void Require(bool value) { if (!value) std::abort(); } struct SslCaCerts { @@ -23,6 +33,9 @@ int main() { const int context = sceSslInit_nid_postfix(0x10000); Require(context > 0); + SslMemoryPoolStats stats{1, 1, 1, 1}; + Require(sceSslGetMemoryPoolStats(context, &stats) == 0); + Require(stats.pool_size == 0x10000 && stats.max_inuse_size == 0 && stats.current_inuse_size == 0 && stats.reserved == 0); Require(sceSslGetCaCerts(context, nullptr) == invalidArg); Require(sceSslFreeCaCerts(context, nullptr) == invalidArg); diff --git a/core/libs/tests/GuestStandardStreams.cpp b/core/libs/tests/GuestStandardStreams.cpp index 714912017..99f344536 100644 --- a/core/libs/tests/GuestStandardStreams.cpp +++ b/core/libs/tests/GuestStandardStreams.cpp @@ -180,6 +180,10 @@ int main() { Require(&guest == &stream.GuestState()); Require(guest.position == nullptr && guest.readRemaining == 0 && guest.writeRemaining == 0); Require(guest.descriptor == fileno_nid_postfix(&stream)); + FileStream lineBuffered(std::tmpfile()); + Require(setvbuf_nid_postfix(&lineBuffered, nullptr, 1, 0) == 0); + FileStream fullyBuffered(std::tmpfile()); + Require(setvbuf_nid_postfix(&fullyBuffered, nullptr, 0, 0) == 0); Require(setvbuf_nid_postfix(&stream, nullptr, 2, 0) == 0); Require(fputc_nid_postfix('A', &stream) == 'A'); Require(--guest.writeRemaining < 0 && __swbuf_nid_postfix('\n', &stream) == '\n'); diff --git a/core/libs/tests/GuestStrings.cpp b/core/libs/tests/GuestStrings.cpp index 145980fed..019e4425c 100644 --- a/core/libs/tests/GuestStrings.cpp +++ b/core/libs/tests/GuestStrings.cpp @@ -93,6 +93,11 @@ static void CheckSscanfS() { Require(sscanf_s_nid_postfix(" 12 abc x", "%d %s %c", &number, word, 4u, &letter, 1u) == 3 && number == 12 && std::strcmp(word, "abc") == 0 && letter == 'x'); Require(sscanf_s_nid_postfix("12 abcd", "%d %s", &number, word, 4u) == 1 && word[0] == '\0'); Require(sscanf_s_nid_postfix("key=val", "%3[a-z]=%3s", word, 4u, value, 8u) == 2 && std::strcmp(word, "key") == 0 && std::strcmp(value, "val") == 0); + Require(sscanf_s_nid_postfix("abcdef", "%3s", word, 4u) == 1 && std::strcmp(word, "abc") == 0); + Require(sscanf_s_nid_postfix("abcdef", "%3s", value, 8u) == 1 && std::strcmp(value, "abc") == 0); + Require(sscanf_s_nid_postfix("abcdef", "%3[a-z]", value, 8u) == 1 && std::strcmp(value, "abc") == 0); + Require(sscanf_s_nid_postfix("2024ABCD 7", "%4s%s", word, 4u, value, 8u) == 0 && word[0] == '\0'); + Require(sscanf_s_nid_postfix("2024ABCD", "%3s%4s", word, 4u, value, 8u) == 2 && std::strcmp(word, "202") == 0 && std::strcmp(value, "4ABC") == 0); int position = 0; Require(sscanf_s_nid_postfix("7 %", "%d %%%n", &number, &position) == 1 && position == 3); Require(sscanf_s_nid_postfix(" ", "%d", &number) == EOF); diff --git a/core/libs/tests/GuestSystemFont.cpp b/core/libs/tests/GuestSystemFont.cpp index 53f205e5d..40a1c5588 100644 --- a/core/libs/tests/GuestSystemFont.cpp +++ b/core/libs/tests/GuestSystemFont.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -131,7 +132,7 @@ int main() { constexpr std::uint32_t VietnameseBold = 0x18070057u; constexpr std::uint32_t JapaneseJg2Light = 0x1A0835D3u; constexpr std::uint32_t ChineseGb = 0x180CB0D4u; - const std::filesystem::path root = std::filesystem::temp_directory_path() / "anyps5_guest_system_font"; + const std::filesystem::path root = std::filesystem::temp_directory_path() / ("anyps5_guest_system_font-" + std::to_string(std::random_device{}())); std::filesystem::remove_all(root); const std::filesystem::path empty = root / "empty"; const std::filesystem::path fonts = root / "fonts"; @@ -196,7 +197,10 @@ int main() { FontHandle missing = nullptr; Require(sceFontOpenFontSet(library, ChineseGb, 1, nullptr, &missing) == SCE_FONT_ERROR_FONT_OPEN_FAILED && missing == nullptr); - std::filesystem::remove(fonts / "SST-Bold.otf"); + const auto substituteFonts = root / "substitute-fonts"; + std::filesystem::create_directories(substituteFonts); + WriteFile(substituteFonts / "NotoSans-Bold.ttf", SquareGlyphFont('A', 300)); + SetFontDirectory(substituteFonts); FontHandle substitute = nullptr; Require(sceFontOpenFontSet(library, EuropeanBold, 3, nullptr, &substitute) == SCE_FONT_OK); Require(sceFontSetScalePixel(substitute, 100.0f, 100.0f) == SCE_FONT_OK); diff --git a/core/libs/tests/GuestSystemKillApp.cpp b/core/libs/tests/GuestSystemKillApp.cpp new file mode 100644 index 000000000..300d6524f --- /dev/null +++ b/core/libs/tests/GuestSystemKillApp.cpp @@ -0,0 +1,52 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libc/include/Shutdown.hpp" +#include +#include +#include +#include +extern "C" int APS5_VABI sceSystemServiceGetAppIdOfRunningBigApp(void); +extern "C" int APS5_VABI sceSystemServiceKillApp(int, int, int, int); +namespace { +bool cleaned = false; +bool finished = false; +void Cleanup() { + try { LibcAwaitExit_nid_postfix(); } + catch (const ProcessShutdown&) { return; } + catch (const std::runtime_error&) { cleaned = true; } +} +void VerifyExit() { + if (!cleaned) std::_Exit(1); + std::puts("Guest shutdown and atexit completed"); +} +void UnexpectedShutdown() { + if (!finished) std::_Exit(3); +} +void Require(bool value) { if (!value) std::abort(); } +bool Rejects(int appId, int how, int reason, int coreDump, const char* expected) { + try { sceSystemServiceKillApp(appId, how, reason, coreDump); } + catch (const std::runtime_error& error) { return std::strstr(error.what(), expected) != nullptr; } + return false; +} +} +int main(int argc, char** argv) { + const int appId = sceSystemServiceGetAppIdOfRunningBigApp(); + Require(appId > 0 && sceSystemServiceGetAppIdOfRunningBigApp() == appId); + if (argc > 1 && std::strcmp(argv[1], "--kill") == 0) { + LibcRegisterShutdown_nid_postfix(Cleanup); + Require(std::atexit(VerifyExit) == 0); + sceSystemServiceKillApp(appId, -1, 0, 0); + std::_Exit(4); + } + LibcRegisterShutdown_nid_postfix(UnexpectedShutdown); + Require(std::atexit(UnexpectedShutdown) == 0); + constexpr char otherApp[] = "sceSystemServiceKillApp: application other than the running title"; + constexpr char otherArguments[] = "sceSystemServiceKillApp: arguments other than -1, 0 and 0"; + Require(Rejects(appId + 1, -1, 0, 0, otherApp)); + Require(Rejects(-1, -1, 0, 0, otherApp)); + Require(Rejects(0, -1, 0, 0, otherApp)); + Require(Rejects(appId, 0, 0, 0, otherArguments)); + Require(Rejects(appId, -2, 0, 0, otherArguments)); + Require(Rejects(appId, -1, 1, 0, otherArguments)); + Require(Rejects(appId, -1, 0, 1, otherArguments)); + finished = true; +} diff --git a/core/libs/tests/GuestSystemService.cpp b/core/libs/tests/GuestSystemService.cpp index 6341c8d40..f013e0f5a 100644 --- a/core/libs/tests/GuestSystemService.cpp +++ b/core/libs/tests/GuestSystemService.cpp @@ -27,6 +27,8 @@ extern "C" int APS5_VABI sceSystemServicePowerTick(void); extern "C" int APS5_VABI sceSystemServiceReportAbnormalTermination(const void* info); extern "C" int APS5_VABI sceSystemServiceDisableMusicPlayer(void); extern "C" int APS5_VABI sceSystemServiceReenableMusicPlayer(void); +extern "C" int APS5_VABI sceSystemServiceDisableMediaPlay(void); +extern "C" int APS5_VABI sceSystemServiceReenableMediaPlay(void); int main() { Require(sceSystemServicePowerTick() == SYSTEM_SERVICE_OK); @@ -38,6 +40,10 @@ int main() { Require(sceSystemServiceDisableMusicPlayer() == SYSTEM_SERVICE_OK); Require(sceSystemServiceReenableMusicPlayer() == SYSTEM_SERVICE_OK); Require(sceSystemServiceReenableMusicPlayer() == SYSTEM_SERVICE_OK); + Require(sceSystemServiceDisableMediaPlay() == SYSTEM_SERVICE_OK); + Require(sceSystemServiceDisableMediaPlay() == SYSTEM_SERVICE_OK); + Require(sceSystemServiceReenableMediaPlay() == SYSTEM_SERVICE_OK); + Require(sceSystemServiceReenableMediaPlay() == SYSTEM_SERVICE_OK); Require(sceSystemServiceGetHdrToneMapLuminance(nullptr) == SYSTEM_SERVICE_ERROR_PARAMETER); SystemServiceHdrToneMapLuminance luminance{-1.0f, -1.0f, -1.0f}; Require(sceSystemServiceGetHdrToneMapLuminance(&luminance) == SYSTEM_SERVICE_OK); diff --git a/core/libs/tests/GuestTcp.cpp b/core/libs/tests/GuestTcp.cpp index 0b9b42f2a..f82ec5ab4 100644 --- a/core/libs/tests/GuestTcp.cpp +++ b/core/libs/tests/GuestTcp.cpp @@ -49,11 +49,17 @@ int main() { const char request[] = "guest TCP loopback"; char received[sizeof(request)]{}; - Require(send_nid_postfix(client, request, sizeof(request), 0) == sizeof(request)); + Require(send_nid_postfix(client, request, sizeof(request), 0x1) == -1 && *__error_nid_postfix() == 45); + Require(send_nid_postfix(client, request, sizeof(request), 0x20000) == sizeof(request)); Require(recv_nid_postfix(accepted, received, sizeof(received), 0) == sizeof(received)); Require(std::strcmp(request, received) == 0); Require(close_nid_postfix(accepted) == 0); +#ifndef _WIN32 + std::int64_t sent = 0; + for (int i = 0; i < 100 && sent >= 0; ++i) sent = send_nid_postfix(client, request, sizeof(request), 0x20000); + Require(sent == -1 && *__error_nid_postfix() == 32); +#endif Require(close_nid_postfix(client) == 0); Require(close_nid_postfix(listener) == 0); Require(close_nid_postfix(listener) == -1 && *__error_nid_postfix() == 9); diff --git a/core/libs/tests/GuestTextToSpeech2.cpp b/core/libs/tests/GuestTextToSpeech2.cpp new file mode 100644 index 000000000..f3ee2c80e --- /dev/null +++ b/core/libs/tests/GuestTextToSpeech2.cpp @@ -0,0 +1,31 @@ +#include "prx/libc/include/general/VabiMacros.hpp" + +#include +#include +#include +#include + +extern "C" { +std::int32_t APS5_VABI sceTextToSpeech2Initialize(const void* param); +int APS5_VABI sceTextToSpeech2Open(); +} + +int main() { + const std::uint32_t param[12]{0x2000000, 0x26c}; + if (sceTextToSpeech2Initialize(param) != static_cast(0x8002002D)) { + std::puts("TextToSpeech2: initialization did not report the unsupported operation"); + std::abort(); + } + bool threw = false; + try { + sceTextToSpeech2Open(); + } catch (const std::runtime_error&) { + threw = true; + } + if (!threw) { + std::puts("TextToSpeech2: open after failed initialization did not throw"); + std::abort(); + } + std::puts("TextToSpeech2 tests passed"); + return 0; +} diff --git a/core/libs/tests/GuestThreadCallbackAbi.cpp b/core/libs/tests/GuestThreadCallbackAbi.cpp new file mode 100644 index 000000000..a496f8851 --- /dev/null +++ b/core/libs/tests/GuestThreadCallbackAbi.cpp @@ -0,0 +1,96 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +using GuestDestructor = void(APS5_VABI *)(void*); + +extern "C" { +int APS5_VABI __cxa_thread_atexit_impl_nid_postfix(GuestDestructor, void*, void*); +int APS5_VABI LibcInternalExtCxaThreadAtexit_nid_postfix(GuestDestructor, void*, void*); +} + +namespace { + +struct Call { + int id; + void* object; +}; + +std::array calls{}; +std::atomic callCount{0}; +std::atomic registrationFailed{false}; + +void Record(int id, void* object) { + const auto index = callCount.fetch_add(1, std::memory_order_relaxed); + if (index < calls.size()) + calls[index] = {id, object}; +} + +void APS5_VABI FirstDestructor(void* object) { Record(1, object); } +void APS5_VABI SecondDestructor(void* object) { Record(2, object); } +void APS5_VABI ThirdDestructor(void* object) { Record(3, object); } + +void Fail(const char* message) { std::fprintf(stderr, "%s\n", message); } + +} + +int main() { + void* module = +#ifdef _WIN32 + reinterpret_cast(GetModuleHandleW(nullptr)); +#else + dlopen(nullptr, RTLD_NOW); +#endif + if (module == nullptr) { + Fail("could not acquire the test module handle"); + return 1; + } + + void* directDso = module; +#ifdef _WIN32 + directDso = nullptr; +#endif + + int firstObject = 1; + int secondObject = 2; + int thirdObject = 3; + std::thread worker([&] { + if (__cxa_thread_atexit_impl_nid_postfix(FirstDestructor, &firstObject, directDso) != 0) + registrationFailed.store(true); + if (__cxa_thread_atexit_impl_nid_postfix(SecondDestructor, &secondObject, directDso) != 0) + registrationFailed.store(true); + if (LibcInternalExtCxaThreadAtexit_nid_postfix(ThirdDestructor, &thirdObject, module) != 0) + registrationFailed.store(true); + }); + worker.join(); + +#ifndef _WIN32 + dlclose(module); +#endif + + if (registrationFailed.load()) { + Fail("thread destructor registration failed"); + return 1; + } + if (callCount.load() != calls.size()) { + Fail("thread exit did not run each registered destructor once"); + return 1; + } + const std::array expected = {{{3, &thirdObject}, {2, &secondObject}, {1, &firstObject}}}; + for (std::size_t index = 0; index < expected.size(); ++index) { + if (calls[index].id != expected[index].id || calls[index].object != expected[index].object) { + std::fprintf(stderr, "thread destructor %d received object %p; expected %p\n", calls[index].id, + calls[index].object, expected[index].object); + return 1; + } + } + return 0; +} diff --git a/core/libs/tests/GuestTimeConversion.cpp b/core/libs/tests/GuestTimeConversion.cpp new file mode 100644 index 000000000..005c15451 --- /dev/null +++ b/core/libs/tests/GuestTimeConversion.cpp @@ -0,0 +1,82 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +std::tm* APS5_VABI libc_gmtime_nid_postfix(const std::int64_t*); +std::tm* APS5_VABI gmtime_nid_postfix(const std::int64_t*); +std::tm* APS5_VABI libc_localtime_nid_postfix(const std::int64_t*); +std::tm* APS5_VABI localtime_nid_postfix(const std::int64_t*); +std::tm* APS5_VABI gmtime_s_nid_postfix(const std::int64_t*, std::tm*); +std::tm* APS5_VABI localtime_s_nid_postfix(const std::int64_t*, std::tm*); +} + +using Converter = std::tm* (APS5_VABI *)(const std::int64_t*); +using BufferedConverter = std::tm* (APS5_VABI *)(const std::int64_t*, std::tm*); + +static void Require(bool value, const char* message) { + if (!value) { + std::fprintf(stderr, "%s\n", message); + std::abort(); + } +} + +static bool Equal(const std::tm& left, const std::tm& right) { + return left.tm_sec == right.tm_sec && left.tm_min == right.tm_min && left.tm_hour == right.tm_hour + && left.tm_mday == right.tm_mday && left.tm_mon == right.tm_mon && left.tm_year == right.tm_year + && left.tm_wday == right.tm_wday && left.tm_yday == right.tm_yday && left.tm_isdst == right.tm_isdst; +} + +static void CheckConcurrent(Converter convert, BufferedConverter buffered, const char* name) { + constexpr std::array timers{0, 86400, 946684800, 1078012800, 1609459200, 1709164800, 1893456000, 2145916799}; + std::array expected{}; + for (std::size_t index = 0; index < timers.size(); ++index) + Require(buffered(&timers[index], &expected[index]) == &expected[index], "Buffered conversion failed"); + std::barrier start(static_cast(timers.size())); + std::atomic matches{true}; + std::array workers; + for (std::size_t index = 0; index < workers.size(); ++index) { + workers[index] = std::thread([&, index] { + start.arrive_and_wait(); + for (int iteration = 0; iteration < 50000; ++iteration) { + const auto* result = convert(&timers[index]); + if (result == nullptr || !Equal(*result, expected[index])) { + matches = false; + break; + } + } + }); + } + for (auto& worker : workers) worker.join(); + Require(matches, name); + const std::int64_t invalid = std::numeric_limits::max(); + Require(convert(&invalid) == nullptr, "Unrepresentable time was accepted"); +} + +int main() { +#ifdef _WIN32 + _putenv_s("TZ", "UTC-2"); + _tzset(); +#else + setenv("TZ", "UTC-2", 1); + tzset(); +#endif + const std::int64_t epoch = 0; + std::tm utc{}, local{}; + Require(gmtime_s_nid_postfix(&epoch, &utc) == &utc && utc.tm_year == 70 && utc.tm_mon == 0 + && utc.tm_mday == 1 && utc.tm_hour == 0, "UTC epoch conversion failed"); + Require(localtime_s_nid_postfix(&epoch, &local) == &local && local.tm_year == 70 && local.tm_mon == 0 + && local.tm_mday == 1 && local.tm_hour == 2, "Local epoch conversion failed"); + CheckConcurrent(libc_gmtime_nid_postfix, gmtime_s_nid_postfix, "Concurrent libc_gmtime returned another thread's date"); + CheckConcurrent(gmtime_nid_postfix, gmtime_s_nid_postfix, "Concurrent gmtime returned another thread's date"); + CheckConcurrent(libc_localtime_nid_postfix, localtime_s_nid_postfix, "Concurrent libc_localtime returned another thread's date"); + CheckConcurrent(localtime_nid_postfix, localtime_s_nid_postfix, "Concurrent localtime returned another thread's date"); +} diff --git a/core/libs/tests/GuestUdp.cpp b/core/libs/tests/GuestUdp.cpp index 488ea5221..ac9b76ea1 100644 --- a/core/libs/tests/GuestUdp.cpp +++ b/core/libs/tests/GuestUdp.cpp @@ -54,6 +54,12 @@ int main() { Require(getsockopt_nid_postfix(sender, 0xffff, 0x20, &option, &optionSize) == 0 && option != 0); Require(setsockopt_nid_postfix(sender, 0xffff, 12345, &enabled, sizeof(enabled)) == -1); Require(*__error_nid_postfix() == 42); + const int unsized = socket_nid_postfix(2, 2, 0); + std::array unsizedAddress{0, 2, 0, 0, 127, 0, 0, 1}; + Require(unsized >= 0 && bind_nid_postfix(unsized, unsizedAddress.data(), unsizedAddress.size()) == 0); + std::uint32_t unsizedLength = unsizedAddress.size(); + Require(getsockname_nid_postfix(unsized, unsizedAddress.data(), &unsizedLength) == 0 && unsizedAddress[0] == 16); + Require(close_nid_postfix(unsized) == 0); Require(ioctl_nid_postfix(receiver, 0x8004667e, &enabled) == 0); Require(fcntl_nid_postfix(receiver, 3) == 6); Require(fcntl_nid_postfix(receiver, 4, 2) == 0); @@ -66,7 +72,7 @@ int main() { Require(recvfrom_nid_postfix(receiver, buffer, sizeof(buffer), 0, nullptr, nullptr) == -1); Require(*__error_nid_postfix() == 35); const char message[] = "guest UDP loopback"; - Require(sendto_nid_postfix(sender, message, sizeof(message), 0, destination.data(), destination.size()) == sizeof(message)); + Require(sendto_nid_postfix(sender, message, sizeof(message), 0x20000, destination.data(), destination.size()) == sizeof(message)); int queued = 0; for (int i = 0; i < 100 && queued == 0; ++i) { Require(ioctl_nid_postfix(receiver, 0x4004667f, &queued) == 0); diff --git a/core/libs/tests/GuestUltThread.cpp b/core/libs/tests/GuestUltThread.cpp new file mode 100644 index 000000000..e46308a40 --- /dev/null +++ b/core/libs/tests/GuestUltThread.cpp @@ -0,0 +1,94 @@ +#include "SceTypes.hpp" +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceUltInitialize(); +int APS5_VABI sceUltFinalize(); +std::uint64_t APS5_VABI sceUltUlthreadRuntimeGetWorkAreaSize(std::uint32_t, std::uint32_t); +int APS5_VABI sceUltUlthreadRuntimeCreate(void*, const char*, std::uint32_t, std::uint32_t, void*, const void*, std::uint32_t); +int APS5_VABI sceUltUlthreadCreate(void*, const char*, UltUlthreadEntry, std::uint64_t, void*, std::uint64_t, void*, const void*, std::uint32_t); +int APS5_VABI sceUltUlthreadJoin(void*, std::int32_t*); +} + +static constexpr int Ok = 0; +static constexpr int Null = -2139029503; +static constexpr int State = -2139029498; +static constexpr int Busy = -2139029497; + +static void Require(bool value, int line) { + if (!value) { + std::fprintf(stderr, "ULT thread check failed at line %d\n", line); + std::abort(); + } +} + +#define CHECK(value) Require((value), __LINE__) + +static std::atomic gStarted{false}; +static std::atomic gRelease{false}; + +static std::int32_t APS5_VABI Parked(std::uint64_t arg) { + gStarted.store(true, std::memory_order_release); + while (!gRelease.load(std::memory_order_acquire)) std::this_thread::yield(); + return static_cast(arg); +} + +static std::int32_t APS5_VABI Immediate(std::uint64_t arg) { + return static_cast(arg); +} + +struct alignas(8) Runtime { std::uint8_t bytes[4096]{}; }; +struct alignas(8) Thread { std::uint8_t bytes[512]{}; }; + +static void Outstanding() { + Runtime runtime; + Thread thread; + CHECK(sceUltUlthreadRuntimeCreate(&runtime, "runtime", 1, 1, nullptr, nullptr, 0) == Ok); + CHECK(sceUltUlthreadCreate(&thread, "parked", Parked, 0x5A, nullptr, 0, &runtime, nullptr, 0) == Ok); + while (!gStarted.load(std::memory_order_acquire)) std::this_thread::yield(); + + const int outstanding = sceUltFinalize(); + gRelease.store(true, std::memory_order_release); + std::int32_t status = 0; + const int joined = sceUltUlthreadJoin(&thread, &status); + + CHECK(outstanding == Busy); + CHECK(joined == Ok && status == 0x5A); + CHECK(sceUltUlthreadJoin(&thread, &status) == State); + CHECK(sceUltFinalize() == Ok); +} + +static void Validation() { + Runtime runtime; + Thread thread; + gStarted.store(false, std::memory_order_release); + gRelease.store(false, std::memory_order_release); + CHECK(sceUltUlthreadRuntimeCreate(nullptr, "runtime", 1, 1, nullptr, nullptr, 0) == Null); + CHECK(sceUltUlthreadRuntimeCreate(&runtime, "runtime", 1, 1, nullptr, nullptr, 0) == Ok); + CHECK(sceUltUlthreadCreate(nullptr, "thread", Immediate, 0, nullptr, 0, &runtime, nullptr, 0) == Null); + CHECK(sceUltUlthreadCreate(&thread, "thread", nullptr, 0, nullptr, 0, &runtime, nullptr, 0) == Null); + CHECK(sceUltUlthreadCreate(&thread, "thread", Immediate, 0, nullptr, 0, nullptr, nullptr, 0) == Null); + CHECK(sceUltUlthreadJoin(nullptr, nullptr) == Null); + CHECK(sceUltUlthreadJoin(&thread, nullptr) == State); + + CHECK(sceUltUlthreadCreate(&thread, "thread", Immediate, 0x27, nullptr, 0, &runtime, nullptr, 0) == Ok); + CHECK(sceUltUlthreadCreate(&thread, "thread", Immediate, 0x27, nullptr, 0, &runtime, nullptr, 0) == State); + CHECK(sceUltFinalize() == Busy); + std::int32_t status = 0; + CHECK(sceUltUlthreadJoin(&thread, &status) == Ok && status == 0x27); + CHECK(sceUltFinalize() == Ok); + CHECK(sceUltUlthreadRuntimeGetWorkAreaSize(1, 1) == 256u + 16u * 1024u); +} + +int main(int argc, char** argv) { + CHECK(argc == 2); + CHECK(sceUltInitialize() == Ok); + if (std::strcmp(argv[1], "outstanding") == 0) Outstanding(); + else if (std::strcmp(argv[1], "validation") == 0) Validation(); + else CHECK(false); +} diff --git a/core/libs/tests/GuestUnwindModuleInfo.cpp b/core/libs/tests/GuestUnwindModuleInfo.cpp new file mode 100644 index 000000000..b5af722f4 --- /dev/null +++ b/core/libs/tests/GuestUnwindModuleInfo.cpp @@ -0,0 +1,58 @@ +#include "SceTypes.hpp" +#include "tests/GuestUnwindModuleInfoFixture.hpp" +#include +#include +#include +#include + +extern "C" int APS5_VABI sceKernelGetModuleInfoForUnwind(uint64_t addr, int flags, ModuleInfoForUnwind* info); + +static void Require(bool value) { if (!value) std::abort(); } + +static bool Throws(std::uint64_t address) { + ModuleInfoForUnwind info{}; + try { + sceKernelGetModuleInfoForUnwind(address, 0, &info); + } catch (const std::runtime_error&) { + return true; + } + return false; +} + +int main() { + auto* fixture = GetUnwindFixture(); + Require(fixture != nullptr); + const auto address = reinterpret_cast(fixture); + MEMORY_BASIC_INFORMATION memory{}; + Require(VirtualQuery(reinterpret_cast(address), &memory, sizeof(memory)) != 0); + const auto base = reinterpret_cast(memory.AllocationBase); + Require(base != reinterpret_cast(GetModuleHandleW(nullptr))); + + ModuleInfoForUnwind info{}; + Require(sceKernelGetModuleInfoForUnwind(address, 0, &info) == 0); + Require(info.st_size == sizeof(ModuleInfoForUnwind)); + Require(info.eh_frame_hdr_addr == reinterpret_cast(&fixture->header)); + Require(info.eh_frame_addr == reinterpret_cast(&fixture->frames)); + Require(info.eh_frame_size == 4 + 12); + Require(info.seg0_addr == base); + const auto* dos = reinterpret_cast(base); + const auto* nt = reinterpret_cast(base + dos->e_lfanew); + Require(info.seg0_size == nt->OptionalHeader.SizeOfImage); + + fixture->header.version = 2; + Require(Throws(address)); + fixture->header.version = 1; + for (const std::uint8_t encoding : {0x3b, 0x9b, 0x0f, 0xff}) { + fixture->header.framePointerEncoding = encoding; + Require(Throws(address)); + } + fixture->header.framePointerEncoding = 0x1b; + fixture->frames.cieLength = 0xffffffffu; + Require(Throws(address)); + fixture->frames.cieLength = 12; + Require(!Throws(address)); + + ModuleInfoForUnwind host{}; + Require(sceKernelGetModuleInfoForUnwind(reinterpret_cast(&GetModuleHandleW), 0, &host) == 0); + Require(host.eh_frame_hdr_addr == 0 && host.eh_frame_addr == 0 && host.eh_frame_size == 0); +} diff --git a/core/libs/tests/GuestUnwindModuleInfoFixture.cpp b/core/libs/tests/GuestUnwindModuleInfoFixture.cpp new file mode 100644 index 000000000..2e473b4bf --- /dev/null +++ b/core/libs/tests/GuestUnwindModuleInfoFixture.cpp @@ -0,0 +1,23 @@ +#include "tests/GuestUnwindModuleInfoFixture.hpp" + +#include +#include + +asm(".section .ehmeta,\"dw\"\n" + ".globl UnwindFixtureMeta\n" + ".p2align 2\n" + "UnwindFixtureMeta:\n" + " .long 0\n" + ".text\n"); + +extern "C" IMAGE_DOS_HEADER __ImageBase; +extern "C" std::uint32_t UnwindFixtureMeta; + +static UnwindFixture fixture; + +extern "C" UnwindFixture* GetUnwindFixture() { + const auto base = reinterpret_cast(&__ImageBase); + fixture.header.frames = static_cast(reinterpret_cast(&fixture.frames) - reinterpret_cast(&fixture.header.frames)); + UnwindFixtureMeta = static_cast(reinterpret_cast(&fixture.header) - base); + return &fixture; +} diff --git a/core/libs/tests/GuestUnwindModuleInfoFixture.hpp b/core/libs/tests/GuestUnwindModuleInfoFixture.hpp new file mode 100644 index 000000000..d03be890d --- /dev/null +++ b/core/libs/tests/GuestUnwindModuleInfoFixture.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +struct alignas(4) UnwindFixtureHeader { + std::uint8_t version = 1; + std::uint8_t framePointerEncoding = 0x1b; + std::uint8_t countEncoding = 0x03; + std::uint8_t tableEncoding = 0x3b; + std::int32_t frames = 0; + std::uint32_t count = 0; +}; + +struct alignas(4) UnwindFixtureFrames { + std::uint32_t cieLength = 12; + std::uint32_t cieId = 0; + std::uint8_t cie[8] = {1, 0, 1, 0x78, 16, 0, 0, 0}; + std::uint32_t terminator = 0; +}; + +struct UnwindFixture { + UnwindFixtureHeader header; + UnwindFixtureFrames frames; +}; + +extern "C" UnwindFixture* GetUnwindFixture(); diff --git a/core/libs/tests/GuestUsbd.cpp b/core/libs/tests/GuestUsbd.cpp new file mode 100644 index 000000000..910810ec4 --- /dev/null +++ b/core/libs/tests/GuestUsbd.cpp @@ -0,0 +1,64 @@ +#include "prx/libc/include/general/VabiMacros.hpp" + +#include +#include +#include +#include +#include + +namespace { + +struct UsbdTimeval { + std::int64_t seconds; + std::int64_t microseconds; +}; + +} + +extern "C" { +std::int32_t APS5_VABI sceUsbdInit(); +void APS5_VABI sceUsbdExit(); +std::int64_t APS5_VABI sceUsbdGetDeviceList(void*** list); +void APS5_VABI sceUsbdFreeDeviceList(void** list, std::int32_t unrefDevices); +std::int32_t APS5_VABI sceUsbdHandleEventsTimeout(const UsbdTimeval* timeout); +int APS5_VABI sceUsbdOpen(); +} + +namespace { + +constexpr std::int32_t invalidArgument = static_cast(0x80240002); + +void Require(bool condition, const char* message) { + if (!condition) { + std::fprintf(stderr, "USBD: %s\n", message); + std::abort(); + } +} + +} + +int main() { + Require(sceUsbdInit() == 0, "initialization failed"); + void** list = nullptr; + Require(sceUsbdGetDeviceList(&list) == 0, "a device was listed"); + Require(list != nullptr && list[0] == nullptr, "device list is not an empty null-terminated array"); + sceUsbdFreeDeviceList(list, 1); + Require(sceUsbdGetDeviceList(nullptr) == invalidArgument, "null list accepted"); + Require(sceUsbdHandleEventsTimeout(nullptr) == invalidArgument, "null timeout accepted"); + const UsbdTimeval invalid{0, 1000000}; + Require(sceUsbdHandleEventsTimeout(&invalid) == invalidArgument, "out-of-range microseconds accepted"); + const UsbdTimeval timeout{0, 50000}; + const auto start = std::chrono::steady_clock::now(); + Require(sceUsbdHandleEventsTimeout(&timeout) == 0, "event handling failed"); + Require(std::chrono::steady_clock::now() - start >= std::chrono::milliseconds(45), "event handling returned before the timeout"); + bool threw = false; + try { + sceUsbdOpen(); + } catch (const std::runtime_error&) { + threw = true; + } + Require(threw, "opening a device did not throw"); + sceUsbdExit(); + std::puts("USBD tests passed"); + return 0; +} diff --git a/core/libs/tests/GuestVideoOutWqhdDetection.cpp b/core/libs/tests/GuestVideoOutWqhdDetection.cpp index d0196accf..0cb8a54ec 100644 --- a/core/libs/tests/GuestVideoOutWqhdDetection.cpp +++ b/core/libs/tests/GuestVideoOutWqhdDetection.cpp @@ -1,7 +1,10 @@ #include "SceTypes.hpp" #include "prx/libc/include/Shutdown.hpp" +#include #include #include +#include +#include #include extern "C" { @@ -31,7 +34,20 @@ static bool SameStatus(const VideoOutOutputStatus& a, const VideoOutOutputStatus } int main() { - const int handle = sceVideoOutOpen(SYSTEM_USER, MAIN_BUS, 0, nullptr); + std::filesystem::create_directories("app0/sce_sys"); + { + std::ofstream param("app0/sce_sys/param.json", std::ios::binary); + param << R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}},"downloadDataSize":0})"; + Require(static_cast(param)); + } + int handle = 0; + try { + handle = sceVideoOutOpen(SYSTEM_USER, MAIN_BUS, 0, nullptr); + } catch (const std::runtime_error& error) { + if (std::getenv("ANYPS5_REQUIRE_DISPLAY") != nullptr) throw; + std::printf("skipped, no display or Vulkan device: %s\n", error.what()); + return 77; + } Require(handle > 0); VideoOutOutputStatus before{}; diff --git a/core/libs/tests/GuestVideodec2.cpp b/core/libs/tests/GuestVideodec2.cpp index 89e1d2328..a0fac8c59 100644 --- a/core/libs/tests/GuestVideodec2.cpp +++ b/core/libs/tests/GuestVideodec2.cpp @@ -194,6 +194,178 @@ constexpr std::array PictureHashes{ constexpr std::array DisplayOrderUnits{0, 2, 3, 1, 5, 4}; +constexpr std::uint32_t CodecHevc = 974921; +constexpr std::uint32_t CodecVp9 = 2382845; + +constexpr std::array HevcStream{ + 0x00, 0x00, 0x00, 0x01, 0x46, 0x01, 0x10, 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, + 0xff, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x1e, + 0x95, 0x90, 0x09, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, + 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x1e, 0xa0, 0x20, 0x82, 0x96, 0x59, 0x59, 0x2e, + 0x4c, 0xaf, 0x01, 0x68, 0x08, 0x00, 0x00, 0x03, 0x00, 0x08, 0x00, 0x00, 0x03, 0x00, 0xf0, 0x40, + 0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xc1, 0x73, 0xd0, 0x89, 0x00, 0x00, 0x01, 0x28, 0x01, 0xaf, + 0x08, 0xe0, 0xbb, 0xcb, 0xc1, 0x71, 0x26, 0x0f, 0xbf, 0x6c, 0xcb, 0x4d, 0x2b, 0x2c, 0xf4, 0x08, + 0xe0, 0x17, 0xeb, 0x16, 0xcc, 0x07, 0xd8, 0xdd, 0x11, 0xf8, 0x5e, 0x73, 0x79, 0x57, 0xcb, 0x69, + 0xa3, 0x1c, 0x9c, 0x43, 0x14, 0xdb, 0x61, 0x0e, 0xc5, 0xc6, 0x9c, 0x9c, 0xf8, 0xf2, 0x07, 0x35, + 0xf1, 0x98, 0x6c, 0x1a, 0x86, 0x53, 0x5a, 0xcd, 0x4d, 0xda, 0xff, 0xfb, 0x01, 0xda, 0xc3, 0x61, + 0xb7, 0x8c, 0xae, 0xc9, 0x76, 0xe4, 0x05, 0xd6, 0xfd, 0xe4, 0xcd, 0x93, 0x2c, 0xb1, 0x41, 0x29, + 0xce, 0x22, 0xf7, 0x77, 0x64, 0x0f, 0x1f, 0xd2, 0xc3, 0x68, 0x32, 0x2f, 0x83, 0xd7, 0x53, 0xbe, + 0x23, 0xff, 0x8c, 0x46, 0x18, 0x33, 0x90, 0x28, 0xa3, 0xef, 0xd7, 0xb1, 0x46, 0x84, 0xa0, 0xc2, + 0xd4, 0xd9, 0xce, 0xdf, 0x0a, 0x05, 0xec, 0xad, 0x63, 0xa4, 0x78, 0x27, 0x80, 0x4e, 0x3b, 0x4c, + 0xa6, 0x42, 0xbc, 0xe3, 0x39, 0x4d, 0xc4, 0xd3, 0x44, 0xf1, 0x22, 0xd0, 0x64, 0x9f, 0xd0, 0x88, + 0x7f, 0x39, 0x6d, 0x06, 0xc7, 0x79, 0x75, 0x2a, 0x44, 0x3d, 0x21, 0x41, 0x53, 0x60, 0x8d, 0x08, + 0x9f, 0xd0, 0x34, 0xa6, 0x72, 0x70, 0x3b, 0xa9, 0xfa, 0x62, 0xb1, 0x19, 0x9b, 0x0b, 0xa2, 0xfc, + 0xb4, 0xb3, 0x2c, 0x58, 0xad, 0xc6, 0xf3, 0xea, 0xc4, 0xd0, 0xf9, 0xea, 0x9e, 0xa7, 0xa4, 0x87, + 0xb8, 0x19, 0x68, 0xf1, 0xc1, 0x6b, 0x87, 0xa5, 0x99, 0xa0, 0x69, 0x24, 0x28, 0x80, 0xba, 0xd2, + 0x57, 0xc6, 0xc0, 0x1e, 0x69, 0x1b, 0xce, 0x96, 0xcd, 0x85, 0x9d, 0x59, 0x32, 0x92, 0x71, 0xe7, + 0x15, 0xf2, 0x9b, 0x71, 0x4b, 0xd4, 0xb6, 0x99, 0xb0, 0x71, 0x9b, 0x9b, 0xdb, 0x33, 0x28, 0x86, + 0x5e, 0x44, 0xf3, 0x5d, 0x5d, 0x7b, 0x87, 0x6c, 0x99, 0x8a, 0x6d, 0xaf, 0xf8, 0xdd, 0x39, 0x14, + 0xc0, 0x10, 0x85, 0xaf, 0x3a, 0x77, 0x60, 0x8b, 0xb7, 0x5f, 0x67, 0x45, 0xb4, 0xa7, 0xe7, 0x44, + 0x77, 0x0a, 0xb9, 0xe1, 0xb2, 0xbe, 0xde, 0x1b, 0x5a, 0xb8, 0xc8, 0x9b, 0x1f, 0x15, 0x37, 0xdc, + 0xeb, 0x55, 0x12, 0x37, 0xa1, 0xab, 0x9c, 0xcd, 0xdd, 0x42, 0x5d, 0x90, 0x89, 0xc5, 0x99, 0x54, + 0x6e, 0x9a, 0xd2, 0x71, 0x07, 0xe3, 0x81, 0x20, 0xad, 0xd4, 0x7c, 0xad, 0x69, 0xaf, 0xc7, 0x2d, + 0x7c, 0x84, 0xf1, 0x5d, 0x43, 0x4a, 0x3c, 0xf6, 0xf8, 0x7e, 0x6a, 0xb2, 0xdd, 0x00, 0xda, 0x30, + 0x00, 0x00, 0x00, 0x01, 0x46, 0x01, 0x30, 0x00, 0x00, 0x01, 0x02, 0x01, 0xd0, 0x19, 0x5f, 0x84, + 0x31, 0x84, 0x50, 0x62, 0xcd, 0x44, 0xa6, 0x4e, 0xbb, 0x2e, 0x87, 0xda, 0x21, 0x25, 0x1c, 0x13, + 0xfe, 0xc2, 0x97, 0xa3, 0xd5, 0xfd, 0xe2, 0x62, 0xd6, 0xe5, 0x50, 0x7a, 0xda, 0x98, 0x8d, 0xcf, + 0xbe, 0x40, 0x5d, 0x69, 0xb6, 0x5f, 0xdc, 0x32, 0x68, 0x05, 0x67, 0x70, 0xfd, 0xa0, 0xd3, 0xcd, + 0x88, 0xeb, 0x12, 0x2e, 0xac, 0xf5, 0x57, 0xa2, 0xc2, 0x81, 0x22, 0x6b, 0x8d, 0x69, 0x6d, 0xac, + 0x41, 0x05, 0x5b, 0x0b, 0x79, 0xe0, 0x86, 0x05, 0x86, 0x8b, 0x02, 0x51, 0x7d, 0x6e, 0x13, 0x63, + 0x36, 0xe6, 0x89, 0x22, 0x75, 0xd9, 0xb0, 0x39, 0x27, 0xcd, 0xac, 0x78, 0xe0, 0x3a, 0x6a, 0x17, + 0x15, 0x05, 0x83, 0x3a, 0xe9, 0x38, 0x7c, 0x75, 0x38, 0xf3, 0xd8, 0x42, 0x57, 0xa1, 0xef, 0x68, + 0xb8, 0xb5, 0x44, 0xfe, 0x4d, 0x66, 0xa1, 0x09, 0xc5, 0x62, 0xff, 0x87, 0x7e, 0xdb, 0x41, 0x6f, + 0x4d, 0x5f, 0x16, 0x49, 0xef, 0xca, 0xda, 0x67, 0x01, 0x29, 0x0b, 0xd6, 0xd8, 0xfc, 0x0b, 0x22, + 0xc8, 0x28, 0xff, 0xb4, 0xb3, 0xa9, 0xf8, 0x9b, 0x78, 0x7d, 0x33, 0xbf, 0xa6, 0x9b, 0x39, 0x64, + 0x57, 0x41, 0x1f, 0x27, 0x05, 0xfc, 0xcd, 0x44, 0xcd, 0xb2, 0xea, 0x79, 0xe0, 0x00, 0x00, 0x00, + 0x01, 0x46, 0x01, 0x50, 0x00, 0x00, 0x01, 0x02, 0x01, 0xe0, 0x44, 0x97, 0xe1, 0x8f, 0x40, 0x3d, + 0x73, 0xde, 0xdf, 0xf5, 0x1c, 0xff, 0xf6, 0xe2, 0x12, 0x16, 0x3d, 0xb6, 0x75, 0xad, 0x9e, 0xb5, + 0x2b, 0xe7, 0x4a, 0x40, 0xe9, 0x91, 0xd6, 0x86, 0x86, 0x2d, 0xcb, 0xe4, 0xb6, 0x58, 0x4a, 0x2d, + 0x7b, 0x1b, 0xdb, 0xc5, 0x48, 0x2d, 0x09, 0x4d, 0x32, 0x30, 0xde, 0xaa, 0x61, 0xc2, 0x27, 0x64, + 0x54, 0x71, 0x89, 0x32, 0x05, 0xc0, 0x00, 0x00, 0x00, 0x01, 0x46, 0x01, 0x50, 0x00, 0x00, 0x01, + 0x00, 0x01, 0xe0, 0x24, 0xff, 0xfa, 0x2c, 0x6e, 0xdc, 0x78, 0x3d, 0x08, 0x6d, 0x3b, 0xcf, 0xcc, + 0xb6, 0x16, 0xeb, 0x9d, 0x5f, 0x46, 0x0b, 0x24, 0x0e, 0x97, 0x99, 0x06, 0x00, 0x00, 0x00, 0x01, + 0x46, 0x01, 0x30, 0x00, 0x00, 0x01, 0x02, 0x01, 0xd0, 0x28, 0x95, 0xd7, 0xdc, 0x41, 0xc0, 0x61, + 0x14, 0xd6, 0xee, 0xd4, 0xe8, 0x24, 0x3e, 0x7f, 0x82, 0x22, 0xb0, 0x20, 0xb9, 0xf1, 0x69, 0x9b, + 0xfe, 0xcc, 0xb7, 0x2e, 0xbd, 0xdb, 0x20, 0xfa, 0xe5, 0x97, 0xd9, 0x95, 0x1d, 0x7e, 0x49, 0x35, + 0x69, 0xdb, 0x40, 0xa3, 0x5c, 0x0f, 0x74, 0x6f, 0x00, 0x3d, 0x36, 0x05, 0xc0, 0xc4, 0x37, 0x15, + 0xf9, 0x01, 0x2d, 0xdf, 0xcb, 0x70, 0x3d, 0xfa, 0x47, 0xec, 0x80, 0x07, 0x48, 0x08, 0xc8, 0x54, + 0xd3, 0xe3, 0x00, 0xa3, 0xe4, 0xce, 0x40, 0x05, 0x0b, 0x51, 0x73, 0x56, 0x33, 0x82, 0xa8, 0x0a, + 0xc0, 0xd3, 0xc5, 0xf1, 0xeb, 0x32, 0x8e, 0x4e, 0xa7, 0x6f, 0xb2, 0x64, 0x43, 0x54, 0x71, 0xd4, + 0xaf, 0xc0, 0x00, 0x00, 0x00, 0x01, 0x46, 0x01, 0x50, 0x00, 0x00, 0x01, 0x00, 0x01, 0xe0, 0x82, + 0x2f, 0x5f, 0xdc, 0x63, 0x70, 0x3f, 0xaf, 0x66, 0x75, 0x52, 0xe7, 0x7f, 0x62, 0xfb, 0x84, 0x9c, + 0xef, 0x60, 0x8f, 0xfa, 0x7e, 0x5f, 0x8b, 0x40, 0x15, 0x34, 0x69, 0xe2, 0xc4, 0xfc, 0xff, 0x53, + 0xb8, 0x1e, 0xc7, 0x20, 0x55, 0xdb, 0x5e, 0x05, 0xbd, 0x7a, 0x85, 0xb4, 0xb7, 0xc1, 0x8d, 0xdb, + 0x0f, 0xa6, 0xab, 0xb9, 0x00, 0x2a, 0xdb, 0x83, 0xa0, 0xb0, 0xc5, 0x76, 0x6e, 0xe1, 0xb8, 0x72, + 0xfd, 0x09, 0x61, 0x19, 0xe0, +}; + +constexpr std::array HevcHashes{ + 0x7291c31ee57c5065ull, 0x37b15adc47810098ull, 0xc84737b65a65b247ull, + 0xe39bc891f8286267ull, 0x734caf1ba961b3c2ull, 0x7deb44bd1ed30531ull, +}; + +constexpr std::array Vp9Stream{ + 0x44, 0x4b, 0x49, 0x46, 0x00, 0x00, 0x20, 0x00, 0x56, 0x50, 0x39, 0x30, 0x40, 0x00, 0x28, 0x00, + 0x1e, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x82, 0x49, 0x83, 0x42, + 0x00, 0x03, 0xf0, 0x02, 0x76, 0x08, 0x38, 0x24, 0x1c, 0x18, 0x0e, 0x00, 0x01, 0xc0, 0x6f, 0xb1, + 0xfb, 0x17, 0xa6, 0xd2, 0x0f, 0x5f, 0xee, 0xf7, 0xec, 0xbf, 0xb4, 0x0c, 0x93, 0xbe, 0x87, 0x11, + 0x35, 0x1c, 0x51, 0x84, 0x8b, 0x70, 0xad, 0xe4, 0x00, 0x00, 0x79, 0x3b, 0xff, 0xff, 0xf5, 0x94, + 0xad, 0x57, 0x6e, 0xae, 0x8e, 0x1b, 0xfe, 0x64, 0xff, 0xc9, 0x64, 0xba, 0xca, 0x1d, 0x7f, 0x45, + 0x16, 0x95, 0x13, 0x8d, 0xd0, 0x61, 0x19, 0x80, 0xb7, 0xdf, 0xff, 0xff, 0xd6, 0x52, 0x1e, 0x7b, + 0x3c, 0x16, 0x3f, 0xff, 0xfb, 0x52, 0xe7, 0x26, 0xab, 0xd6, 0x3a, 0x6a, 0xff, 0xf8, 0x1b, 0x47, + 0xcd, 0xe2, 0xf5, 0x48, 0x65, 0x1e, 0x9f, 0x5c, 0x1a, 0x74, 0x0a, 0x99, 0xc0, 0x04, 0x41, 0x02, + 0xa9, 0x86, 0x36, 0x1f, 0xff, 0xf3, 0x4f, 0x71, 0xdf, 0x8a, 0xcd, 0x8a, 0x88, 0xdd, 0x37, 0xa2, + 0x27, 0x3a, 0x20, 0xb8, 0xf0, 0x4f, 0x67, 0x59, 0x8a, 0x82, 0xa2, 0xde, 0x08, 0xe3, 0xae, 0x87, + 0xff, 0xf0, 0x36, 0x8f, 0x9b, 0xc5, 0xea, 0x90, 0xca, 0x3d, 0x3e, 0xb8, 0x34, 0xe8, 0x15, 0x33, + 0x80, 0x08, 0x82, 0x05, 0x53, 0x0c, 0x6c, 0x3f, 0xff, 0xe6, 0x9e, 0xe3, 0xbf, 0x15, 0x9d, 0xec, + 0xdf, 0xe9, 0xce, 0x60, 0x88, 0x09, 0x48, 0x81, 0xa2, 0xb6, 0x27, 0x3e, 0xf9, 0xa9, 0x7e, 0xa7, + 0xe1, 0x9d, 0xdd, 0x3a, 0xa0, 0x39, 0x3f, 0x8a, 0xba, 0x73, 0x50, 0xde, 0xcf, 0x29, 0x5c, 0x2a, + 0x92, 0x01, 0xec, 0x06, 0xbe, 0x8b, 0x1e, 0x14, 0x34, 0x6b, 0xb1, 0x3a, 0x01, 0x47, 0x08, 0xe7, + 0xbe, 0xb0, 0xa7, 0x03, 0x77, 0xeb, 0xf0, 0xed, 0xd8, 0xa1, 0x4e, 0x7c, 0xd9, 0xe4, 0x7c, 0x9d, + 0x67, 0xfe, 0x36, 0xab, 0x5e, 0xa9, 0x5e, 0x81, 0x74, 0x68, 0xa1, 0x4b, 0x61, 0x81, 0xe2, 0xe4, + 0x4c, 0x6a, 0x4f, 0xe5, 0x65, 0x3f, 0x4c, 0x78, 0xea, 0x68, 0x5c, 0xf9, 0xc6, 0x58, 0x1f, 0x58, + 0xe0, 0x40, 0x4b, 0x88, 0x6d, 0x15, 0x94, 0x8c, 0x83, 0x4a, 0xa3, 0xf3, 0xd5, 0x80, 0x7f, 0x83, + 0x52, 0x7d, 0xbe, 0xa6, 0x9f, 0xb4, 0x89, 0xda, 0xb6, 0x5a, 0xdf, 0x1f, 0x78, 0xd8, 0xbf, 0xfe, + 0x7f, 0x52, 0xfa, 0xfd, 0xa4, 0xeb, 0xda, 0x91, 0x9c, 0xb6, 0xd0, 0xc3, 0xfd, 0x8e, 0x04, 0x04, + 0xb8, 0x86, 0xd1, 0x56, 0xa2, 0x24, 0xe6, 0xbc, 0x5b, 0xbf, 0x82, 0x36, 0xb5, 0xa7, 0x87, 0x73, + 0xd8, 0xbd, 0x30, 0x33, 0x3f, 0x7c, 0xbb, 0x4f, 0x70, 0xed, 0x3f, 0x8d, 0x2b, 0xe2, 0xce, 0x62, + 0x09, 0xfa, 0x04, 0x31, 0x67, 0xff, 0x09, 0xc1, 0x5c, 0x29, 0xf8, 0x77, 0xfa, 0xb6, 0xa7, 0xde, + 0xaa, 0xc1, 0x87, 0xd9, 0xb4, 0x10, 0x0d, 0xda, 0x5d, 0xf7, 0xdd, 0x7d, 0x08, 0x7a, 0xbe, 0x60, + 0xeb, 0x85, 0x00, 0x08, 0x21, 0x8e, 0xa0, 0x4d, 0xff, 0x73, 0x66, 0x55, 0xb8, 0x61, 0x69, 0xc5, + 0xf2, 0xef, 0x96, 0x27, 0x16, 0x2f, 0x0d, 0x93, 0xcd, 0xcd, 0xba, 0x2a, 0x80, 0x59, 0x90, 0x2b, + 0xc5, 0x43, 0x49, 0x25, 0x49, 0x01, 0x15, 0x9f, 0xf4, 0xdd, 0x02, 0x31, 0x8b, 0xf0, 0xbe, 0x28, + 0x87, 0xbd, 0x3f, 0xce, 0xb1, 0x6e, 0xd2, 0x7f, 0x77, 0x41, 0x09, 0x8a, 0x04, 0x17, 0x87, 0x57, + 0x1b, 0x72, 0x79, 0x69, 0x41, 0x7d, 0xc5, 0x78, 0xb7, 0x53, 0xe7, 0x80, 0xd2, 0x7f, 0x1f, 0xb9, + 0xd5, 0x0f, 0xff, 0xf5, 0xb8, 0xd0, 0xc5, 0x47, 0x30, 0x09, 0x36, 0x6e, 0xb1, 0xfd, 0xe1, 0xe9, + 0xf9, 0x81, 0x1d, 0x98, 0x00, 0x6d, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x86, 0x00, 0x40, 0x92, 0x9c, 0x10, 0x46, 0x00, 0x00, 0x03, 0x60, 0x00, 0x00, 0x76, 0x21, + 0x2d, 0x37, 0xff, 0xff, 0xc3, 0x7f, 0xff, 0xff, 0xff, 0xd8, 0x59, 0x6b, 0xff, 0xa9, 0xff, 0xff, + 0xff, 0xda, 0x3d, 0xca, 0x5b, 0xc7, 0xff, 0xfd, 0x52, 0xf6, 0x2e, 0x02, 0xe0, 0x37, 0xe3, 0x5f, + 0xff, 0xea, 0x82, 0x5b, 0xff, 0xff, 0xfa, 0x7b, 0x85, 0xdf, 0x66, 0xbf, 0xff, 0xe6, 0xde, 0xef, + 0xed, 0x97, 0xff, 0xf9, 0x38, 0x4a, 0x7f, 0x61, 0xb5, 0xeb, 0x56, 0x73, 0xa7, 0xff, 0xff, 0x31, + 0x8b, 0xa1, 0xcf, 0xff, 0x5b, 0x90, 0x07, 0x17, 0xff, 0xd6, 0xe1, 0x74, 0x39, 0xff, 0xeb, 0x72, + 0x5d, 0x14, 0xff, 0xf5, 0xb8, 0x95, 0x54, 0x7f, 0xf7, 0x5d, 0x1e, 0x26, 0x68, 0x00, 0x7f, 0x00, + 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x00, 0x40, 0x92, 0x9c, 0x10, + 0x45, 0x00, 0x00, 0x03, 0x60, 0x00, 0x00, 0x7d, 0xb8, 0xff, 0xff, 0xb4, 0x7f, 0xff, 0xff, 0xfe, + 0x4d, 0x81, 0xbf, 0xff, 0x98, 0x9f, 0xff, 0xff, 0xe4, 0xd8, 0x1b, 0xff, 0xf9, 0x89, 0xff, 0xff, + 0xfe, 0x4d, 0x81, 0xbf, 0xd6, 0xbf, 0xff, 0xf9, 0x36, 0x0c, 0x1f, 0xff, 0xfc, 0x10, 0x3e, 0xfe, + 0x1f, 0xa1, 0x76, 0x5c, 0x22, 0x5f, 0x3b, 0x56, 0x57, 0xa0, 0x3c, 0x43, 0xf0, 0xc8, 0x00, 0x00, + 0x00, 0x19, 0xbb, 0xff, 0xff, 0xd8, 0x43, 0x10, 0x26, 0x67, 0xff, 0xff, 0xfe, 0x4a, 0x07, 0x8f, + 0x80, 0x7f, 0xff, 0xff, 0x94, 0xd2, 0x59, 0x02, 0xff, 0xff, 0x0d, 0x89, 0x18, 0x02, 0xff, 0xff, + 0x0d, 0x84, 0xb2, 0x05, 0xff, 0xfe, 0x1b, 0x14, 0xdd, 0x42, 0xff, 0xff, 0x0d, 0x82, 0xc8, 0xb1, + 0xff, 0xf9, 0x5c, 0x32, 0xfe, 0x00, 0x78, 0x60, 0x00, 0x56, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x00, 0x40, 0x92, 0x9c, 0x20, 0x46, 0x00, 0x00, 0x03, 0x60, + 0x00, 0x00, 0x73, 0xaf, 0xff, 0xff, 0xc3, 0x7f, 0xff, 0xff, 0xff, 0xd8, 0x19, 0x3f, 0xff, 0xe6, + 0x1b, 0xfb, 0x80, 0x95, 0x82, 0x3b, 0xd9, 0x4c, 0x81, 0xce, 0x52, 0x2b, 0xff, 0xff, 0xa0, 0xad, + 0xc9, 0x0e, 0x62, 0xff, 0xff, 0xff, 0x60, 0xf8, 0x75, 0x5f, 0xff, 0xff, 0x31, 0x92, 0xaa, 0x8f, + 0xfe, 0xeb, 0xa4, 0x4e, 0x5f, 0xff, 0x5b, 0x89, 0x55, 0x47, 0xff, 0x75, 0xd4, 0xd9, 0xef, 0xff, + 0xe4, 0x51, 0x64, 0x58, 0xff, 0xee, 0xba, 0xfc, 0x8d, 0x8f, 0x00, 0x95, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x00, 0x40, 0x92, 0x9c, 0x10, 0x44, 0x00, 0x00, + 0x03, 0x60, 0x00, 0x00, 0x73, 0xaf, 0xff, 0xff, 0xfb, 0x50, 0x3f, 0xff, 0xff, 0xff, 0xea, 0xc5, + 0x0d, 0xff, 0xff, 0xe0, 0x09, 0xfc, 0xad, 0x23, 0x5f, 0xf8, 0xa3, 0x64, 0x9b, 0x40, 0x76, 0x8d, + 0xd6, 0xd1, 0x83, 0x6f, 0x24, 0xd5, 0xff, 0xff, 0xd4, 0xa9, 0x45, 0xac, 0xda, 0xdc, 0xd2, 0xbf, + 0xff, 0xff, 0xfd, 0x58, 0xbf, 0x7f, 0x8a, 0x7f, 0xff, 0xff, 0xad, 0xb9, 0x2d, 0x3c, 0xbf, 0xff, + 0xd4, 0xca, 0x46, 0x7d, 0x7f, 0xff, 0xe1, 0x64, 0x96, 0x9e, 0x5f, 0xff, 0xea, 0x65, 0x4d, 0xe9, + 0x97, 0xff, 0xfa, 0x99, 0x16, 0x71, 0xaf, 0xff, 0xfc, 0x2c, 0x99, 0x7f, 0x5b, 0x8b, 0x50, 0x9c, + 0xdc, 0x43, 0xa8, 0xb7, 0xd5, 0x68, 0x6a, 0xbc, 0x78, 0xef, 0xc2, 0x3d, 0xfb, 0x3f, 0xcc, 0x51, + 0x40, 0x86, 0x54, 0x02, 0xa4, 0xb2, 0xbd, 0x79, 0x93, 0xa2, 0x4a, 0x35, 0x36, 0xc8, 0x7c, 0xdb, + 0xb0, 0x1d, 0xc1, 0xd0, 0x1d, 0x51, 0x9a, 0xe4, 0x19, 0xe6, 0x08, 0x00, 0x95, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x00, 0x40, 0x92, 0x9c, 0x10, 0x46, 0x00, + 0x00, 0x03, 0x60, 0x00, 0x00, 0x73, 0xaf, 0xff, 0xff, 0xa9, 0xef, 0xff, 0xff, 0xff, 0xd8, 0x59, + 0x9f, 0xff, 0x53, 0xff, 0xff, 0xff, 0xb4, 0x78, 0x24, 0x37, 0xff, 0xfb, 0x89, 0x98, 0x26, 0xc0, + 0xe3, 0x5f, 0x50, 0x5c, 0x2f, 0xff, 0xff, 0xf1, 0xd2, 0xd2, 0x4d, 0xfd, 0xf2, 0x7f, 0xff, 0x9f, + 0xff, 0x17, 0x85, 0xd2, 0xab, 0x4b, 0xbe, 0x3d, 0xff, 0xff, 0xff, 0xdf, 0x55, 0xbf, 0xff, 0xff, + 0xff, 0xfe, 0x6e, 0xe3, 0xe4, 0x07, 0x53, 0xd2, 0x5f, 0xff, 0xff, 0xff, 0xbe, 0xab, 0x7f, 0xff, + 0xff, 0xff, 0xcd, 0xdc, 0x1a, 0xbf, 0xff, 0xfe, 0xfa, 0xad, 0xff, 0xff, 0xff, 0xcd, 0xdc, 0x4e, + 0x1d, 0x57, 0xff, 0xff, 0xe0, 0x44, 0xaa, 0xa3, 0xff, 0xd3, 0x89, 0x13, 0x97, 0xff, 0xe5, 0xa2, + 0x55, 0x51, 0xff, 0xe9, 0xc4, 0xba, 0x29, 0xff, 0xeb, 0x71, 0x2a, 0xa8, 0xff, 0xee, 0xb9, 0xc4, + 0x7f, 0xff, 0xfb, 0xea, 0xb7, 0xff, 0xff, 0xff, 0x37, 0x71, 0x29, 0xfa, 0x00, +}; + +constexpr std::array Vp9Hashes{ + 0x552fd3c320b37408ull, 0xb2fb78bac8d2d7dfull, 0x312469345c491c3dull, + 0x9aeb0fb400c95a75ull, 0xe02e07646faa48beull, 0x1399072031abb375ull, +}; + +constexpr std::array HevcMain10Stream{ + 0x00, 0x00, 0x00, 0x01, 0x46, 0x01, 0x10, 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0c, 0x01, 0xff, + 0xff, 0x02, 0x20, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x1e, + 0x95, 0x98, 0x09, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x02, 0x20, 0x00, 0x00, 0x03, 0x00, + 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x1e, 0xa0, 0x20, 0x82, 0x93, 0x65, 0x95, 0x9a, + 0xe4, 0xca, 0xf0, 0x16, 0x80, 0x80, 0x00, 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x0f, 0x04, 0x00, + 0x00, 0x00, 0x01, 0x44, 0x01, 0xc1, 0x73, 0xd0, 0x89, 0x00, 0x00, 0x01, 0x28, 0x01, 0xaf, 0x08, + 0x60, 0xf2, 0xfb, 0x04, 0xef, 0x67, 0x67, 0xdc, 0xc7, 0x8e, 0xc7, 0x42, 0x63, 0xeb, 0x7d, 0x28, + 0x73, 0x95, 0x14, 0x0c, 0x71, 0x10, 0x8b, 0x65, 0x04, 0xbd, 0x03, 0xe6, 0x04, 0x35, 0xb9, 0x9b, + 0xec, 0x60, 0x2e, 0xea, 0xcf, 0xd0, 0xdd, 0x2d, 0x3a, 0x62, 0x33, 0x9d, 0x7a, 0x74, 0xb0, 0xeb, + 0x0c, 0x06, 0xe6, 0xa5, 0x30, 0xa7, 0xea, 0xed, 0xc2, 0xa5, 0x8e, 0x7c, 0xda, 0x1b, 0x9d, 0xe4, + 0xac, 0x2b, 0x38, 0xdb, 0x9c, 0xe8, 0x51, 0x34, 0x46, 0xd2, 0x04, 0x1a, 0x77, 0x87, 0x7a, 0x19, + 0x70, 0xf9, 0x4f, 0xcb, 0x03, 0x3c, 0x74, 0x25, 0xb9, 0x92, 0x3b, 0x95, 0xe9, 0xd8, 0xaa, 0x9b, + 0x6b, 0x9b, 0x1f, 0x70, 0x25, 0x2a, 0xe7, 0x6c, 0xc3, 0x7e, 0x55, 0x39, 0xf5, 0x06, 0x5a, 0x23, + 0xb1, 0x2e, 0xf6, 0x01, 0xac, 0x71, 0xb9, 0x20, 0xb4, 0xef, 0x1d, 0x42, 0x7e, 0x0e, 0x01, 0x1d, + 0xaf, 0x0a, 0x13, 0x04, 0x8e, 0x03, 0x4d, 0x1e, 0xc4, 0xe0, +}; + void check(bool condition, const std::string& message) { if (!condition) throw std::runtime_error(message); } @@ -290,6 +462,86 @@ void testFailures() { check(sceVideodec2DeleteDecoder_nid_postfix(handle) == 0, "delete failed"); } +std::vector> hevcUnits(const std::uint8_t* stream, std::size_t size) { + std::vector starts; + for (std::size_t index = 0; index + 4 < size; ++index) { + if (stream[index] == 0 && stream[index + 1] == 0 && stream[index + 2] == 0 && stream[index + 3] == 1 && ((stream[index + 4] >> 1u) & 0x3fu) == 35) starts.push_back(index); + } + starts.push_back(size); + std::vector> units; + for (std::size_t unit = 0; unit + 1 < starts.size(); ++unit) units.emplace_back(stream + starts[unit], stream + starts[unit + 1]); + return units; +} + +std::vector> ivfFrames() { + std::vector> frames; + for (std::size_t index = 32; index + 12 <= Vp9Stream.size();) { + const std::size_t length = Vp9Stream[index] | (Vp9Stream[index + 1] << 8u) | (Vp9Stream[index + 2] << 16u) | (static_cast(Vp9Stream[index + 3]) << 24u); + check(index + 12 + length <= Vp9Stream.size(), "truncated IVF frame"); + frames.emplace_back(Vp9Stream.begin() + static_cast(index + 12), Vp9Stream.begin() + static_cast(index + 12 + length)); + index += 12 + length; + } + return frames; +} + +void testClip(std::uint32_t codec, std::uint32_t profile, const std::vector>& units, const std::array& hashes) { + const DecoderConfigInfo config{sizeof(DecoderConfigInfo), 1, codec, profile, 0, static_cast(Width), static_cast(Height), 4, 1}; + DecoderMemoryInfo memory{sizeof(DecoderMemoryInfo)}; + check(sceVideodec2QueryDecoderMemoryInfo_nid_postfix(&config, &memory) == 0, "memory query failed"); + std::uint64_t handle = 0; + check(sceVideodec2CreateDecoder_nid_postfix(&config, &memory, &handle) == 0, "decoder creation failed"); + std::vector buffer(memory.maxFrameBufferSize); + std::size_t pictures = 0; + const auto take = [&](const OutputInfo& output, const FrameBuffer& frame) { + check(output.isValid == frame.isAccepted, "frame buffer acceptance disagrees with the output"); + if (!output.isValid) return false; + check(pictures < hashes.size(), "more pictures than access units"); + check(output.codecType == codec && !output.isErrorFrame && output.frameWidth == Width && output.frameHeight == Height && output.framePitch == 256 && output.frameBuffer == buffer.data(), "unexpected picture geometry"); + AvcPictureInfo info{sizeof(AvcPictureInfo)}; + checkThrows([&] { sceVideodec2GetPictureInfo_nid_postfix(&output, &info, nullptr); }); + check(hashNv12(buffer.data(), output.framePitch) == hashes[pictures], "picture " + std::to_string(pictures) + " differs from the reference"); + ++pictures; + return true; + }; + check(units.size() == hashes.size(), "unexpected access unit count"); + for (std::size_t unit = 0; unit < units.size(); ++unit) { + const InputData input{sizeof(InputData), units[unit].data(), units[unit].size(), 1000 + unit, unit, 0}; + FrameBuffer frame{sizeof(FrameBuffer), buffer.data(), buffer.size(), false}; + OutputInfo output{sizeof(OutputInfo)}; + check(sceVideodec2Decode_nid_postfix(handle, &input, &frame, &output) == 0, "decode failed"); + take(output, frame); + } + for (;;) { + FrameBuffer frame{sizeof(FrameBuffer), buffer.data(), buffer.size(), false}; + OutputInfo output{sizeof(OutputInfo)}; + check(sceVideodec2Flush_nid_postfix(handle, &frame, &output) == 0, "flush failed"); + if (!take(output, frame)) break; + } + check(pictures == hashes.size(), "missing pictures after flush"); + check(sceVideodec2DeleteDecoder_nid_postfix(handle) == 0, "delete failed"); +} + +void testTenBit() { + for (const auto codec : {CodecHevc, CodecVp9}) { + const DecoderConfigInfo config{sizeof(DecoderConfigInfo), 1, codec, 2, 0, static_cast(Width), static_cast(Height), 4, 1}; + DecoderMemoryInfo memory{sizeof(DecoderMemoryInfo)}; + check(sceVideodec2QueryDecoderMemoryInfo_nid_postfix(&config, &memory) == 0, "memory query failed"); + std::uint64_t handle = 0; + check(sceVideodec2CreateDecoder_nid_postfix(&config, &memory, &handle) == 0, "ten-bit decoder creation failed"); + if (codec == CodecHevc) { + std::vector buffer(memory.maxFrameBufferSize); + const InputData input{sizeof(InputData), HevcMain10Stream.data(), HevcMain10Stream.size(), 0, 0, 0}; + FrameBuffer frame{sizeof(FrameBuffer), buffer.data(), buffer.size(), false}; + OutputInfo output{sizeof(OutputInfo)}; + checkThrows([&] { + sceVideodec2Decode_nid_postfix(handle, &input, &frame, &output); + sceVideodec2Flush_nid_postfix(handle, &frame, &output); + }); + } + check(sceVideodec2DeleteDecoder_nid_postfix(handle) == 0, "delete failed"); + } +} + void testDecode(bool lengthPrefixed) { const DecoderConfigInfo config{sizeof(DecoderConfigInfo), 0, 1, 100, 0, static_cast(Width), static_cast(Height), 4, 1}; DecoderMemoryInfo memory{sizeof(DecoderMemoryInfo)}; @@ -343,6 +595,30 @@ int main() { testFailures(); testDecode(false); testDecode(true); + const auto hevc = hevcUnits(HevcStream.data(), HevcStream.size()); + testClip(CodecHevc, 1, hevc, HevcHashes); + std::vector> hevcLengthPrefixed; + for (const auto& unit : hevc) { + std::vector converted; + std::vector nals; + for (std::size_t index = 0; index + 3 < unit.size(); ++index) { + if (unit[index] == 0 && unit[index + 1] == 0 && unit[index + 2] == 1) { + nals.push_back(index + 3); + index += 2; + } + } + for (std::size_t nal = 0; nal < nals.size(); ++nal) { + auto end = nal + 1 < nals.size() ? nals[nal + 1] - 3 : unit.size(); + while (end > nals[nal] && unit[end - 1] == 0) --end; + const auto length = end - nals[nal]; + for (int shift = 24; shift >= 0; shift -= 8) converted.push_back(static_cast(length >> shift)); + converted.insert(converted.end(), unit.begin() + static_cast(nals[nal]), unit.begin() + static_cast(end)); + } + hevcLengthPrefixed.push_back(std::move(converted)); + } + testClip(CodecHevc, 1, hevcLengthPrefixed, HevcHashes); + testClip(CodecVp9, 0, ivfFrames(), Vp9Hashes); + testTenBit(); std::puts("Videodec2 tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/tests/GuestVoiceQoS.cpp b/core/libs/tests/GuestVoiceQoS.cpp index 552a0bf73..bf8b3467c 100644 --- a/core/libs/tests/GuestVoiceQoS.cpp +++ b/core/libs/tests/GuestVoiceQoS.cpp @@ -1,7 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include #include -#include #include extern "C" { @@ -10,24 +9,20 @@ int APS5_VABI sceVoiceQoSInit(void*, std::uint32_t, std::int32_t); static void Require(bool value) { if (!value) std::abort(); } -template -static bool Throws(TFunction function) { - try { - function(); - } catch (const TException&) { - return true; - } - return false; -} - int main(int argc, char** argv) { Require(argc == 2); + constexpr int argumentInvalid = static_cast(0x804E0902); + constexpr int initialized = static_cast(0x804E0905); const auto appType = static_cast(std::strtoul(argv[1], nullptr, 16)); - std::vector memory(0x40000); - Require(Throws([&] { sceVoiceQoSInit(nullptr, 0x40000, appType); })); - Require(Throws([&] { sceVoiceQoSInit(memory.data(), 0, appType); })); - Require(Throws([&] { sceVoiceQoSInit(memory.data(), 0x40000, 0); })); - Require(Throws([&] { sceVoiceQoSInit(memory.data(), 0x40000, 0x30000000); })); + std::vector memory(0x40000, 0xAA); + Require(sceVoiceQoSInit(nullptr, 0x40000, appType) == argumentInvalid); + Require(sceVoiceQoSInit(nullptr, 0, appType) == argumentInvalid); + for (std::uint32_t size : {0u, 1u, 0x100u, 0x3FFFFu, 0x40001u, 0x80000u, 0xFFFFFFFFu}) Require(sceVoiceQoSInit(memory.data(), size, appType) == argumentInvalid); + for (std::int32_t type : {0, 1, 0x08000000, 0x20000001, 0x30000000, 0x40000000, static_cast(0x80000000), -1}) { + Require(sceVoiceQoSInit(memory.data(), 0x40000, type) == argumentInvalid); + } + for (std::uint8_t byte : memory) Require(byte == 0xAA); Require(sceVoiceQoSInit(memory.data(), 0x40000, appType) == 0); - Require(Throws([&] { sceVoiceQoSInit(memory.data(), 0x40000, appType); })); + Require(sceVoiceQoSInit(memory.data(), 0x40000, appType) == initialized); + Require(sceVoiceQoSInit(nullptr, 0, 0) == initialized); } diff --git a/core/libs/tests/GuestWideStrings.cpp b/core/libs/tests/GuestWideStrings.cpp new file mode 100644 index 000000000..e08349ab4 --- /dev/null +++ b/core/libs/tests/GuestWideStrings.cpp @@ -0,0 +1,146 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include + +extern "C" { +const char16_t* APS5_VABI wmemchr_nid_postfix(const char16_t* s, char16_t c, std::size_t n); +int APS5_VABI wmemcmp_nid_postfix(const char16_t* s1, const char16_t* s2, std::size_t n); +char16_t* APS5_VABI wmemcpy_nid_postfix(char16_t* dest, const char16_t* src, std::size_t n); +char16_t* APS5_VABI wmemmove_nid_postfix(char16_t* dest, const char16_t* src, std::size_t n); +std::size_t APS5_VABI wcslen_nid_postfix(const char16_t* s); +int APS5_VABI wcscmp_nid_postfix(const char16_t* s1, const char16_t* s2); +int APS5_VABI wcsncmp_nid_postfix(const char16_t* s1, const char16_t* s2, std::size_t n); +char16_t* APS5_VABI wcscpy_nid_postfix(char16_t* dest, const char16_t* src); +char16_t* APS5_VABI wcsncpy_nid_postfix(char16_t* dest, const char16_t* src, std::size_t n); +const char16_t* APS5_VABI wcschr_nid_postfix(const char16_t* s, char16_t c); +const char16_t* APS5_VABI wcsrchr_nid_postfix(const char16_t* s, char16_t c); +const char16_t* APS5_VABI wcsstr_nid_postfix(const char16_t* haystack, const char16_t* needle); +const char16_t* APS5_VABI wcspbrk_nid_postfix(const char16_t* s, const char16_t* accept); +std::size_t APS5_VABI wcsspn_nid_postfix(const char16_t* s, const char16_t* accept); +char16_t* APS5_VABI wmemset_nid_postfix(char16_t* s, char16_t c, std::size_t n); +double APS5_VABI wcstod_nid_postfix(const char16_t* str, char16_t** endptr); +float APS5_VABI wcstof_nid_postfix(const char16_t* str, char16_t** endptr); +long double APS5_VABI wcstold_nid_postfix(const char16_t* str, char16_t** endptr); +long long APS5_VABI wcstol_nid_postfix(const char16_t* str, char16_t** endptr, int base); +long long APS5_VABI wcstoll_nid_postfix(const char16_t* str, char16_t** endptr, int base); +unsigned long long APS5_VABI wcstoul_nid_postfix(const char16_t* str, char16_t** endptr, int base); +unsigned long long APS5_VABI wcstoull_nid_postfix(const char16_t* str, char16_t** endptr, int base); +int APS5_VABI wcscoll_nid_postfix(const char16_t* first, const char16_t* second); +std::size_t APS5_VABI wcsxfrm_nid_postfix(char16_t* destination, const char16_t* source, std::size_t count); +} + +namespace { + +void require(bool condition) { + if (!condition) std::abort(); +} + +bool same(const char16_t* left, const char16_t* right, std::size_t count) { + for (std::size_t index = 0; index < count; ++index) { + if (left[index] != right[index]) return false; + } + return true; +} + +} + +int main() { + const char16_t split[] = {u'a', u'b', 0, u'c', 0}; + require(wcslen_nid_postfix(u"") == 0); + require(wcslen_nid_postfix(u"héllo") == 5); + require(wcslen_nid_postfix(split) == 2); + + require(wcscmp_nid_postfix(u"abc", u"abc") == 0); + require(wcscmp_nid_postfix(u"abc", u"abd") < 0); + require(wcscmp_nid_postfix(u"abd", u"abc") > 0); + require(wcscmp_nid_postfix(u"ab", u"abc") < 0); + require(wcscmp_nid_postfix(u"￿", u"a") > 0); + require(wcscmp_nid_postfix(u"耀", u"翿") > 0); + require(wcsncmp_nid_postfix(u"abcx", u"abcy", 3) == 0); + require(wcsncmp_nid_postfix(u"abcx", u"abcy", 4) < 0); + require(wcsncmp_nid_postfix(u"ab", u"ab", 10) == 0); + require(wcsncmp_nid_postfix(u"a", u"b", 0) == 0); + + char16_t buffer[8]; + wmemset_nid_postfix(buffer, 0xaaaa, 8); + require(wcscpy_nid_postfix(buffer, u"wide") == buffer); + require(same(buffer, u"wide", 5) && buffer[5] == 0xaaaa); + + wmemset_nid_postfix(buffer, 0xaaaa, 8); + require(wcsncpy_nid_postfix(buffer, u"ab", 5) == buffer); + const char16_t padded[] = {u'a', u'b', 0, 0, 0, 0xaaaa}; + require(same(buffer, padded, 6)); + wmemset_nid_postfix(buffer, 0xaaaa, 8); + wcsncpy_nid_postfix(buffer, u"abcdef", 3); + const char16_t truncated[] = {u'a', u'b', u'c', 0xaaaa}; + require(same(buffer, truncated, 4)); + + const char16_t* text = u"a世b世c"; + require(wcschr_nid_postfix(text, u'世') == text + 1); + require(wcsrchr_nid_postfix(text, u'世') == text + 3); + require(wcschr_nid_postfix(text, u'z') == nullptr); + require(wcsrchr_nid_postfix(text, u'z') == nullptr); + require(wcschr_nid_postfix(text, 0) == text + 5); + require(wcsrchr_nid_postfix(text, 0) == text + 5); + + const char16_t* haystack = u"one two two"; + require(wcsstr_nid_postfix(haystack, u"two") == haystack + 4); + require(wcsstr_nid_postfix(haystack, u"") == haystack); + require(wcsstr_nid_postfix(haystack, u"three") == nullptr); + require(wcsstr_nid_postfix(u"tw", u"two") == nullptr); + + require(wcspbrk_nid_postfix(haystack, u"wt") == haystack + 4); + require(wcspbrk_nid_postfix(haystack, u"xyz") == nullptr); + require(wcsspn_nid_postfix(u"aabbc", u"ab") == 4); + require(wcsspn_nid_postfix(u"abc", u"") == 0); + + const char16_t units[] = {u'x', 0, u'￿', u'y'}; + require(wmemchr_nid_postfix(units, u'￿', 4) == units + 2); + require(wmemchr_nid_postfix(units, u'y', 3) == nullptr); + const char16_t lower[] = {u'x', 0, u'\u0001', u'y'}; + require(wmemcmp_nid_postfix(units, units, 4) == 0); + require(wmemcmp_nid_postfix(units, lower, 4) > 0); + require(wmemcmp_nid_postfix(lower, units, 4) < 0); + require(wmemcmp_nid_postfix(units, lower, 2) == 0); + + char16_t copy[4] = {}; + require(wmemcpy_nid_postfix(copy, units, 4) == copy && same(copy, units, 4)); + char16_t overlap[] = {u'1', u'2', u'3', u'4', u'5'}; + require(wmemmove_nid_postfix(overlap + 1, overlap, 4) == overlap + 1); + const char16_t shifted[] = {u'1', u'1', u'2', u'3', u'4'}; + require(same(overlap, shifted, 5)); + require(wmemset_nid_postfix(copy, u'世', 3) == copy && copy[0] == u'世' && copy[2] == u'世' && copy[3] == u'y'); + + char16_t* end = nullptr; + const char16_t* negative = u" -42xyz"; + require(wcstol_nid_postfix(negative, &end, 10) == -42 && end == negative + 5); + const char16_t* hex = u"0x1F!"; + require(wcstoll_nid_postfix(hex, &end, 16) == 31 && end == hex + 4); + require(wcstoll_nid_postfix(hex, &end, 0) == 31 && end == hex + 4); + const char16_t* wideSpace = u"　12"; + require(wcstoul_nid_postfix(wideSpace, &end, 10) == 0 && end == wideSpace); + const char16_t* wideDigit = u"12١"; + require(wcstoll_nid_postfix(wideDigit, &end, 10) == 12 && end == wideDigit + 2); + require(wcstoull_nid_postfix(u"18446744073709551615", nullptr, 10) == ULLONG_MAX); + const char16_t* letters = u"abc"; + require(wcstol_nid_postfix(letters, &end, 10) == 0 && end == letters); + + const char16_t* scientific = u"3.5e2!"; + require(wcstod_nid_postfix(scientific, &end) == 350.0 && end == scientific + 5); + require(wcstod_nid_postfix(letters, &end) == 0.0 && end == letters); + require(wcstof_nid_postfix(u"0.25", nullptr) == 0.25f); + const char16_t* half = u"-1.5é"; + require(wcstold_nid_postfix(half, &end) == -1.5L && end == half + 4); + + require(wcscoll_nid_postfix(u"a", u"b") < 0); + require(wcscoll_nid_postfix(u"￿", u"a") > 0); + require(wcscoll_nid_postfix(u"same", u"same") == 0); + + char16_t transformed[8]; + wmemset_nid_postfix(transformed, 0xaaaa, 8); + require(wcsxfrm_nid_postfix(transformed, u"wide", 8) == 4 && same(transformed, u"wide", 5) && transformed[5] == 0xaaaa); + require(wcsxfrm_nid_postfix(transformed, u"much too long", 4) == 13); + require(wcsxfrm_nid_postfix(nullptr, u"abc", 0) == 3); + return 0; +} diff --git a/core/libs/tests/ImageBy.cpp b/core/libs/tests/ImageBy.cpp new file mode 100644 index 000000000..c636525f0 --- /dev/null +++ b/core/libs/tests/ImageBy.cpp @@ -0,0 +1,110 @@ +#include "Translation/TranslationContext.hpp" +#include "RdnaDecoder/RdnaImageOpDecoder.hpp" +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +void Require(bool value, std::source_location location = std::source_location::current()) { + if (!value) throw std::runtime_error("image BY2/BY4 regression at line " + std::to_string(location.line())); +} + +constexpr std::array Encodings{0x42u, 0x43u, 0x4au, 0x4bu, 0x52u, 0x53u, 0x5au, 0x5bu}; +constexpr std::array Opcodes{RdnaOpcode::ImageLoadBy2, RdnaOpcode::ImageLoadBy4, RdnaOpcode::ImageLoadMipBy2, RdnaOpcode::ImageLoadMipBy4, RdnaOpcode::ImageStoreBy2, RdnaOpcode::ImageStoreBy4, RdnaOpcode::ImageStoreMipBy2, RdnaOpcode::ImageStoreMipBy4}; + +RdnaInstruction Decode(std::uint32_t encoding, std::uint32_t controls = 0u, std::uint32_t word1 = 0x00101e1eu, std::uint32_t mask = 15u, std::uint32_t dimension = 1u) { + const std::array code{0xf0000000u | (dimension << 3u) | (encoding << 18u) | (mask << 8u) | controls, word1, 0x0000201fu}; + return DecodeRdnaMimg(0u, code, 0u); +} + +void Check(std::uint32_t index, bool nsa, std::uint32_t mask) { + const auto encoding = Encodings[index]; + const auto inst = Decode(encoding, nsa ? 2u : 0u, 0x00101e1eu, mask); + const auto elements = (encoding & 1u) != 0u ? 4u : 2u; + const auto dwords = static_cast(std::popcount(mask)); + const bool store = (encoding & 0x10u) != 0u; + Require(inst.op == Opcodes[index] && IsImageOpcode(inst.op)); + Require(inst.dataDwordCount == dwords && inst.dataComponents == dwords); + Require(inst.imageAddressComponents == ((encoding & 8u) != 0u ? 3u : 2u)); + Require(inst.wordCount == (nsa ? 3u : 2u)); + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + TranslationContext context(program, block, 256); + if (store) { + bool refused = false; + try { context.TranslateInstruction(inst); } + catch (const std::runtime_error& error) { refused = std::string(error.what()).find("stores are not implemented") != std::string::npos; } + Require(refused); + return; + } + context.TranslateInstruction(inst); + std::vector reads; + std::uint32_t writes = 0; + for (auto* value : block.Instructions()) { + if (value->Opcode() == IrOpcode::SetVectorRegister) ++writes; + if (value->Opcode() == IrOpcode::ImageRead) { + Require(writes == 0u); + reads.push_back(value); + } + } + Require(reads.size() == 1u && writes == dwords); + const auto& memory = program.Resources().memoryInfo[reads[0]->Flags().index]; + Require(memory.imageByElements == elements && memory.dmask == mask && memory.dataDwords == dwords); + Require(memory.imageHasMip == ((encoding & 8u) != 0u)); +} + +void CheckOrdinary(std::uint32_t encoding) { + const auto inst = Decode(encoding); + const bool store = (encoding & 8u) != 0u; + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + TranslationContext context(program, block, 256); + context.TranslateInstruction(inst); + std::uint32_t operations = 0; + for (auto* value : block.Instructions()) { + if (value->Opcode() == (store ? IrOpcode::ImageWrite : IrOpcode::ImageRead)) { + const auto& memory = program.Resources().memoryInfo[value->Flags().index]; + Require(memory.imageByElements == 0u && memory.dmask == 15u && memory.dataDwords == 4u); + ++operations; + } + } + Require(operations == 1u); +} + +void Refused(std::uint32_t encoding, std::uint32_t control, std::uint32_t word1 = 0x00101e1eu, std::uint32_t mask = 15u, std::uint32_t dimension = 1u) { + bool refused = false; + try { (void)Decode(encoding, control, word1, mask, dimension); } + catch (const std::runtime_error&) { refused = true; } + Require(refused); +} + +} + +int main() { + for (auto encoding : {0u, 1u, 8u, 9u}) CheckOrdinary(encoding); + for (std::uint32_t index = 0; index < Encodings.size(); ++index) { + const bool by2 = (Encodings[index] & 1u) == 0u; + for (const bool nsa : {false, true}) { + Check(index, nsa, 15u); + if (by2) Check(index, nsa, 3u); + } + for (std::uint32_t mask = 0u; mask < 15u; ++mask) { + if (!(by2 && mask == 3u)) Refused(Encodings[index], 0u, 0x00101e1eu, mask); + } + for (auto bits : {0x8000u, 0x10000u, 0x20000u}) Refused(Encodings[index], bits); + for (auto bits : {0x40000000u, 0x80000000u}) Refused(Encodings[index], 0u, 0x00101e1eu | bits); + Refused(Encodings[index], 0u, 0x0010fd1eu); + Refused(Encodings[index], 0u, 0x00101effu); + for (std::uint32_t dimension = 0; dimension < 8u; ++dimension) { + if (dimension != 1u) Refused(Encodings[index], 0u, 0x00101e1eu, 15u, dimension); + } + } +} diff --git a/core/libs/tests/ImagePckN.cpp b/core/libs/tests/ImagePckN.cpp new file mode 100644 index 000000000..11898091d --- /dev/null +++ b/core/libs/tests/ImagePckN.cpp @@ -0,0 +1,106 @@ +#include "Translation/TranslationContext.hpp" +#include "RdnaDecoder/RdnaImageOpDecoder.hpp" +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +void Require(bool value, std::source_location location = std::source_location::current()) { + if (!value) throw std::runtime_error("image PCK2/PCK4 regression at line " + std::to_string(location.line())); +} + +constexpr std::array Encodings{0x70u, 0x71u, 0x73u, 0x74u, 0x76u, 0x77u, 0x79u, 0x7au}; +constexpr std::array Opcodes{RdnaOpcode::ImageLoadPck2, RdnaOpcode::ImageLoadPck4, RdnaOpcode::ImageLoadMipPck2, RdnaOpcode::ImageLoadMipPck4, RdnaOpcode::ImageStorePck2, RdnaOpcode::ImageStorePck4, RdnaOpcode::ImageStoreMipPck2, RdnaOpcode::ImageStoreMipPck4}; + +RdnaInstruction Decode(std::uint32_t encoding, std::uint32_t controls = 0u, std::uint32_t word1 = 0x00101e1eu, std::uint32_t mask = 15u, std::uint32_t dimension = 1u) { + const std::array code{0xf0000000u | (dimension << 3u) | (encoding << 18u) | (mask << 8u) | controls, word1, 0x0000201fu}; + return DecodeRdnaMimg(0u, code, 0u); +} + +void Check(std::uint32_t index, bool nsa, std::uint32_t mask) { + const auto encoding = Encodings[index]; + const auto inst = Decode(encoding, nsa ? 2u : 0u, 0x00101e1eu, mask); + const auto elements = encoding == 0x71u || encoding == 0x74u || encoding == 0x77u || encoding == 0x7au ? 4u : 2u; + const bool mip = encoding == 0x73u || encoding == 0x74u || encoding == 0x79u || encoding == 0x7au; + const auto dwords = static_cast(std::popcount(mask)); + const bool store = encoding >= 0x76u; + Require(inst.op == Opcodes[index] && IsImageOpcode(inst.op)); + Require(inst.dataDwordCount == dwords && inst.dataComponents == dwords); + Require(inst.imageAddressComponents == (mip ? 3u : 2u)); + Require(inst.wordCount == (nsa ? 3u : 2u)); + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + TranslationContext context(program, block, 256); + if (store) { + bool refused = false; + try { context.TranslateInstruction(inst); } + catch (const std::runtime_error& error) { refused = std::string(error.what()).find("stores are not implemented") != std::string::npos; } + Require(refused); + return; + } + context.TranslateInstruction(inst); + std::vector reads; + std::uint32_t writes = 0; + for (auto* value : block.Instructions()) { + if (value->Opcode() == IrOpcode::SetVectorRegister) ++writes; + if (value->Opcode() == IrOpcode::ImageRead) { + Require(writes == 0u); + reads.push_back(value); + } + } + Require(reads.size() == 1u && writes == dwords); + const auto& memory = program.Resources().memoryInfo[reads[0]->Flags().index]; + Require(memory.imageByElements == elements && memory.imagePacked && memory.dmask == mask && memory.dataDwords == dwords); + Require(memory.imageHasMip == mip); +} + +void CheckOrdinary(std::uint32_t encoding) { + const auto inst = Decode(encoding); + const bool store = (encoding & 8u) != 0u; + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + TranslationContext context(program, block, 256); + context.TranslateInstruction(inst); + std::uint32_t operations = 0; + for (auto* value : block.Instructions()) { + if (value->Opcode() == (store ? IrOpcode::ImageWrite : IrOpcode::ImageRead)) { + const auto& memory = program.Resources().memoryInfo[value->Flags().index]; + Require(memory.imageByElements == 0u && memory.dmask == 15u && memory.dataDwords == 4u); + ++operations; + } + } + Require(operations == 1u); +} + +void Refused(std::uint32_t encoding, std::uint32_t control, std::uint32_t word1 = 0x00101e1eu, std::uint32_t mask = 15u, std::uint32_t dimension = 1u) { + bool refused = false; + try { (void)Decode(encoding, control, word1, mask, dimension); } + catch (const std::runtime_error&) { refused = true; } + Require(refused); +} + +} + +int main() { + for (auto encoding : {0u, 1u, 8u, 9u}) CheckOrdinary(encoding); + for (std::uint32_t index = 0; index < Encodings.size(); ++index) { + for (const bool nsa : {false, true}) Check(index, nsa, 1u); + for (std::uint32_t mask = 0u; mask < 16u; ++mask) { + if (mask != 1u) Refused(Encodings[index], 0u, 0x00101e1eu, mask); + } + for (auto bits : {0x8000u, 0x10000u, 0x20000u}) Refused(Encodings[index], bits, 0x00101e1eu, 1u); + for (auto bits : {0x40000000u, 0x80000000u}) Refused(Encodings[index], 0u, 0x00101e1eu | bits, 1u); + Refused(Encodings[index], 0u, 0x00101effu, 1u); + for (std::uint32_t dimension = 0; dimension < 8u; ++dimension) { + if (dimension != 1u) Refused(Encodings[index], 0u, 0x00101e1eu, 1u, dimension); + } + } +} diff --git a/core/libs/tests/ImageSampleGather.cpp b/core/libs/tests/ImageSampleGather.cpp index 7d0db7266..5332d836a 100644 --- a/core/libs/tests/ImageSampleGather.cpp +++ b/core/libs/tests/ImageSampleGather.cpp @@ -17,6 +17,26 @@ static void Check(std::uint32_t encoding, RdnaOpcode opcode, std::uint32_t dmask TranslationContext context(program, block, 256); context.TranslateInstruction(instruction); } +static void CheckPck(std::uint32_t encoding, RdnaOpcode opcode, std::uint32_t dmask, std::uint32_t dataComponents) { + const std::array code{(0x3cu << 26u) | (encoding << 18u) | (dmask << 8u) | (1u << 3u), 0u}; + const RdnaInstruction instruction = DecodeRdnaMimg(0u, code, 0u); + Require(instruction.op == opcode); + Require(instruction.imageOpcodeId == encoding); + Require(instruction.imageDmask == dmask); + Require(instruction.dataComponents == dataComponents); + Require(instruction.dataDwordCount == dataComponents); + Require(IsImageOpcode(instruction.op)); +} +static void CheckPckRejectsZeroMask(std::uint32_t encoding) { + const std::array code{(0x3cu << 26u) | (encoding << 18u) | (1u << 3u), 0u}; + bool rejected = false; + try { + DecodeRdnaMimg(0u, code, 0u); + } catch (const std::runtime_error&) { + rejected = true; + } + Require(rejected); +} int main() { Check(0x26u, RdnaOpcode::ImageSampleBCl, 0xfu); Check(0x36u, RdnaOpcode::ImageSampleBClO, 0xfu); @@ -61,4 +81,11 @@ int main() { Check(0x44u, RdnaOpcode::ImageGather4L, 0x8u); Check(0x54u, RdnaOpcode::ImageGather4LO, 0x8u); Check(0x50u, RdnaOpcode::ImageGather4O, 0x8u); + CheckPck(0x62u, RdnaOpcode::ImageGather4hPck, 0x1u, 1u); + CheckPck(0x62u, RdnaOpcode::ImageGather4hPck, 0x3u, 2u); + CheckPck(0x62u, RdnaOpcode::ImageGather4hPck, 0xFu, 4u); + CheckPck(0x63u, RdnaOpcode::ImageGather8hPck, 0x1u, 1u); + CheckPck(0x63u, RdnaOpcode::ImageGather8hPck, 0xFu, 4u); + CheckPckRejectsZeroMask(0x62u); + CheckPckRejectsZeroMask(0x63u); } diff --git a/core/libs/tests/Mspace.cpp b/core/libs/tests/Mspace.cpp index cd712b92f..2d61f1c56 100644 --- a/core/libs/tests/Mspace.cpp +++ b/core/libs/tests/Mspace.cpp @@ -8,6 +8,7 @@ #include extern "C" { +int* APS5_VABI __error_nid_postfix(); void* APS5_VABI sceLibcMspaceCreate_nid_postfix(const char*, void*, std::size_t, unsigned); int APS5_VABI sceLibcMspaceDestroy_nid_postfix(void*); void* APS5_VABI sceLibcMspaceMalloc_nid_postfix(void*, std::size_t); @@ -42,6 +43,23 @@ int main() { Require(first > storage.data() && first + 128 <= storage.data() + storage.size()); for (int i = 0; i < 128; ++i) { Require(first[i] == 0); first[i] = static_cast(i); } void* blocker = sceLibcMspaceMalloc_nid_postfix(arena, 128); + MallocManagedSize beforeOverflow{sizeof(MallocManagedSize), 1, 0, 0, 0, 0, 0}; + Require(sceLibcMspaceMallocStats_nid_postfix(arena, &beforeOverflow) == 0); + for (std::size_t delta = 0; delta < 16; ++delta) { + const auto size = std::numeric_limits::max() - delta; + *__error_nid_postfix() = 0; + Require(sceLibcMspaceRealloc_nid_postfix(arena, first, size) == nullptr); + Require(*__error_nid_postfix() == 12); + *__error_nid_postfix() = 0; + Require(sceLibcMspaceReallocalign_nid_postfix(arena, first, size, 16) == nullptr); + Require(*__error_nid_postfix() == 12); + Require(sceLibcMspaceMallocUsableSize_nid_postfix(first) == 128); + for (int i = 0; i < 128; ++i) Require(first[i] == i); + MallocManagedSize afterOverflow{sizeof(MallocManagedSize), 1, 0, 0, 0, 0, 0}; + Require(sceLibcMspaceMallocStats_nid_postfix(arena, &afterOverflow) == 0); + Require(afterOverflow.currentInuseSize == beforeOverflow.currentInuseSize); + Require(afterOverflow.maxInuseSize == beforeOverflow.maxInuseSize); + } auto* grown = static_cast(sceLibcMspaceRealloc_nid_postfix(arena, first, 4096)); Require(grown && grown != first); for (int i = 0; i < 128; ++i) Require(grown[i] == i); diff --git a/core/libs/tests/ParamJsonParser.cpp b/core/libs/tests/ParamJsonParser.cpp index 88684685d..653fb6745 100644 --- a/core/libs/tests/ParamJsonParser.cpp +++ b/core/libs/tests/ParamJsonParser.cpp @@ -33,6 +33,22 @@ static ParsedParamJson ParseText(const std::filesystem::path& path, const std::s return parseParamJson(path); } +static void CheckUserDefinedParams(const std::filesystem::path& path) { + const auto parsed = ParseText(path, R"({"titleId":"PPSA23566","localizedParameters":{"en-US":{"titleName":"Example"}},)" + R"("userDefinedParam1":23566,"userDefinedParam3":-7,"userDefinedParam4":2147483647})"); + Require(parsed.userDefinedParams[0] == 23566 && parsed.userDefinedParams[1] == 0 && parsed.userDefinedParams[2] == -7 && + parsed.userDefinedParams[3] == 2147483647, "Incorrect user defined params"); + for (const auto* text : {"2147483648", "1.5", "1e3", "\"1\"", "true"}) { + bool rejected = false; + try { + ParseText(path, std::string(R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}},"userDefinedParam2":)") + text + "}"); + } catch (const std::exception&) { + rejected = true; + } + Require(rejected, std::string("Accepted invalid user defined param: ") + text); + } +} + static void CheckLanguages(const std::filesystem::path& path) { const auto nested = ParseText(path, "{\r\n \"localizedParameters\": {\r\n \"defaultLanguage\": \"en-GB\",\r\n" " \"en-GB\": { \"titleName\": \"British\" }\r\n },\r\n \"titleId\": \"PPSA00001\"\r\n}\r\n"); @@ -55,17 +71,37 @@ static void CheckLanguages(const std::filesystem::path& path) { Require(rejected, "Accepted localizedParameters without a language entry"); } +static void CheckNumberSyntax(const std::filesystem::path& path) { + const std::string prefix = R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}})"; + const std::pair fields[] = { + {R"(,"number":)", "}"}, {R"(,"values":[)", "]}"}, {R"(,"values":[{"number":)", "}]}"} + }; + for (const auto& [before, after] : fields) { + for (const auto* text : {"0", "-0", "0.25", "-0.25", "0e3", "0E+3", "-0e-3", + "10", "-10", "1e3", "1.25", "1e01", "1e+003", "1e-003"}) { + const auto parsed = ParseText(path, prefix + before + text + after); + Require(parsed.titleId == "PPSA00000" && parsed.title == "Example", "Title metadata changed"); + } + for (const auto* text : {"00", "01", "-00", "-01", "00.25", "01.25", "-01.25", "00e3", "01e3", "-01e3"}) { + bool rejected = false; + try { ParseText(path, prefix + before + text + after); } catch (const std::exception&) { rejected = true; } + Require(rejected, std::string("Accepted leading-zero number: ") + text); + } + } +} + int main() { const auto path = std::filesystem::temp_directory_path() / ("anyps5-param-json-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + ".json"); int result = 0; try { + CheckNumberSyntax(path); const std::pair sizes[] = { - {"", 0}, {"0", 0}, {"-0", 0}, {"4096", 4096}, + {"", 0}, {"0", 0}, {"-0", 0}, {"0.25", 0}, {"0e3", 0}, {"0E+3", 0}, {"-0e-3", 0}, {"4096", 4096}, {"9007199254740993", 9007199254740993ull}, {"18446744073709551614", std::numeric_limits::max() - 1}, {"18446744073709551615", std::numeric_limits::max()}, - {"1e3", 1000}, {"1.25", 1} + {"1e3", 1000}, {"1.25", 1}, {"1e01", 10}, {"1e+003", 1000}, {"1e-003", 0} }; for (const auto& [text, expected] : sizes) { const auto parsed = ParseSize(path, text); @@ -73,12 +109,13 @@ int main() { Require(parsed.titleId == "PPSA00000" && parsed.title == "Example", "Title metadata changed"); } for (const auto* text : {"18446744073709551616", "18446744073709551617", "1.8446744073709552e19", - "1e40", "-1", "-0.5", "true", "null", "\"4096\""}) { + "1e40", "-1", "-0.5", "true", "null", "\"4096\"", "00", "01", "00.25", "01e3", "01.25"}) { bool rejected = false; try { ParseSize(path, text); } catch (const std::exception&) { rejected = true; } Require(rejected, std::string("Accepted invalid download size: ") + text); } CheckLanguages(path); + CheckUserDefinedParams(path); } catch (const std::exception& error) { std::cerr << error.what() << '\n'; result = 1; diff --git a/core/libs/tests/PatchedLibraries.cpp b/core/libs/tests/PatchedLibraries.cpp new file mode 100644 index 000000000..ec0b43273 --- /dev/null +++ b/core/libs/tests/PatchedLibraries.cpp @@ -0,0 +1,36 @@ +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +int main(int argc, char** argv) { + if (argc != 2) { + std::fprintf(stderr, "usage: %s \n", argv[0]); + return 2; + } + int loaded = 0; + int failed = 0; + for (const auto& entry : std::filesystem::directory_iterator(argv[1])) { + if (entry.path().extension() != ".prx") continue; +#ifdef _WIN32 + if (LoadLibraryExW(entry.path().c_str(), nullptr, LOAD_WITH_ALTERED_SEARCH_PATH) != nullptr) { + ++loaded; + continue; + } + std::fprintf(stderr, "%s: LoadLibraryEx failed with error %lu\n", entry.path().filename().string().c_str(), GetLastError()); +#else + if (dlopen(entry.path().c_str(), RTLD_NOW | RTLD_LOCAL) != nullptr) { + ++loaded; + continue; + } + std::fprintf(stderr, "%s\n", dlerror()); +#endif + ++failed; + } + std::printf("%d patched libraries loaded, %d failed\n", loaded, failed); + return loaded > 0 && failed == 0 ? 0 : 1; +} diff --git a/core/libs/tests/SaveDataMemoryMetadata.cpp b/core/libs/tests/SaveDataMemoryMetadata.cpp index 5c75de021..70724bd18 100644 --- a/core/libs/tests/SaveDataMemoryMetadata.cpp +++ b/core/libs/tests/SaveDataMemoryMetadata.cpp @@ -1,12 +1,19 @@ #include "SceTypes.hpp" +#include +#include #include #include #include #include #include #include +#include #include #include +#ifndef _WIN32 +#include +#include +#endif extern "C" { int APS5_VABI sceSaveDataInitialize3(const void*); @@ -14,9 +21,13 @@ int APS5_VABI sceSaveDataTerminate(); int APS5_VABI sceSaveDataSetSaveDataMemory2(const SaveDataMemorySet2*); } -static void Require(bool value) { - if (!value) std::abort(); +static void Check(bool value, int line) { + if (!value) { + std::fprintf(stderr, "Save-data memory check failed at line %d\n", line); + std::abort(); + } } +#define Require(value) Check((value), __LINE__) static std::vector Read(const std::filesystem::path& path) { std::ifstream file(path, std::ios::binary); @@ -31,8 +42,14 @@ int main() { Require(std::filesystem::create_directory(root)); std::filesystem::current_path(root); const auto path = std::filesystem::path("_sd_mem/u7531/slot0.param"); + const auto memoryPath = std::filesystem::path("_sd_mem/u7531/slot0.bin"); std::filesystem::create_directories(path.parent_path()); - { std::ofstream file("_sd_mem/u7531/slot0.bin"); file << "save"; Require(static_cast(file)); } + const std::vector memoryOriginal(32, 's'); + { + std::ofstream file(memoryPath, std::ios::binary); + file.write(memoryOriginal.data(), static_cast(memoryOriginal.size())); + Require(static_cast(file)); + } Require(sceSaveDataInitialize3(nullptr) == 0); Require(sceSaveDataInitialize3(nullptr) == 0); Require(sceSaveDataTerminate() == 0); @@ -55,6 +72,49 @@ int main() { Require(Read(path) == std::vector(bytes, bytes + sizeof(param))); set.param = nullptr; Require(sceSaveDataSetSaveDataMemory2(&set) == 0); + std::array first{'a', '\0', 'b', 'c', 'd', 'e', 'f', 'g'}; + std::array second{'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o'}; + std::array data{{ + {first.data(), first.size(), 4}, + {second.data(), second.size(), 24}, + {nullptr, 0, std::numeric_limits::max()} + }}; + set.data = data.data(); + set.data_num = static_cast(data.size()); + data[1].offset = memoryOriginal.size(); + Require(sceSaveDataSetSaveDataMemory2(&set) == static_cast(0x809F0000u)); + Require(Read(memoryPath) == memoryOriginal); + data[1].offset = 24; + const auto memoryTemporary = memoryPath.string() + ".tmp"; + Require(std::filesystem::create_directory(memoryTemporary)); + Require(sceSaveDataSetSaveDataMemory2(&set) == static_cast(0x809F000Bu)); + Require(Read(memoryPath) == memoryOriginal); + Require(!std::filesystem::exists(memoryTemporary)); +#ifndef _WIN32 + rlimit previousLimit{}; + Require(getrlimit(RLIMIT_FSIZE, &previousLimit) == 0); + auto writeLimit = previousLimit; + writeLimit.rlim_cur = 16; + const auto previousHandler = std::signal(SIGXFSZ, SIG_IGN); + Require(previousHandler != SIG_ERR); + Require(setrlimit(RLIMIT_FSIZE, &writeLimit) == 0); + const int writeStatus = sceSaveDataSetSaveDataMemory2(&set); + Require(setrlimit(RLIMIT_FSIZE, &previousLimit) == 0); + Require(std::signal(SIGXFSZ, previousHandler) != SIG_ERR); + Require(writeStatus == static_cast(0x809F000Bu)); + Require(Read(memoryPath) == memoryOriginal); + Require(!std::filesystem::exists(memoryTemporary)); +#endif + Require(sceSaveDataSetSaveDataMemory2(&set) == 0); + auto memoryExpected = memoryOriginal; + std::copy(first.begin(), first.end(), memoryExpected.begin() + 4); + std::copy(second.begin(), second.end(), memoryExpected.begin() + 24); + Require(Read(memoryPath) == memoryExpected); + first.fill('z'); + set.data_num = 0; + Require(sceSaveDataSetSaveDataMemory2(&set) == 0); + std::copy(first.begin(), first.end(), memoryExpected.begin() + 4); + Require(Read(memoryPath) == memoryExpected); Require(sceSaveDataTerminate() == 0); std::filesystem::current_path(previous); std::filesystem::remove_all(root); diff --git a/core/libs/tests/ScalarCall.cpp b/core/libs/tests/ScalarCall.cpp new file mode 100644 index 000000000..3058f6577 --- /dev/null +++ b/core/libs/tests/ScalarCall.cpp @@ -0,0 +1,207 @@ +#include "Translation/TranslationContext.hpp" +#include "Translation/InstructionTranslator.hpp" +#include "RdnaDecoder/RdnaScalarOpDecoder.hpp" +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" +#include "RdnaDecoder/RdnaMemoryOpDecoder.hpp" +#include "ControlFlow/GraphBuilder.hpp" +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +static void Require(bool value) { + if (!value) throw std::runtime_error("scalar call regression"); +} + +template +static void ExpectThrow(const char* needle, TFunction fn) { + try { + fn(); + } catch (const std::exception& error) { + if (std::string(error.what()).find(needle) == std::string::npos) { + throw std::runtime_error(std::string("expected ") + needle + ", got " + error.what()); + } + return; + } + throw std::runtime_error(std::string("expected throw containing ") + needle); +} + +static RdnaInstruction Call(std::uint32_t pc, std::uint32_t link, std::uint32_t immediate) { + const std::array code{0xbb000000u | (link << 16u) | (immediate & 0xffffu)}; + return DecodeRdnaSopk(pc, code, 0u); +} + +static RdnaInstruction Sop1(std::uint32_t pc, std::uint32_t destination, std::uint32_t op, std::uint32_t source) { + const std::array code{0xbe800000u | (destination << 16u) | (op << 8u) | source}; + return DecodeRdnaSop1(pc, code, 0u); +} + +static RdnaInstruction Sopp(std::uint32_t pc, std::uint32_t op, std::uint32_t immediate = 0u) { + const std::array code{0xbf800000u | (op << 16u) | immediate}; + return DecodeRdnaSopp(pc, code, 0u); +} + +static ControlFlowGraph Build(std::initializer_list instructions) { + RdnaProgram program; + program.instructions.assign(instructions); + return GraphBuilder{}.Build(program); +} + +static void CheckDecode() { + const auto instruction = Call(0x40u, 8u, 7u); + Require(instruction.op == RdnaOpcode::SCallB64); + Require(instruction.family == RdnaInstructionFamily::SOPK); + Require(instruction.destination.kind == RdnaOperandKind::ScalarRegister && instruction.destination.reg == 8u); + Require(instruction.dataDwordCount == 2u && instruction.wordCount == 1u); + Require(instruction.branchTarget == 0x60u); + Require(Call(0x40u, 8u, 0xfff9u).branchTarget == 0x28u); + Require(Call(0x20000u, 8u, 0x8000u).branchTarget == 4u); + Require(Call(0u, 8u, 0x7fffu).branchTarget == 0x20000u); +} + +static std::string TranslateDump(const RdnaInstruction& instruction) { + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + program.BlockOrder() = {&block}; + program.Metadata().blockInfo.resize(1); + TranslationContext context(program, block, 256); + context.TranslateInstruction(instruction); + return ProgramToString(program); +} + +static void CheckLink() { + const auto dump = TranslateDump(Call(0x50u, 8u, 2u)); + Require(dump.find("0x0000000000000054") != std::string::npos); + Require(dump == TranslateDump(Sop1(0x50u, 8u, 0x1fu, 0u))); +} + +static void CheckCallAfterEnd() { + const std::array code{0xbb080001u, 0xbf810000u, 0xbf800000u, 0xbe802008u, 0xbf810000u}; + const auto decoded = RdnaInstructionDecoder{}.Decode(code); + Require(decoded.instructions.size() == code.size()); + const auto cfg = GraphBuilder{}.Build(decoded); + const auto& caller = cfg.FindBlockByProgramCounter(0u); + const auto& callee = cfg.FindBlockByProgramCounter(8u); + Require(caller.terminator.kind == TerminatorKind::Branch && caller.terminator.trueBlock == callee.id); + Require(callee.terminator.kind == TerminatorKind::Branch && callee.terminator.trueBlock == cfg.FindBlockByProgramCounter(4u).id); + ShaderComputeInputInfo computeInfo{}; + TranslateOptions options{}; + options.stage = ShaderStageKind::Compute; + options.inputInfo.compute = &computeInfo; + static_cast(InstructionTranslator{}.Translate(decoded, cfg, options)); +} + +static void CheckRejections() { + ExpectThrow("invalid instruction boundary", [] { + Build({Call(0u, 8u, 16u), Sopp(4u, 1u)}); + }); + ExpectThrow("invalid instruction boundary", [] { + const std::array code{0xbb080002u, 0xbf810000u, 0xbe8003ffu, 0u, 0xbe802008u, 0xbf810000u, 0xbf810000u}; + static_cast(GraphBuilder{}.Build(RdnaInstructionDecoder{}.Decode(code))); + }); + ExpectThrow("code span bounds", [] { + const std::array outside{0xbb080010u, 0xbf810000u}; + static_cast(RdnaInstructionDecoder{}.Decode(outside)); + }); + ExpectThrow("code span bounds", [] { + const std::array outside{0xbb08fffeu, 0xbf810000u}; + static_cast(RdnaInstructionDecoder{}.Decode(outside)); + }); + ExpectThrow("no paired s_setpc_b64 return", [] { + Build({Call(0u, 8u, 0u), Sopp(4u, 1u)}); + }); + for (const auto link : {9u, 105u, 106u, 125u}) { + ExpectThrow("ordinary aligned scalar register pair", [=] { + Build({Call(0u, link, 0u), Sopp(4u, 1u)}); + }); + } + ExpectThrow("escapes the constant-offset call/return model", [] { + Build({Call(0u, 8u, 1u), Sopp(4u, 1u), Sop1(8u, 8u, 3u, 128u), Sop1(12u, 0u, 0x20u, 8u)}); + }); + ExpectThrow("recursive scalar call", [] { + Build({Call(0u, 8u, 0xffffu), Sop1(4u, 0u, 0x20u, 8u), Sopp(8u, 1u)}); + }); + ExpectThrow("is shared with another call", [] { + Build({Call(0u, 8u, 2u), Call(4u, 8u, 1u), Sopp(8u, 1u), Sop1(12u, 0u, 0x20u, 8u)}); + }); + ExpectThrow("multiple s_setpc_b64 returns", [] { + Build({Call(0u, 8u, 1u), Sopp(4u, 1u), Sop1(8u, 0u, 0x20u, 8u), Sop1(12u, 0u, 0x20u, 8u)}); + }); + ExpectThrow("control flow enters the call region", [] { + Build({Call(0u, 8u, 2u), Sopp(4u, 2u, 1u), Sopp(8u, 1u), Sopp(12u, 0u), Sop1(16u, 0u, 0x20u, 8u)}); + }); + ExpectThrow("control flow leaves the call region", [] { + Build({Call(0u, 8u, 1u), Sopp(4u, 1u), Sopp(8u, 2u, 2u), Sop1(12u, 0u, 0x20u, 8u), Sopp(16u, 1u), Sopp(20u, 1u)}); + }); +} + +static void CheckGuards(const std::string& only) { + bool checked = false; + const auto check = [&](const char* name, const char* needle, auto fn) { + if (only.empty() || only == name) { + checked = true; + ExpectThrow(needle, fn); + } + }; + check("tuple", "escapes the constant-offset call/return model", [] { + auto load = Sopp(8u, 0u); + load.family = RdnaInstructionFamily::SMEM; + load.op = RdnaOpcode::SLoadDwordx8; + load.dataDwordCount = 8u; + load.destination.kind = RdnaOperandKind::ScalarRegister; + load.destination.reg = 4u; + Build({Call(0u, 8u, 1u), Sopp(4u, 1u), load, Sop1(12u, 0u, 0x20u, 8u)}); + }); + check("wrong_return", "escapes the constant-offset call/return model", [] { + Build({Call(0u, 8u, 1u), Sopp(4u, 1u), Sop1(8u, 0u, 0x20u, 9u)}); + }); + check("fallthrough", "control flow enters the call region", [] { + Build({Call(0u, 8u, 2u), Sopp(4u, 0u), Sopp(8u, 0u), Sopp(12u, 0u), Sop1(16u, 0u, 0x20u, 8u)}); + }); + check("conditional_fallthrough", "control flow enters the call region", [] { + Build({Call(0u, 8u, 2u), Sopp(4u, 2u), Sopp(8u, 4u, 2u), Sopp(12u, 0u), Sop1(16u, 0u, 0x20u, 8u), Sopp(20u, 1u)}); + }); + check("return_before_target", "return precedes the call target", [] { + Build({Call(0u, 8u, 2u), Sopp(4u, 1u), Sop1(8u, 0u, 0x20u, 8u), Sopp(12u, 0u), Sopp(16u, 1u)}); + }); + check("descriptor", "escapes the constant-offset call/return model", [] { + const std::array code{0xe0381000u, 0x80020401u}; + const auto load = DecodeRdnaMubuf(8u, code, 0u); + Require(load.source1.reg == 8u); + Build({Call(0u, 10u, 1u), Sopp(4u, 1u), load, Sop1(16u, 0u, 0x20u, 10u)}); + }); + check("relative_read", "escapes the constant-offset call/return model", [] { + Build({Call(0u, 8u, 1u), Sopp(4u, 1u), Sop1(8u, 0u, 0x2eu, 0u), Sop1(12u, 0u, 0x20u, 8u)}); + }); + if (only.empty()) { + for (const auto link : {0u, 104u}) { + const auto cfg = Build({Call(0u, link, 1u), Sopp(4u, 1u), Sop1(8u, 0u, 0x20u, link)}); + Require(cfg.FindBlockByProgramCounter(0u).terminator.kind == TerminatorKind::Branch); + } + } + Require(checked); +} + +int main(int argc, char** argv) { + try { + if (argc == 1) { + CheckDecode(); + CheckLink(); + CheckCallAfterEnd(); + CheckRejections(); + CheckGuards(""); + } else { + Require(argc == 2); + CheckGuards(argv[1]); + } + } catch (const std::exception& error) { + std::fprintf(stderr, "scalar call regression: %s\n", error.what()); + return 1; + } + return 0; +} diff --git a/core/libs/tests/ScalarSwappc.cpp b/core/libs/tests/ScalarSwappc.cpp new file mode 100644 index 000000000..6ab8dc4f0 --- /dev/null +++ b/core/libs/tests/ScalarSwappc.cpp @@ -0,0 +1,228 @@ +#include "ControlFlow/GraphBuilder.hpp" +#include "IntermediateRepresentation/IrProgram.hpp" +#include "RdnaDecoder/RdnaScalarOpDecoder.hpp" +#include "Translation/TranslationContext.hpp" +#include +#include +#include +#include + +using namespace ShaderRecompiler; +static void Require(bool value) { if (!value) throw std::runtime_error("scalar swappc regression"); } + +template +static void ExpectThrow(const char* needle, Fn fn) { + try { + fn(); + } catch (const std::exception& error) { + if (std::string(error.what()).find(needle) == std::string::npos) { + throw std::runtime_error(std::string("wrong error for \"") + needle + "\": " + error.what()); + } + return; + } + throw std::runtime_error(std::string("expected throw containing \"") + needle + "\""); +} + +static RdnaInstruction Sop1(std::uint32_t pc, std::uint32_t sdst, std::uint32_t opcode, std::uint32_t ssrc0) { + const std::array words{0x80000000u | (0x7du << 23u) | (sdst << 16u) | (opcode << 8u) | ssrc0, 0u}; + return DecodeRdnaSop1(pc, words, 0u); +} + +static RdnaInstruction Sop2Literal(std::uint32_t pc, std::uint32_t sdst, std::uint32_t opcode, std::uint32_t ssrc0, std::uint32_t literal) { + const std::array words{0x80000000u | (opcode << 23u) | (sdst << 16u) | (0xffu << 8u) | ssrc0, literal}; + return DecodeRdnaSop2(pc, words, 0u); +} + +static RdnaInstruction Sopp(std::uint32_t pc, std::uint32_t opcode) { + const std::array words{0x80000000u | (0x7fu << 23u) | (opcode << 16u)}; + return DecodeRdnaSopp(pc, words, 0u); +} + +static RdnaInstruction Sopk(std::uint32_t pc, std::uint32_t opcode, std::uint32_t sdst, std::uint32_t simm16) { + const std::array words{0x80000000u | ((0x60u + opcode) << 23u) | (sdst << 16u) | (simm16 & 0xffffu)}; + return DecodeRdnaSopk(pc, words, 0u); +} + +static RdnaProgram Program(std::initializer_list instructions) { + RdnaProgram program; + program.instructions.assign(instructions); + return program; +} + +static ControlFlowGraph Build(const RdnaProgram& program, const SwappcInfo* swappc = nullptr) { + return GraphBuilder{}.Build(program, swappc); +} + +static void CheckDecodeIdentity() { + const std::array words{0x80000000u | (0x7du << 23u) | (8u << 16u) | (0x21u << 8u) | 4u, 0u}; + const RdnaInstruction instruction = DecodeRdnaSop1(0x24u, words, 0u); + Require(instruction.op == RdnaOpcode::SSwappcB64); + Require(instruction.family == RdnaInstructionFamily::SOP1); + Require(instruction.programCounter == 0x24u); + Require(instruction.dataDwordCount == 2u); + Require(instruction.sourceCount == 1u); + Require(instruction.source0.kind == RdnaOperandKind::ScalarRegister && instruction.source0.reg == 4u); + Require(instruction.destination.kind == RdnaOperandKind::ScalarRegister && instruction.destination.reg == 8u); + + const std::array pcWords{0x80000000u | (0x7du << 23u) | (125u << 16u) | (0x21u << 8u) | 4u, 0u}; + Require(DecodeRdnaSop1(0u, pcWords, 0u).op == RdnaOpcode::SSetpcB64); + + + const RdnaInstruction add = Sop2Literal(0x4u, 4u, 0x00u, 4u, 0x18u); + Require(add.op == RdnaOpcode::SAddU32); + Require(add.wordCount == 2u); +} + +static std::string TranslateDump(const RdnaInstruction& instruction) { + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + program.BlockOrder() = {&block}; + program.Metadata().blockInfo.resize(1); + TranslationContext context(program, block, 256); + context.TranslateInstruction(instruction); + return ProgramToString(program); +} + +static void CheckLinkModelsNextInstructionAddress() { + const std::array getpcWords{0x80000000u | (0x7du << 23u) | (2u << 16u) | (0x1fu << 8u), 0u}; + const std::array swappcWords{0x80000000u | (0x7du << 23u) | (2u << 16u) | (0x21u << 8u) | 4u, 0u}; + const RdnaInstruction getpc = DecodeRdnaSop1(0x10u, getpcWords, 0u); + const RdnaInstruction swappc = DecodeRdnaSop1(0x10u, swappcWords, 0u); + Require(!TranslateDump(getpc).empty()); + Require(TranslateDump(getpc) == TranslateDump(swappc)); +} + +static void CheckStaticCallRegion() { + const auto program = Program({ + Sop1(0x00u, 4u, 0x1fu, 0u), + Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x18u), + Sop2Literal(0x0cu, 5u, 0x04u, 5u, 0u), + Sop1(0x14u, 8u, 0x21u, 4u), + Sopp(0x18u, 0x01u), + Sopp(0x1cu, 0x00u), + Sop1(0x20u, 0u, 0x20u, 8u), + }); + const auto cfg = Build(program); + Require(cfg.FindBlockByProgramCounter(0x00u).terminator.kind == TerminatorKind::Branch); + Require(cfg.FindBlockByProgramCounter(0x00u).terminator.trueBlock == cfg.FindBlockByProgramCounter(0x1cu).id); + Require(cfg.FindBlockByProgramCounter(0x1cu).terminator.kind == TerminatorKind::Branch); + Require(cfg.FindBlockByProgramCounter(0x1cu).terminator.trueBlock == cfg.FindBlockByProgramCounter(0x18u).id); +} + +static void CheckFetchCallAtEntry() { + SwappcInfo info; + info.fetchCallAllowed = true; + info.userDataBaseRegister = 16u; + info.userDataCount = 8u; + const auto program = Program({ + Sopp(0x00u, 0x00u), + Sop1(0x04u, 2u, 0x21u, 16u), + Sopp(0x08u, 0x01u), + }); + const auto cfg = Build(program, &info); + Require(cfg.hasFetchCall); + Require(cfg.fetchCallProgramCounter == 0x04u); +} + +static void CheckThrows() { + ExpectThrow("computed/data-dependent s_swappc_b64", [] { + Build(Program({ + Sop1(0x00u, 8u, 0x21u, 4u), + Sopp(0x04u, 0x01u), + })); + }); + ExpectThrow("escapes the constant-offset call/return model", [] { + Build(Program({ + Sop1(0x00u, 4u, 0x1fu, 0u), + Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x18u), + Sop2Literal(0x0cu, 5u, 0x04u, 5u, 0u), + Sop1(0x14u, 8u, 0x21u, 4u), + Sop1(0x18u, 8u, 0x03u, 8u), + Sopp(0x1cu, 0x01u), + Sopp(0x20u, 0x00u), + Sop1(0x24u, 0u, 0x20u, 8u), + })); + }); + ExpectThrow("no paired s_setpc_b64 return", [] { + Build(Program({ + Sop1(0x00u, 4u, 0x1fu, 0u), + Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x18u), + Sop2Literal(0x0cu, 5u, 0x04u, 5u, 0u), + Sop1(0x14u, 8u, 0x21u, 4u), + Sopp(0x18u, 0x01u), + Sopp(0x1cu, 0x00u), + })); + }); + ExpectThrow("multiple s_setpc_b64 returns", [] { + Build(Program({ + Sop1(0x00u, 4u, 0x1fu, 0u), + Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x18u), + Sop2Literal(0x0cu, 5u, 0x04u, 5u, 0u), + Sop1(0x14u, 8u, 0x21u, 4u), + Sopp(0x18u, 0x01u), + Sopp(0x1cu, 0x00u), + Sop1(0x20u, 0u, 0x20u, 8u), + Sop1(0x24u, 0u, 0x20u, 8u), + })); + }); + ExpectThrow("is shared with another call", [] { + Build(Program({ + Sop1(0x00u, 4u, 0x1fu, 0u), + Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x18u), + Sop2Literal(0x0cu, 5u, 0x04u, 5u, 0u), + Sop1(0x14u, 8u, 0x21u, 4u), + Sop1(0x18u, 8u, 0x21u, 4u), + Sopp(0x1cu, 0x01u), + Sopp(0x20u, 0x00u), + Sop1(0x24u, 0u, 0x20u, 8u), + })); + }); + ExpectThrow("recursive scalar call", [] { + Build(Program({ + Sop1(0x00u, 4u, 0x1fu, 0u), + Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x10u), + Sop2Literal(0x0cu, 5u, 0x04u, 5u, 0u), + Sop1(0x14u, 8u, 0x21u, 4u), + Sopp(0x18u, 0x01u), + Sop1(0x1cu, 0u, 0x20u, 8u), + })); + }); + ExpectThrow("computed/data-dependent s_swappc_b64", [] { + SwappcInfo info; + info.fetchCallAllowed = true; + info.userDataBaseRegister = 16u; + info.userDataCount = 8u; + Build(Program({ + Sopp(0x00u, 0x00u), + Sop1(0x04u, 2u, 0x21u, 4u), + Sopp(0x08u, 0x01u), + }), &info); + }); + ExpectThrow("must live in the fetch-shader code", [] { + SwappcInfo info; + info.fetchCallAllowed = true; + info.userDataBaseRegister = 16u; + info.userDataCount = 8u; + Build(Program({ + Sopp(0x00u, 0x00u), + Sop1(0x04u, 2u, 0x21u, 16u), + Sopp(0x08u, 0x01u), + Sop1(0x0cu, 0u, 0x20u, 2u), + }), &info); + }); +} + +int main() { + try { + CheckDecodeIdentity(); + CheckLinkModelsNextInstructionAddress(); + CheckStaticCallRegion(); + CheckFetchCallAtEntry(); + CheckThrows(); + } catch (const std::exception& error) { + std::fprintf(stderr, "scalar swappc regression: %s\n", error.what()); + return 1; + } + return 0; +} diff --git a/core/libs/tests/VectorFusedMultiplyAdd.cpp b/core/libs/tests/VectorFusedMultiplyAdd.cpp index e004d27b2..3315e155a 100644 --- a/core/libs/tests/VectorFusedMultiplyAdd.cpp +++ b/core/libs/tests/VectorFusedMultiplyAdd.cpp @@ -51,6 +51,7 @@ void Check(const std::string& name, std::uint32_t encoding, std::uint32_t wordCo auto& block = program.CreateBlock(); program.SetEntryBlock(block); TranslationContext context(program, block, 256); + context.SetFloatMode(ShaderFloatMode{0xf0u, false, false, false}); context.TranslateInstruction(instruction); const IrOpcode rejected = expected == IrOpcode::FPFma32 ? IrOpcode::FPMad32 : IrOpcode::FPFma32; @@ -69,6 +70,26 @@ void Check(const std::string& name, std::uint32_t encoding, std::uint32_t wordCo } } +void CheckMix(const std::string& name, const std::array& words) { + const RdnaInstruction instruction = DecodeRdnaVectorOp(std::span(words), 0u); + Require(instruction.family == RdnaInstructionFamily::VOP3P, name + " is not decoded as VOP3P"); + Require(instruction.op == RdnaOpcode::VFmaF32, name + " decodes to another opcode"); + + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + TranslationContext context(program, block, 256); + context.SetFloatMode(ShaderFloatMode{0xf0u, false, false, false}); + context.TranslateInstruction(instruction); + + std::uint32_t fused = 0u; + for (const auto* value : block.Instructions()) { + Require(value->Opcode() != IrOpcode::FPMad32, name + " rounds the product separately"); + if (value->Opcode() == IrOpcode::FPFma32) ++fused; + } + Require(fused == 1u, name + " emits " + std::to_string(fused) + " fused multiply-adds"); +} + } int main() { @@ -82,6 +103,8 @@ int main() { Check("v_fmac_f32", 0x2bu, 1u, RdnaOpcode::VMacF32, IrOpcode::FPFma32, accumulate); Check("v_fmamk_f32", 0x2cu, 2u, RdnaOpcode::VMadmkF32, IrOpcode::FPFma32, multiplyLiteral); Check("v_fmaak_f32", 0x2du, 2u, RdnaOpcode::VMadakF32, IrOpcode::FPFma32, addLiteral); + CheckMix("v_fma_mix_f32", {0xcc200005u, 0x04220f06u}); + CheckMix("v_fma_mix_f32 op_sel_hi:[1,0,1]", {0xcc204005u, 0x0c220f06u}); return 0; } catch (const std::exception& error) { std::cerr << error.what() << '\n'; diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 1b3e069fe..f1d7705e2 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -131,6 +131,11 @@ if(BUILD_TESTING) target_include_directories(autorun_exit_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cli/include) add_test(NAME autorun_exit_status COMMAND autorun_exit_tests) set_tests_properties(autorun_exit_status PROPERTIES TIMEOUT 20) + add_test_executable(cli_args_tests + ${CMAKE_CURRENT_SOURCE_DIR}/cli/tests/CliArgsTests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/cli/src/CliArgs.cpp) + target_include_directories(cli_args_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cli/include) + add_test(NAME cli_args COMMAND cli_args_tests) add_test_executable(buffer_bounds_tests ${CMAKE_CURRENT_SOURCE_DIR}/io/tests/BufferBoundsTests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/io/src/ByteReader.cpp @@ -194,6 +199,8 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_elf_offsets.py $) add_test(NAME dynamic_segment COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_dynamic_segment.py $) + add_test(NAME rela_size COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_rela_size.py $) add_test(NAME optional_plt COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_optional_plt.py $) add_test(NAME empty_tls COMMAND ${Python3_EXECUTABLE} @@ -204,6 +211,10 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_tls_diagnostics.py $) add_test(NAME linux_load_alignment COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_load_alignment.py $) + add_test(NAME linux_eh_frame_header COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_eh_frame_header.py $) + add_test(NAME linux_dynamic_strings COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_dynamic_strings.py $) add_test(NAME executable_segments COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_executable_segments.py $) add_test(NAME linux_entry_argv COMMAND ${Python3_EXECUTABLE} @@ -236,6 +247,10 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_input_magic.py $) add_test(NAME string_table_bounds COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_string_table_bounds.py $) + add_test(NAME call_registry COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_call_registry.py $) + add_test(NAME jmprel_table_bounds COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_jmprel_table_bounds.py $) else() message(STATUS "Python3 interpreter not found; skipping Python relinker tests") endif() @@ -277,10 +292,19 @@ if(BUILD_TESTING) ) add_test(NAME strict_nid_filter COMMAND strict_nid_filter_tests) - if(UNIX AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") add_test_executable(to_intel_linux_stubs_tests ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/ToIntelLinuxStubsTests.cpp ) add_test(NAME to_intel_linux_stubs COMMAND to_intel_linux_stubs_tests $) + + add_test_executable(rela_bounds_tests + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/RelaBoundsTests.cpp + ) + add_test(NAME rela_bounds COMMAND rela_bounds_tests $) + add_test_executable(guest_intel_trampolines_tests + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/GuestIntelTrampolinesTests.cpp + ) + add_test(NAME guest_intel_trampolines_execution COMMAND guest_intel_trampolines_tests $) endif() endif() diff --git a/core/relinker/cli/tests/CliArgsTests.cpp b/core/relinker/cli/tests/CliArgsTests.cpp new file mode 100644 index 000000000..98f3b2997 --- /dev/null +++ b/core/relinker/cli/tests/CliArgsTests.cpp @@ -0,0 +1,215 @@ +#include +#include +#include +#include +#include + +namespace { + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +Cli::Args parse(const std::vector& arguments) { + std::string program = "relinker"; + std::vector argv; + argv.reserve(arguments.size() + 1); + argv.push_back(program.data()); + std::vector storage = arguments; + for (auto& argument : storage) argv.push_back(argument.data()); + return Cli::ParseArgs(static_cast(argv.size()), argv.data()); +} + +std::string join(const std::vector& arguments) { + std::string text; + for (const auto& argument : arguments) text += (text.empty() ? "" : " ") + argument; + return text; +} + +void expectThrows(const std::vector& arguments, const std::string& expectedMessage) { + try { + parse(arguments); + } catch (const std::exception& error) { + require(error.what() == expectedMessage, + "wrong error for [" + join(arguments) + "]: got \"" + error.what() + "\", expected \"" + expectedMessage + "\""); + return; + } + throw std::runtime_error("expected an exception for [" + join(arguments) + "]"); +} + +void expectUsage(const std::vector& arguments) { + try { + parse(arguments); + } catch (const std::exception& error) { + require(std::string(error.what()).rfind("Usage: relinker", 0) == 0, + "expected a usage message for [" + join(arguments) + "], got \"" + error.what() + "\""); + return; + } + throw std::runtime_error("expected a usage message for [" + join(arguments) + "]"); +} + +void defaults() { + const auto args = parse({"input.elf", "output.elf"}); + require(args.inputPath == "input.elf", "input path was not captured"); + require(args.outputPath == "output.elf", "output path was not captured"); + require(!args.skipSyscallCheck && !args.skipSceModule && !args.toIntel && !args.writeRegistry && + !args.toWindows && !args.lazyBinding && !args.autorun && !args.windowsDiagnostics && !args.windowsGui, + "a flag defaulted to true"); + require(args.unusedFilterLevel == 0u, "unused filter level did not default to 0"); + require(args.runPath == "$ORIGIN/libs", "run path did not keep its default"); + require(args.excludedSceModules.empty(), "excluded modules were not empty by default"); +} + +void booleanFlags() { + const auto args = parse({"--skip-syscall-check", "--skip-sce-module", "--to-intel", "--registry", + "--windows", "--lazy-binding", "--autorun", "--windows-diagnostics", "--windows-gui", + "input.elf", "output.elf"}); + require(args.skipSyscallCheck, "--skip-syscall-check was ignored"); + require(args.skipSceModule, "--skip-sce-module was ignored"); + require(args.toIntel, "--to-intel was ignored"); + require(args.writeRegistry, "--registry was ignored"); + require(args.toWindows, "--windows was ignored"); + require(args.lazyBinding, "--lazy-binding was ignored"); + require(args.autorun, "--autorun was ignored"); + require(args.windowsDiagnostics, "--windows-diagnostics was ignored"); + require(args.windowsGui, "--windows-gui was ignored"); + require(args.inputPath == "input.elf" && args.outputPath == "output.elf", "flags consumed the positional paths"); +} + +void positionalsInterleaveWithFlags() { + const auto args = parse({"--windows", "input.elf", "--registry", "output.elf", "--autorun"}); + require(args.toWindows && args.writeRegistry && args.autorun, "a flag after a positional was ignored"); + require(args.inputPath == "input.elf" && args.outputPath == "output.elf", "positionals were misordered around flags"); +} + +void singleDashIsPositional() { + const auto args = parse({"-x", "-o"}); + require(args.inputPath == "-x" && args.outputPath == "-o", "a single-dash argument was not treated as a positional"); +} + +void rpath() { + const auto args = parse({"--rpath", "/custom/libs", "input.elf", "output.elf"}); + require(args.runPath == "/custom/libs", "--rpath value was not captured"); +} + +void rpathConsumesNextTokenVerbatim() { + const auto args = parse({"--rpath", "--windows", "input.elf", "output.elf"}); + require(args.runPath == "--windows", "--rpath did not consume the following token as its value"); + require(!args.toWindows, "the token consumed by --rpath was also parsed as a flag"); +} + +void rpathWithoutValue() { + expectThrows({"--rpath"}, "--rpath requires a value"); +} + +void excludeSceModule() { + const auto args = parse({"--exclude-sce-module", "libc.prx", "--exclude-sce-module", "libkernel.prx", + "input.elf", "output.elf"}); + require(args.excludedSceModules.size() == 2, "both excluded modules were not recorded"); + require(args.excludedSceModules.count("libc.prx") == 1 && args.excludedSceModules.count("libkernel.prx") == 1, + "an excluded module name was lost"); +} + +void excludeSceModuleConsumesNextTokenVerbatim() { + const auto args = parse({"--exclude-sce-module", "--skip-sce-module", "input.elf", "output.elf"}); + require(args.excludedSceModules.count("--skip-sce-module") == 1, "--exclude-sce-module did not consume the following token as its value"); + require(!args.skipSceModule, "the token consumed by --exclude-sce-module was also parsed as a flag"); +} + +void excludeSceModuleWithoutValue() { + expectThrows({"--exclude-sce-module"}, "--exclude-sce-module requires a file name"); +} + +void excludeConflictsWithSkip() { + const std::string message = "--exclude-sce-module conflicts with --skip-sce-module"; + expectThrows({"--skip-sce-module", "--exclude-sce-module", "m", "input.elf", "output.elf"}, message); + expectThrows({"--exclude-sce-module", "m", "--skip-sce-module", "input.elf", "output.elf"}, message); +} + +void unusedFilter() { + for (const auto& value : {std::string("0"), std::string("1"), std::string("2")}) { + const auto args = parse({"unused-filter=" + value, "input.elf", "output.elf"}); + require(args.unusedFilterLevel == static_cast(value[0] - '0'), + "unused-filter=" + value + " produced the wrong level"); + } +} + +void unusedFilterRejectsBadValues() { + const std::string message = "unused-filter must be specified once with a value of 0, 1 or 2"; + expectThrows({"unused-filter=3", "input.elf", "output.elf"}, message); + expectThrows({"unused-filter=", "input.elf", "output.elf"}, message); + expectThrows({"unused-filter=12", "input.elf", "output.elf"}, message); + expectThrows({"unused-filter=a", "input.elf", "output.elf"}, message); +} + +void unusedFilterRejectsDuplicates() { + expectThrows({"unused-filter=1", "unused-filter=2", "input.elf", "output.elf"}, + "unused-filter must be specified once with a value of 0, 1 or 2"); +} + +void unusedFilterBareWordIsUnknown() { + expectThrows({"unused-filter", "input.elf", "output.elf"}, "unknown option: unused-filter"); +} + +void unknownOption() { + expectThrows({"--nonexistent", "input.elf", "output.elf"}, "unknown option: --nonexistent"); + expectThrows({"--", "input.elf", "output.elf"}, "unknown option: --"); +} + +void tooManyPositionals() { + expectThrows({"input.elf", "output.elf", "extra.elf"}, "unexpected argument: extra.elf"); +} + +void windowsDiagnosticsRequiresWindows() { + const std::string message = "--windows-diagnostics requires --windows"; + expectThrows({"--windows-diagnostics", "input.elf", "output.elf"}, message); + const auto before = parse({"--windows-diagnostics", "--windows", "input.elf", "output.elf"}); + require(before.windowsDiagnostics && before.toWindows, "--windows-diagnostics before --windows was rejected"); + const auto after = parse({"--windows", "--windows-diagnostics", "input.elf", "output.elf"}); + require(after.windowsDiagnostics && after.toWindows, "--windows-diagnostics after --windows was rejected"); +} + +void windowsGuiRequiresWindows() { + const std::string message = "--windows-gui requires --windows"; + expectThrows({"--windows-gui", "input.elf", "output.elf"}, message); + const auto ok = parse({"--windows", "--windows-gui", "input.elf", "output.elf"}); + require(ok.windowsGui && ok.toWindows, "--windows-gui with --windows was rejected"); +} + +void missingPositionals() { + expectUsage({}); + expectUsage({"input.elf"}); + expectUsage({"--windows"}); + expectUsage({"--rpath", "/libs"}); +} + +} + +int main() { + try { + defaults(); + booleanFlags(); + positionalsInterleaveWithFlags(); + singleDashIsPositional(); + rpath(); + rpathConsumesNextTokenVerbatim(); + rpathWithoutValue(); + excludeSceModule(); + excludeSceModuleConsumesNextTokenVerbatim(); + excludeSceModuleWithoutValue(); + excludeConflictsWithSkip(); + unusedFilter(); + unusedFilterRejectsBadValues(); + unusedFilterRejectsDuplicates(); + unusedFilterBareWordIsUnknown(); + unknownOption(); + tooManyPositionals(); + windowsDiagnosticsRequiresWindows(); + windowsGuiRequiresWindows(); + missingPositionals(); + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/core/relinker/codegen/include/codegen/CodegenTypes.hpp b/core/relinker/codegen/include/codegen/CodegenTypes.hpp index ee1963333..e41f6cd53 100644 --- a/core/relinker/codegen/include/codegen/CodegenTypes.hpp +++ b/core/relinker/codegen/include/codegen/CodegenTypes.hpp @@ -36,6 +36,7 @@ struct TrampolineSite { std::vector OriginalBytes; std::vector Body; std::size_t ReturnBranchOffset; + std::vector Relocations = {}; }; struct ConvertResult { diff --git a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp index f54e2e1e0..2b835fd94 100644 --- a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp +++ b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTypes.hpp @@ -15,6 +15,12 @@ enum class Amd64OnlyLowering : std::uint8_t { Kept }; +struct StubRelocation { + std::size_t DisplacementOffset; + std::size_t InstructionEnd; + std::int64_t SiteTarget; +}; + struct Amd64OnlyMatch { std::string InstructionName; std::size_t Length; @@ -23,6 +29,7 @@ struct Amd64OnlyMatch { std::vector StubBody; std::size_t ReturnBranchOffset; bool Optional = false; + std::vector Relocations = {}; }; struct Amd64OnlySubstitutionReport { diff --git a/core/relinker/codegen/include/codegen/x86/ClzeroLowering.hpp b/core/relinker/codegen/include/codegen/x86/ClzeroLowering.hpp index 82b15ed41..029fd2bcc 100644 --- a/core/relinker/codegen/include/codegen/x86/ClzeroLowering.hpp +++ b/core/relinker/codegen/include/codegen/x86/ClzeroLowering.hpp @@ -4,14 +4,13 @@ #include #include #include -#include namespace Codegen { class ClzeroLowering { public: void EmitOutOfLine(StubBodyBuilder& body, const ClzeroOperands& operands) const; - [[nodiscard]] LoweredBody LowerOutOfLine(const ClzeroOperands& operands, std::span trailing = {}) const; + [[nodiscard]] LoweredBody LowerOutOfLine(const ClzeroOperands& operands) const; }; } diff --git a/core/relinker/codegen/include/codegen/x86/ReciprocalLowering.hpp b/core/relinker/codegen/include/codegen/x86/ReciprocalLowering.hpp index a4f20ea25..55eef20f2 100644 --- a/core/relinker/codegen/include/codegen/x86/ReciprocalLowering.hpp +++ b/core/relinker/codegen/include/codegen/x86/ReciprocalLowering.hpp @@ -3,14 +3,13 @@ #include #include -#include namespace Codegen { class ReciprocalLowering { public: void EmitOutOfLine(StubBodyBuilder& body, const ReciprocalOperands& operands) const; - [[nodiscard]] LoweredBody LowerOutOfLine(const ReciprocalOperands& operands, std::span trailing = {}) const; + [[nodiscard]] LoweredBody LowerOutOfLine(const ReciprocalOperands& operands) const; }; } diff --git a/core/relinker/codegen/include/codegen/x86/Sha1Lowering.hpp b/core/relinker/codegen/include/codegen/x86/Sha1Lowering.hpp index 1eabbd720..3507185d3 100644 --- a/core/relinker/codegen/include/codegen/x86/Sha1Lowering.hpp +++ b/core/relinker/codegen/include/codegen/x86/Sha1Lowering.hpp @@ -4,14 +4,13 @@ #include #include #include -#include namespace Codegen { class Sha1Lowering { public: void EmitOutOfLine(StubBodyBuilder& body, const Sha1Operands& operands) const; - [[nodiscard]] LoweredBody LowerOutOfLine(const Sha1Operands& operands, std::span trailing = {}) const; + [[nodiscard]] LoweredBody LowerOutOfLine(const Sha1Operands& operands) const; }; } diff --git a/core/relinker/codegen/include/codegen/x86/Sha256Lowering.hpp b/core/relinker/codegen/include/codegen/x86/Sha256Lowering.hpp index 2441149c6..12367e75b 100644 --- a/core/relinker/codegen/include/codegen/x86/Sha256Lowering.hpp +++ b/core/relinker/codegen/include/codegen/x86/Sha256Lowering.hpp @@ -4,14 +4,13 @@ #include #include #include -#include namespace Codegen { class Sha256Lowering { public: void EmitOutOfLine(StubBodyBuilder& body, const Sha256Operands& operands) const; - [[nodiscard]] LoweredBody LowerOutOfLine(const Sha256Operands& operands, std::span trailing = {}) const; + [[nodiscard]] LoweredBody LowerOutOfLine(const Sha256Operands& operands) const; }; } diff --git a/core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp b/core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp index 3df185400..f7e16a7da 100644 --- a/core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp +++ b/core/relinker/codegen/include/codegen/x86/Sse4aLowering.hpp @@ -15,8 +15,8 @@ class Sse4aLowering { public: [[nodiscard]] std::optional> LowerInPlace(const Sse4aOperands& operands, std::size_t originalLength) const; void EmitOutOfLine(StubBodyBuilder& body, const Sse4aOperands& operands) const; - [[nodiscard]] LoweredBody LowerOutOfLine(const Sse4aOperands& operands, std::span trailing = {}) const; - [[nodiscard]] LoweredBody LowerOutOfLine(std::span sequence, std::span trailing) const; + [[nodiscard]] LoweredBody LowerOutOfLine(const Sse4aOperands& operands) const; + [[nodiscard]] LoweredBody LowerOutOfLine(std::span sequence) const; }; } diff --git a/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp b/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp index 692cbc6a4..212eeea54 100644 --- a/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp +++ b/core/relinker/codegen/include/codegen/x86/StubBodyBuilder.hpp @@ -1,10 +1,13 @@ #ifndef CODEGEN_X86_STUBBODYBUILDER_HPP #define CODEGEN_X86_STUBBODYBUILDER_HPP +#include +#include #include #include #include #include +#include #include #include @@ -13,6 +16,7 @@ namespace Codegen { struct LoweredBody { std::vector Bytes; std::size_t ReturnBranchOffset; + std::vector Relocations = {}; }; using StubConstant = std::array; @@ -26,12 +30,15 @@ struct MemoryOperand { std::int32_t Displacement; bool StackBase; std::size_t EncodedSize; + bool RipRelative; + std::size_t InstructionLength; }; [[nodiscard]] MemoryOperand DecodeMemoryOperand(const std::uint8_t* data, std::size_t length, std::size_t modRmOffset, std::uint8_t rex, std::vector prefixes); void EmitSse(std::vector& out, std::uint8_t prefix, std::initializer_list opcode, std::uint8_t dst, std::uint8_t src); void EmitShiftImm(std::vector& out, std::uint8_t extension, std::uint8_t reg, std::uint8_t imm); +void ApplyStubRelocations(std::span body, std::span relocations, std::uint64_t siteAddress, std::uint64_t stubAddress, Domain::FileByteOffset failureOffset); class StubBodyBuilder { public: @@ -45,6 +52,8 @@ public: void Spill(std::uint8_t reg); void Restore(std::uint8_t reg); void Raw(std::span bytes); + void Move(std::span instruction, std::optional ripDisplacementOffset); + void Advance(std::size_t length); [[nodiscard]] LoweredBody Finish(); private: @@ -57,7 +66,9 @@ private: std::vector _bytes; std::vector _fixups; std::vector _constants; + std::vector _relocations; std::size_t _stackDepth = 0; + std::size_t _siteOffset = 0; }; } diff --git a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp index 022721a23..969a1fd30 100644 --- a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp +++ b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp @@ -110,6 +110,8 @@ inline constexpr std::uint8_t OneByteInt = 0xCD; inline constexpr std::uint8_t OneByteRetImm16 = 0xC2; inline constexpr std::uint8_t OneByteRetFarImm16 = 0xCA; inline constexpr std::uint8_t OneByteEnter = 0xC8; +inline constexpr std::uint8_t OneByteMovMoffsMin = 0xA0; +inline constexpr std::uint8_t OneByteMovMoffsMax = 0xA3; inline constexpr std::uint8_t TwoByteJccRel32Min = 0x80; inline constexpr std::uint8_t TwoByteJccRel32Max = 0x8F; @@ -117,7 +119,7 @@ inline constexpr std::uint8_t TwoByteMovImm8ModRm = 0xBA; inline constexpr std::uint8_t TwoByteCmovRangeMin = 0x40; inline constexpr std::uint8_t TwoByteCmovRangeMax = 0x4F; inline constexpr std::uint8_t TwoByteModRmRangeAMin = 0xA3; -inline constexpr std::uint8_t TwoByteModRmRangeAMax = 0xAB; +inline constexpr std::uint8_t TwoByteModRmRangeAMax = 0xA7; inline constexpr std::uint8_t TwoByteModRmRangeBMin = 0xB0; inline constexpr std::uint8_t TwoByteModRmRangeBMax = 0xB7; inline constexpr std::uint8_t TwoByteModRmRangeCMin = 0xBC; @@ -172,6 +174,7 @@ inline constexpr std::uint8_t TwoByteMovCrDrMin = 0x20; inline constexpr std::uint8_t TwoByteMovCrDrMax = 0x23; inline constexpr std::uint8_t TwoBytePopcnt = 0xB8; inline constexpr std::uint8_t TwoByteUd1 = 0xB9; +inline constexpr std::uint8_t TwoByteBts = 0xAB; inline constexpr std::uint8_t TwoByteBtc = 0xBB; inline constexpr std::uint8_t TwoByteXaddRm8 = 0xC0; inline constexpr std::uint8_t TwoByteMovnti = 0xC3; @@ -212,7 +215,6 @@ inline constexpr std::uint8_t SibBaseDisp32 = 0x05; inline constexpr std::size_t Disp8Size = 1; inline constexpr std::size_t Disp32Size = 4; -inline constexpr std::size_t Rel32InstructionLength = 5; inline constexpr std::size_t Vex2PrefixLength = 2; inline constexpr std::size_t Vex3PrefixLength = 3; diff --git a/core/relinker/codegen/src/Amd64OnlyConverter.cpp b/core/relinker/codegen/src/Amd64OnlyConverter.cpp index aa0bc0d09..0b3239d47 100644 --- a/core/relinker/codegen/src/Amd64OnlyConverter.cpp +++ b/core/relinker/codegen/src/Amd64OnlyConverter.cpp @@ -144,7 +144,7 @@ void Amd64OnlyConverter::_convertSegment( if (amdOnly && trailingBytes == 0) { sequence.push_back(bytes); taken.push_back(next); - } else if (amdOnly || info.FlowKind != ControlFlowKind::Sequential || info.HasRipRelativeDisp || info.HasBranchTarget) { + } else if (amdOnly || info.FlowKind != ControlFlowKind::Sequential || info.HasBranchTarget) { throw CodegenException("AMD-only instruction too short for a jump is followed by an instruction that cannot move", ph.Offset + following.Offset); } else { trailingBytes += following.Length; @@ -167,7 +167,8 @@ void Amd64OnlyConverter::_convertSegment( siteLength, std::vector(begin, begin + static_cast(siteLength)), stub.StubBody, - stub.ReturnBranchOffset + stub.ReturnBranchOffset, + stub.Relocations }); replacementLength = stub.StubBody.size(); consumed.insert(taken.begin(), taken.end()); diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 48bde0aaf..caff05f46 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -12,11 +12,14 @@ #include #include #include +#include #include #include #include #include +#include #include +#include #include namespace Codegen { @@ -67,6 +70,30 @@ const Entry& _sha1Entry(const Sha1Operands& operands) { return kSha1rnds4; } +Amd64OnlyMatch _trampoline(std::string name, const std::size_t length, LoweredBody body, const bool optional = false) { + return Amd64OnlyMatch{std::move(name), length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset, optional, std::move(body.Relocations)}; +} + +void _moveTrailing(StubBodyBuilder& body, std::span trailing) { + const X64InstructionDecoder decoder; + while (!trailing.empty()) { + const auto info = decoder.DecodeInstruction(trailing.data(), trailing.size()); + if (info.Length == 0 || info.Length > trailing.size()) + throw CodegenException("Trailing bytes do not split into whole instructions"); + body.Move(trailing.first(info.Length), info.HasRipRelativeDisp ? std::optional(info.RipRelativeDispOffset) : std::nullopt); + trailing = trailing.subspan(info.Length); + } +} + +template +LoweredBody _outOfLine(const std::size_t length, const std::span trailing, const TEmit& emit) { + StubBodyBuilder body; + emit(body); + body.Advance(length); + _moveTrailing(body, trailing); + return body.Finish(); +} + Amd64OnlyMatch _inPlace(const Entry& entry, const std::size_t length, std::vector replacement) { while (replacement.size() < length) { const auto& nop = kNops[std::min(length - replacement.size(), std::size(kNops)) - 1]; @@ -131,26 +158,23 @@ Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSse4a(const DecodedInstruction if (auto inPlace = _lowering.LowerInPlace(operands, instr.Length)) return Amd64OnlyMatch{entry.Name, instr.Length, Amd64OnlyLowering::InPlace, std::move(*inPlace), {}, 0}; } - auto body = _lowering.LowerOutOfLine(operands, trailing); const auto& name = operands.RegisterForm ? registerFormEntry.Name : entry.Name; - return Amd64OnlyMatch{name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; + return _trampoline(name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { _lowering.EmitOutOfLine(body, operands); })); } Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSha256(const DecodedInstruction& instr, std::span trailing) const { const auto operands = DecodeSha256(instr.Data, instr.Length); - auto body = _sha256Lowering.LowerOutOfLine(operands, trailing); - return Amd64OnlyMatch{_sha256Entry(operands).Name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; + return _trampoline(_sha256Entry(operands).Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { _sha256Lowering.EmitOutOfLine(body, operands); })); } Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchSha1(const DecodedInstruction& instr, std::span trailing) const { const auto operands = DecodeSha1(instr.Data, instr.Length); - auto body = _sha1Lowering.LowerOutOfLine(operands, trailing); - return Amd64OnlyMatch{_sha1Entry(operands).Name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; + return _trampoline(_sha1Entry(operands).Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { _sha1Lowering.EmitOutOfLine(body, operands); })); } Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchClzero(const DecodedInstruction& instr, std::span trailing) const { - auto body = _clzeroLowering.LowerOutOfLine(DecodeClzero(instr.Data, instr.Length), trailing); - return Amd64OnlyMatch{kClzero.Name, instr.Length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset}; + const auto operands = DecodeClzero(instr.Data, instr.Length); + return _trampoline(kClzero.Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { _clzeroLowering.EmitOutOfLine(body, operands); })); } std::optional Amd64OnlyInstructionMatcher::MatchSequence( @@ -190,11 +214,11 @@ std::optional Amd64OnlyInstructionMatcher::MatchSequence( } else { return std::nullopt; } + body.Advance(instr.Length); } - body.Raw(trailing); - auto lowered = body.Finish(); + _moveTrailing(body, trailing); const bool optional = DecodeVexReciprocal(instructions.front().data(), instructions.front().size()).has_value(); - return Amd64OnlyMatch{name, instructions.front().size(), Amd64OnlyLowering::Trampoline, {}, std::move(lowered.Bytes), lowered.ReturnBranchOffset, optional}; + return _trampoline(name, instructions.front().size(), body.Finish(), optional); } std::optional Amd64OnlyInstructionMatcher::Match( @@ -229,8 +253,7 @@ std::optional Amd64OnlyInstructionMatcher::Match( return _validWait(instr) ? _inPlace(kMwaitx, length, std::vector(kPause.Bytes, kPause.Bytes + kPause.Size)) : _unsupported(kMwaitx, length); if (const auto reciprocal = DecodeVexReciprocal(data, length)) { - auto body = _reciprocalLowering.LowerOutOfLine(*reciprocal, trailing); - return Amd64OnlyMatch{_reciprocalEntry(*reciprocal).Name, length, Amd64OnlyLowering::Trampoline, {}, std::move(body.Bytes), body.ReturnBranchOffset, true}; + return _trampoline(_reciprocalEntry(*reciprocal).Name, length, _outOfLine(length, trailing, [&](StubBodyBuilder& body) { _reciprocalLowering.EmitOutOfLine(body, *reciprocal); }), true); } if (instr.IsClzero()) diff --git a/core/relinker/codegen/src/x86/ClzeroLowering.cpp b/core/relinker/codegen/src/x86/ClzeroLowering.cpp index c4a1ced26..507796eac 100644 --- a/core/relinker/codegen/src/x86/ClzeroLowering.cpp +++ b/core/relinker/codegen/src/x86/ClzeroLowering.cpp @@ -41,10 +41,9 @@ void ClzeroLowering::EmitOutOfLine(StubBodyBuilder& body, const ClzeroOperands& body.Restore(kScratch); } -LoweredBody ClzeroLowering::LowerOutOfLine(const ClzeroOperands& operands, std::span trailing) const { +LoweredBody ClzeroLowering::LowerOutOfLine(const ClzeroOperands& operands) const { StubBodyBuilder body; EmitOutOfLine(body, operands); - body.Raw(trailing); return body.Finish(); } diff --git a/core/relinker/codegen/src/x86/ReciprocalLowering.cpp b/core/relinker/codegen/src/x86/ReciprocalLowering.cpp index b6f8fcadc..8f6d794b4 100644 --- a/core/relinker/codegen/src/x86/ReciprocalLowering.cpp +++ b/core/relinker/codegen/src/x86/ReciprocalLowering.cpp @@ -55,10 +55,9 @@ void ReciprocalLowering::EmitOutOfLine(StubBodyBuilder& body, const ReciprocalOp body.Restore(scratch); } -LoweredBody ReciprocalLowering::LowerOutOfLine(const ReciprocalOperands& operands, std::span trailing) const { +LoweredBody ReciprocalLowering::LowerOutOfLine(const ReciprocalOperands& operands) const { StubBodyBuilder body; EmitOutOfLine(body, operands); - body.Raw(trailing); return body.Finish(); } diff --git a/core/relinker/codegen/src/x86/Sha1Lowering.cpp b/core/relinker/codegen/src/x86/Sha1Lowering.cpp index c8a4d2b5c..e8172687b 100644 --- a/core/relinker/codegen/src/x86/Sha1Lowering.cpp +++ b/core/relinker/codegen/src/x86/Sha1Lowering.cpp @@ -183,10 +183,9 @@ void Sha1Lowering::EmitOutOfLine(StubBodyBuilder& body, const Sha1Operands& oper } } -LoweredBody Sha1Lowering::LowerOutOfLine(const Sha1Operands& operands, std::span trailing) const { +LoweredBody Sha1Lowering::LowerOutOfLine(const Sha1Operands& operands) const { StubBodyBuilder body; EmitOutOfLine(body, operands); - body.Raw(trailing); return body.Finish(); } diff --git a/core/relinker/codegen/src/x86/Sha1Operands.cpp b/core/relinker/codegen/src/x86/Sha1Operands.cpp index 950267e12..69ea10a68 100644 --- a/core/relinker/codegen/src/x86/Sha1Operands.cpp +++ b/core/relinker/codegen/src/x86/Sha1Operands.cpp @@ -67,6 +67,8 @@ Sha1Operands DecodeSha1(const std::uint8_t* data, const std::size_t length) { } operands.Function = static_cast(data[immediateOffset] & 3); } + if (operands.Memory && operands.Memory->RipRelative && immediateOffset + (operands.Operation == Sha1Operation::Rnds4 ? 1 : 0) != length) + throw CodegenException("RIP-relative SHA-1 operand does not end the instruction"); return operands; } diff --git a/core/relinker/codegen/src/x86/Sha256Lowering.cpp b/core/relinker/codegen/src/x86/Sha256Lowering.cpp index 462436d0b..b11f652e7 100644 --- a/core/relinker/codegen/src/x86/Sha256Lowering.cpp +++ b/core/relinker/codegen/src/x86/Sha256Lowering.cpp @@ -205,10 +205,9 @@ void Sha256Lowering::EmitOutOfLine(StubBodyBuilder& body, const Sha256Operands& } } -LoweredBody Sha256Lowering::LowerOutOfLine(const Sha256Operands& operands, std::span trailing) const { +LoweredBody Sha256Lowering::LowerOutOfLine(const Sha256Operands& operands) const { StubBodyBuilder body; EmitOutOfLine(body, operands); - body.Raw(trailing); return body.Finish(); } diff --git a/core/relinker/codegen/src/x86/Sha256Operands.cpp b/core/relinker/codegen/src/x86/Sha256Operands.cpp index 625d1d295..a090514a1 100644 --- a/core/relinker/codegen/src/x86/Sha256Operands.cpp +++ b/core/relinker/codegen/src/x86/Sha256Operands.cpp @@ -57,6 +57,8 @@ Sha256Operands DecodeSha256(const std::uint8_t* data, const std::size_t length) operands.Destination = static_cast(((modrm >> ModRmRegShift) & ModRmRegMask) | (((rex & 0x4) != 0) ? 8 : 0)); if (((modrm >> ModRmModShift) & ModRmModMask) != ModRmModRegister) { operands.Memory = DecodeMemoryOperand(data, length, pos + 3, rex, std::move(prefixes)); + if (operands.Memory->RipRelative && pos + 3 + operands.Memory->EncodedSize != length) + throw CodegenException("RIP-relative SHA-256 operand does not end the instruction"); return operands; } operands.Source = static_cast((modrm & ModRmRmMask) | (((rex & 0x1) != 0) ? 8 : 0)); diff --git a/core/relinker/codegen/src/x86/Sse4aLowering.cpp b/core/relinker/codegen/src/x86/Sse4aLowering.cpp index 347231750..156cabc50 100644 --- a/core/relinker/codegen/src/x86/Sse4aLowering.cpp +++ b/core/relinker/codegen/src/x86/Sse4aLowering.cpp @@ -196,15 +196,14 @@ void Sse4aLowering::EmitOutOfLine(StubBodyBuilder& body, const Sse4aOperands& op _emitOutOfLine(body, operands); } -LoweredBody Sse4aLowering::LowerOutOfLine(const Sse4aOperands& operands, std::span trailing) const { - return LowerOutOfLine(std::span(&operands, 1), trailing); +LoweredBody Sse4aLowering::LowerOutOfLine(const Sse4aOperands& operands) const { + return LowerOutOfLine(std::span(&operands, 1)); } -LoweredBody Sse4aLowering::LowerOutOfLine(std::span sequence, std::span trailing) const { +LoweredBody Sse4aLowering::LowerOutOfLine(std::span sequence) const { StubBodyBuilder body; for (const auto& operands : sequence) _emitOutOfLine(body, operands); - body.Raw(trailing); return body.Finish(); } diff --git a/core/relinker/codegen/src/x86/StubBodyBuilder.cpp b/core/relinker/codegen/src/x86/StubBodyBuilder.cpp index 7828b9691..a1b6e44b6 100644 --- a/core/relinker/codegen/src/x86/StubBodyBuilder.cpp +++ b/core/relinker/codegen/src/x86/StubBodyBuilder.cpp @@ -41,7 +41,7 @@ std::size_t _displacementSize(const std::uint8_t mod, const std::uint8_t rm, con using namespace X64OpcodeConstants; if (mod == ModRmModDisp8) return Disp8Size; - if (mod == ModRmModDisp32 || (mod == ModRmModIndirect && rm == ModRmRmSibPresent && (sib & SibBaseMask) == SibBaseDisp32)) + if (mod == ModRmModDisp32 || (mod == ModRmModIndirect && rm == ModRmRmRipRelative) || (mod == ModRmModIndirect && rm == ModRmRmSibPresent && (sib & SibBaseMask) == SibBaseDisp32)) return Disp32Size; return 0; } @@ -58,11 +58,10 @@ MemoryOperand DecodeMemoryOperand(const std::uint8_t* data, const std::size_t le if (modRmOffset >= length) throw CodegenException("Memory operand truncated before its ModRM byte"); const auto modrm = data[modRmOffset]; - MemoryOperand operand{std::move(prefixes), static_cast(rex & (kRexX | kRexB)), static_cast((modrm >> ModRmModShift) & ModRmModMask), static_cast(modrm & ModRmRmMask), 0, 0, false, 0}; + MemoryOperand operand{std::move(prefixes), static_cast(rex & (kRexX | kRexB)), static_cast((modrm >> ModRmModShift) & ModRmModMask), static_cast(modrm & ModRmRmMask), 0, 0, false, 0, false, length}; if (operand.Mod == ModRmModRegister) throw CodegenException("Register operand where a memory operand was expected"); - if (operand.Mod == ModRmModIndirect && operand.Rm == ModRmRmRipRelative) - throw CodegenException("RIP-relative memory operand cannot move into a stub"); + operand.RipRelative = operand.Mod == ModRmModIndirect && operand.Rm == ModRmRmRipRelative; auto pos = modRmOffset + 1; if (operand.Rm == ModRmRmSibPresent) { if (pos >= length) @@ -81,6 +80,21 @@ MemoryOperand DecodeMemoryOperand(const std::uint8_t* data, const std::size_t le return operand; } +void ApplyStubRelocations(const std::span body, const std::span relocations, const std::uint64_t siteAddress, const std::uint64_t stubAddress, const Domain::FileByteOffset failureOffset) { + using namespace X64OpcodeConstants; + for (const auto& relocation : relocations) { + if (relocation.InstructionEnd > body.size() || relocation.DisplacementOffset > relocation.InstructionEnd || relocation.InstructionEnd - relocation.DisplacementOffset < Disp32Size) + throw CodegenException("Stub relocation lies outside the stub body", failureOffset); + const auto target = siteAddress + static_cast(relocation.SiteTarget); + const auto displacement = static_cast(target - (stubAddress + relocation.InstructionEnd)); + if (displacement < std::numeric_limits::min() || displacement > std::numeric_limits::max()) + throw CodegenException("RIP-relative operand is out of rel32 range from its stub", failureOffset); + const auto value = static_cast(static_cast(displacement)); + for (std::size_t index = 0; index < Disp32Size; ++index) + body[relocation.DisplacementOffset + index] = static_cast(value >> (index * 8)); + } +} + void EmitSse(std::vector& out, const std::uint8_t prefix, const std::initializer_list opcode, const std::uint8_t dst, const std::uint8_t src) { _emit(out, prefix, dst, src, opcode, static_cast(kModRmRegister | ((dst & 7) << 3) | (src & 7))); } @@ -135,6 +149,11 @@ void StubBodyBuilder::Load(const std::uint8_t reg, const MemoryOperand& operand) _bytes.insert(_bytes.end(), {TwoByteOpcodeEscape, kMovdqu, static_cast((mod << ModRmModShift) | ((reg & 7) << ModRmRegShift) | operand.Rm)}); if (operand.Rm == ModRmRmSibPresent) _bytes.push_back(operand.Sib); + if (operand.RipRelative) { + _relocations.push_back({_bytes.size(), _bytes.size() + Disp32Size, static_cast(_siteOffset + operand.InstructionLength) + operand.Displacement}); + _bytes.insert(_bytes.end(), Disp32Size, 0); + return; + } const auto value = static_cast(static_cast(displacement)); for (std::size_t index = 0; index < _displacementSize(mod, operand.Rm, operand.Sib); ++index) _bytes.push_back(static_cast(value >> (index * 8))); @@ -158,6 +177,27 @@ void StubBodyBuilder::Raw(std::span bytes) { _bytes.insert(_bytes.end(), bytes.begin(), bytes.end()); } +void StubBodyBuilder::Move(const std::span instruction, const std::optional ripDisplacementOffset) { + using namespace X64OpcodeConstants; + const auto start = _bytes.size(); + Raw(instruction); + if (ripDisplacementOffset) { + if (*ripDisplacementOffset > instruction.size() || instruction.size() - *ripDisplacementOffset < Disp32Size) + throw CodegenException("RIP-relative displacement lies outside its instruction"); + std::uint32_t value = 0; + for (std::size_t index = 0; index < Disp32Size; ++index) { + value |= static_cast(instruction[*ripDisplacementOffset + index]) << (index * 8); + _bytes[start + *ripDisplacementOffset + index] = 0; + } + _relocations.push_back({start + *ripDisplacementOffset, start + instruction.size(), static_cast(_siteOffset + instruction.size()) + static_cast(value)}); + } + Advance(instruction.size()); +} + +void StubBodyBuilder::Advance(const std::size_t length) { + _siteOffset += length; +} + LoweredBody StubBodyBuilder::Finish() { const auto returnBranchOffset = _bytes.size(); _bytes.insert(_bytes.end(), kJmpRel32.Bytes, kJmpRel32.Bytes + kJmpRel32.Size); @@ -174,7 +214,7 @@ LoweredBody StubBodyBuilder::Finish() { for (std::size_t index = 0; index < 4; ++index) _bytes[fixup.DisplacementOffset + index] = static_cast(value >> (index * 8)); } - return {std::move(_bytes), returnBranchOffset}; + return {std::move(_bytes), returnBranchOffset, std::move(_relocations)}; } } diff --git a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp index 065c7f9dd..7a0b2a001 100644 --- a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp +++ b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp @@ -12,6 +12,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t bool rexPresent = false; std::uint8_t rex = 0; bool operandSizeOverride = false; + bool addressSizeOverride = false; bool repnePrefix = false; while (pos < available) { @@ -28,7 +29,9 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t repnePrefix = true; } else if (b == PrefixOperandSize) { operandSizeOverride = true; - } else if (b != PrefixLock && b != PrefixRep && b != PrefixAddressSize && + } else if (b == PrefixAddressSize) { + addressSizeOverride = true; + } else if (b != PrefixLock && b != PrefixRep && b != PrefixSegCs && b != PrefixSegSs && b != PrefixSegDs && b != PrefixSegEs && b != PrefixSegFs && b != PrefixSegGs) { break; @@ -39,6 +42,9 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t pos += 1; } + const bool rexW = rexPresent && (rex & RexWBit) != 0; + const std::size_t operandImmediateSize = operandSizeOverride && !rexW ? ImmSize16 : ImmSize32; + if (pos >= available) { throw CodegenException("Instruction truncated after prefixes"); } @@ -172,10 +178,9 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t opcode == OneByteTestEaxImm32 || (opcode >= OneByteMovImm32RegMin && opcode <= OneByteMovImm32RegMax)) { if (opcode >= OneByteMovImm32RegMin && opcode <= OneByteMovImm32RegMax) { - immediateSize = (rexPresent && (rex & RexWBit) != 0) ? ImmSize64 : - (operandSizeOverride ? ImmSize16 : ImmSize32); + immediateSize = rexW ? ImmSize64 : operandImmediateSize; } else { - immediateSize = operandSizeOverride ? ImmSize16 : ImmSize32; + immediateSize = operandImmediateSize; } } else if (opcode == OneByteImm8Grp1 || opcode == OneByteImulRm32Imm8) { immediateSize = ImmSize8; @@ -203,6 +208,10 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t if (opcode == OneByteEnter) { immediateSize = ImmSizeEnter; } + + if (opcode >= OneByteMovMoffsMin && opcode <= OneByteMovMoffsMax) { + immediateSize = addressSizeOverride ? ImmSize32 : ImmSize64; + } } else { if (opcode == TwoByteExtrqInsertqImm8Imm8) { if (!operandSizeOverride && !repnePrefix) { @@ -237,6 +246,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t hasModRm = true; } else if ((opcode >= TwoByteCmovRangeMin && opcode <= TwoByteCmovRangeMax) || (opcode >= TwoByteModRmRangeAMin && opcode <= TwoByteModRmRangeAMax) || + opcode == TwoByteBts || (opcode >= TwoByteModRmRangeBMin && opcode <= TwoByteModRmRangeBMax) || (opcode >= TwoByteModRmRangeCMin && opcode <= TwoByteModRmRangeCMax) || (opcode >= TwoByteModRmRangeDMin && opcode <= TwoByteModRmRangeDMax) || @@ -270,16 +280,17 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t const std::uint8_t modrm = data[pos]; pos += 1; - const auto mod = static_cast((modrm >> ModRmModShift) & ModRmModMask); + const bool registerOnlyModRm = twoByteOpcode && !threeByteEscape && opcode >= TwoByteMovCrDrMin && opcode <= TwoByteMovCrDrMax; + const auto mod = registerOnlyModRm ? ModRmModRegister : static_cast((modrm >> ModRmModShift) & ModRmModMask); const auto reg = static_cast((modrm >> ModRmRegShift) & ModRmRegMask); const auto rm = static_cast(modrm & ModRmRmMask); - if (!vexPresent && !twoByteOpcode && + if (!vexPresent && !evexPresent && !twoByteOpcode && (opcode == OneByteTestGrp3Rm8 || opcode == OneByteTestGrp3Rm) && reg <= Grp3RegTestMax) { immediateSize = (opcode == OneByteTestGrp3Rm8) ? ImmSize8 - : (operandSizeOverride ? ImmSize16 : ImmSize32); + : operandImmediateSize; } if (mod != ModRmModRegister && rm == ModRmRmSibPresent) { @@ -352,12 +363,12 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( if (pos >= info.Length) return info; - const auto readModRm = [&](const std::size_t modrmPos) { + const auto readModRm = [&](const std::size_t modrmPos, const bool registerOnly) { const std::uint8_t modrm = data[modrmPos]; info.HasModRm = true; info.ModRmByte = modrm; info.ModRmRegField = (modrm >> ModRmRegShift) & ModRmRegMask; - const std::uint8_t mod = (modrm >> ModRmModShift) & ModRmModMask; + const std::uint8_t mod = registerOnly ? ModRmModRegister : (modrm >> ModRmModShift) & ModRmModMask; const std::uint8_t rm = modrm & ModRmRmMask; if (mod == ModRmModIndirect && rm == ModRmRmRipRelative) { info.HasRipRelativeDisp = true; @@ -371,7 +382,7 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( if (op == OneByteVex2 || op == OneByteVex3 || op == EvexPrefix) { pos += (op == OneByteVex2 ? 1 : (op == OneByteVex3 ? 2 : EvexPrefixLength - 1)) + 1; if (pos < info.Length) - readModRm(pos); + readModRm(pos, false); info.FlowKind = ControlFlowKind::Sequential; return info; } @@ -470,6 +481,12 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( info.FlowKind = ControlFlowKind::Trap; return info; } + if (op == TwoByteUd1) { + if (pos < info.Length) + readModRm(pos, false); + info.FlowKind = ControlFlowKind::Trap; + return info; + } if (op >= TwoByteJccRel32Min && op <= TwoByteJccRel32Max) { std::int32_t disp = 0; std::memcpy(&disp, data + pos, 4); @@ -507,6 +524,7 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( } else { if ((op >= TwoByteCmovRangeMin && op <= TwoByteCmovRangeMax) || (op >= TwoByteModRmRangeAMin && op <= TwoByteModRmRangeAMax) || + op == TwoByteBts || (op >= TwoByteModRmRangeBMin && op <= TwoByteModRmRangeBMax) || (op >= TwoByteModRmRangeCMin && op <= TwoByteModRmRangeCMax) || (op >= TwoByteModRmRangeDMin && op <= TwoByteModRmRangeDMax) || @@ -535,7 +553,7 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( } if (hasModRm && pos < info.Length) - readModRm(pos); + readModRm(pos, twoByteOpcode && op >= TwoByteMovCrDrMin && op <= TwoByteMovCrDrMax); info.FlowKind = ControlFlowKind::Sequential; return info; diff --git a/core/relinker/codegen/src/x86/X64InstructionRewriter.cpp b/core/relinker/codegen/src/x86/X64InstructionRewriter.cpp index 0d012bea4..a3899c3dd 100644 --- a/core/relinker/codegen/src/x86/X64InstructionRewriter.cpp +++ b/core/relinker/codegen/src/x86/X64InstructionRewriter.cpp @@ -14,78 +14,20 @@ X64InstructionRewriter::ReferenceSite X64InstructionRewriter::ClassifyInstructio const std::uint8_t* data, std::size_t length ) { - std::size_t pos = 0; + const auto info = X64InstructionDecoder{}.DecodeInstruction(data, length); - while (pos < length) { - const std::uint8_t b = data[pos]; - if ((b >= RexMin && b <= RexMax) || - b == PrefixLock || b == PrefixRepne || b == PrefixRep || - b == PrefixSegCs || b == PrefixSegSs || b == PrefixSegDs || - b == PrefixSegEs || b == PrefixSegFs || b == PrefixSegGs || - b == PrefixOperandSize || b == PrefixAddressSize) { - pos += 1; - continue; - } - break; + if (info.HasRipRelativeDisp) { + return {ReferenceKind::RipRelativeDisp32, info.RipRelativeDispOffset}; } - if (pos >= length) { + if (!info.HasBranchTarget) { return {ReferenceKind::None, 0}; } - std::uint8_t opcode = data[pos]; - const std::size_t opcodeOffset = pos; - pos += 1; - bool twoByteOpcode = false; - - if (opcode == TwoByteOpcodeEscape) { - if (pos >= length) { - return {ReferenceKind::None, 0}; - } - twoByteOpcode = true; - opcode = data[pos]; - pos += 1; - } - - if (!twoByteOpcode) { - if (opcode == OneByteCallRel32 || opcode == OneByteJmpRel32) { - if (length >= opcodeOffset + Rel32InstructionLength) { - return {ReferenceKind::Rel32Branch, pos}; - } - return {ReferenceKind::None, 0}; - } - - if (opcode == OneByteJmpRel8 || (opcode >= OneByteJccRel8Min && opcode <= OneByteJccRel8Max)) { - if (length >= pos + Disp8Size) { - return {ReferenceKind::Rel8Branch, pos}; - } - return {ReferenceKind::None, 0}; - } - } else { - if (opcode >= TwoByteJccRel32Min && opcode <= TwoByteJccRel32Max) { - if (length >= pos + Disp32Size) { - return {ReferenceKind::Rel32Branch, pos}; - } - return {ReferenceKind::None, 0}; - } - } - - if (pos >= length) { - return {ReferenceKind::None, 0}; - } - - const std::uint8_t modrm = data[pos]; - const std::uint8_t mod = static_cast((modrm >> ModRmModShift) & ModRmModMask); - const std::uint8_t rm = static_cast(modrm & ModRmRmMask); - - if (mod == ModRmModIndirect && rm == ModRmRmRipRelative) { - const std::size_t dispOffset = pos + 1; - if (length >= dispOffset + Disp32Size) { - return {ReferenceKind::RipRelativeDisp32, dispOffset}; - } - } - - return {ReferenceKind::None, 0}; + const bool rel32 = info.IsTwoByteOpcode || info.Opcode == OneByteCallRel32 || info.Opcode == OneByteJmpRel32; + return rel32 + ? ReferenceSite{ReferenceKind::Rel32Branch, info.Length - Disp32Size} + : ReferenceSite{ReferenceKind::Rel8Branch, info.Length - Disp8Size}; } std::int32_t X64InstructionRewriter::ReadInt32(const std::uint8_t* data) { diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index c4fab89f4..c3e4a1da2 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -124,9 +125,16 @@ void decoderLengths() { {0x66, 0x0F, 0xC4, 0xC0, 0x01}, {0xC2, 0x08, 0x00}, {0xC8, 0x10, 0x00, 0x00}, {0xF3, 0x0F, 0x2B, 0x07}, {0xF2, 0x44, 0x0F, 0x2B, 0x4C, 0x24, 0x10}, {0x0F, 0x01, 0xFA}, {0x0F, 0xB9, 0x00}, {0x41, 0x0F, 0xBB, 0xF7}, {0x0F, 0xBB, 0x47, 0x08}, + {0xA0, 1, 2, 3, 4, 5, 6, 7, 8}, {0x48, 0xA1, 1, 2, 3, 4, 5, 6, 7, 8}, {0xA2, 1, 2, 3, 4, 5, 6, 7, 8}, + {0x64, 0x48, 0xA3, 1, 2, 3, 4, 5, 6, 7, 8}, {0x67, 0xA1, 1, 2, 3, 4}, {0x48, 0x67, 0xA3, 1, 2, 3, 4}, {0x0F, 0x38, 0xCB, 0xCA}, {0x45, 0x0F, 0x38, 0xCC, 0xE1}, {0x0F, 0x38, 0xCD, 0x08}, {0x0F, 0x38, 0xCB, 0x0D, 0x10, 0x00, 0x00, 0x00}, + {0x62, 0xF1, 0x75, 0x48, 0xF6, 0xC2}, {0x62, 0xF1, 0x7D, 0x48, 0xF6, 0x05, 0x10, 0x00, 0x00, 0x00}, {0x62, 0xF1, 0x6D, 0x48, 0xF6, 0x4C, 0x24, 0x01}, {0x0F, 0x3A, 0xCC, 0xD5, 0x03}, {0x45, 0x0F, 0x3A, 0xCC, 0xE1, 0x00}, {0x0F, 0x3A, 0xCC, 0x08, 0x01}, {0x0F, 0x38, 0xC8, 0xCA}, {0x0F, 0x38, 0xC9, 0xCA}, {0x0F, 0x38, 0xCA, 0x08}, {0x48, 0x66, 0xB8, 0x34, 0x12}, {0x66, 0x48, 0xB8, 1, 2, 3, 4, 5, 6, 7, 8}, {0x41, 0x48, 0xB8, 1, 2, 3, 4, 5, 6, 7, 8}, + {0x66, 0x48, 0x81, 0xC0, 1, 2, 3, 4}, {0x66, 0x48, 0x05, 1, 2, 3, 4}, {0x66, 0x48, 0xC7, 0xC0, 1, 2, 3, 4}, + {0x66, 0x48, 0x69, 0xC0, 1, 2, 3, 4}, {0x66, 0x48, 0x68, 1, 2, 3, 4}, {0x66, 0x48, 0xA9, 1, 2, 3, 4}, + {0x66, 0x48, 0xF7, 0xC0, 1, 2, 3, 4}, {0x66, 0x81, 0xC0, 0x34, 0x12}, {0x48, 0x66, 0x81, 0xC0, 0x34, 0x12}, + {0x66, 0x41, 0x81, 0xC0, 0x34, 0x12}, {0x66, 0xF7, 0xC0, 0x34, 0x12}, {0x48, 0x64, 0x8B, 0x00}, {0x48, 0xF3, 0x0F, 0x2B, 0x00}, {0x48, 0x67, 0x0F, 0x01, 0xFC}, {0x48, 0x48, 0x0F, 0x01, 0xFC}}; Bytes padded; for (const auto& instruction : instructions) { @@ -160,6 +168,36 @@ void decoderRipRelative() { } for (const auto& instruction : kRegisterVectorOperations) require(!decoder.DecodeInstruction(instruction.data(), instruction.size()).HasRipRelativeDisp, "Vector instruction without a RIP-relative operand was reported as RIP-relative"); + const Bytes ud1 = {0x0F, 0xB9, 0x05, 0x10, 0x00, 0x00, 0x00}; + const auto trap = decoder.DecodeInstruction(ud1.data(), ud1.size()); + require(trap.Length == ud1.size() && trap.FlowKind == Codegen::ControlFlowKind::Trap && trap.HasRipRelativeDisp && read(ud1, trap.RipRelativeDispOffset) == 0x10, "UD1 was not decoded as a trap with its RIP-relative operand"); +} + +void decoderTwoByteOpcodeLengths() { + const Codegen::X64InstructionDecoder decoder; + const std::vector instructions = { + {0x0F, 0xA8}, {0x0F, 0xA9}, {0x0F, 0xAA}, {0x66, 0x0F, 0xA8}, {0x41, 0x0F, 0xA9}, + {0x0F, 0xAB, 0xC8}, {0x0F, 0xAB, 0x05, 0x10, 0x00, 0x00, 0x00}, {0x48, 0x0F, 0xAB, 0x44, 0x24, 0x08}, + {0x0F, 0x20, 0xC0}, {0x0F, 0x20, 0x05}, {0x0F, 0x22, 0x04}, {0x0F, 0x21, 0x45}, {0x0F, 0x23, 0x85}, + {0x66, 0x0F, 0x38, 0x20, 0x05, 0x10, 0x00, 0x00, 0x00}, {0x66, 0x0F, 0x38, 0x23, 0x44, 0x24, 0x08}}; + Bytes padded; + for (const auto& instruction : instructions) { + padded = instruction; + padded.insert(padded.end(), 8, 0x90); + require(decoder.Decode(padded.data(), padded.size()) == instruction.size(), "Two-byte opcode was decoded with operand bytes it does not have"); + require(decoder.DecodeInstruction(padded.data(), padded.size()).Length == instruction.size(), "Two-byte opcode was described with operand bytes it does not have"); + } + const Bytes controlRegister = {0x0F, 0x20, 0x05, 0x90, 0x90, 0x90, 0x90}; + const auto control = decoder.DecodeInstruction(controlRegister.data(), controlRegister.size()); + require(control.Length == 3 && control.HasModRm && control.ModRmByte == 0x05 && !control.HasRipRelativeDisp, "MOV from a control register was reported as RIP-relative"); + const Bytes debugRegister = {0x0F, 0x23, 0x05, 0x90, 0x90, 0x90, 0x90}; + require(!decoder.DecodeInstruction(debugRegister.data(), debugRegister.size()).HasRipRelativeDisp, "MOV to a debug register was reported as RIP-relative"); + const Bytes extend = {0x66, 0x0F, 0x38, 0x20, 0x05, 0x10, 0x00, 0x00, 0x00}; + const auto extended = decoder.DecodeInstruction(extend.data(), extend.size()); + require(extended.HasRipRelativeDisp && read(extend, extended.RipRelativeDispOffset) == 0x10, "PMOVSXBW lost its RIP-relative operand"); + const Bytes bitTest = {0x0F, 0xAB, 0x05, 0x10, 0x00, 0x00, 0x00}; + const auto bts = decoder.DecodeInstruction(bitTest.data(), bitTest.size()); + require(bts.HasRipRelativeDisp && read(bitTest, bts.RipRelativeDispOffset) == 0x10, "BTS lost its RIP-relative operand"); } void sse4aOperands() { @@ -201,7 +239,11 @@ void sha256Operands() { const Bytes absolute = {0x0F, 0x38, 0xCC, 0x14, 0x25, 0x78, 0x56, 0x34, 0x12}; const auto noBase = Codegen::DecodeSha256(absolute.data(), absolute.size()); require(!noBase.Memory->StackBase && noBase.Memory->Displacement == 0x12345678, "SIB without a base was decoded incorrectly"); - requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xCC, 0x15, 0, 0, 0, 0}; (void)Codegen::DecodeSha256(bytes.data(), bytes.size()); }, "SHA-256 RIP-relative operand was accepted"); + const Bytes ripRelative = {0x0F, 0x38, 0xCC, 0x15, 0x78, 0x56, 0x34, 0x12}; + const auto rip = Codegen::DecodeSha256(ripRelative.data(), ripRelative.size()); + require(rip.Destination == 2 && rip.Memory && rip.Memory->RipRelative && rip.Memory->Displacement == 0x12345678 && rip.Memory->EncodedSize == 5 && rip.Memory->InstructionLength == 8 && !noBase.Memory->RipRelative, "SHA-256 RIP-relative operand was decoded incorrectly"); + requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xCC, 0x15, 0, 0, 0, 0, 0x90}; (void)Codegen::DecodeSha256(bytes.data(), bytes.size()); }, "SHA-256 RIP-relative operand that does not end the instruction was accepted"); + requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xCC, 0x15, 0, 0, 0}; (void)Codegen::DecodeSha256(bytes.data(), bytes.size()); }, "Truncated SHA-256 RIP-relative operand was accepted"); requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xCC, 0x54, 0x24}; (void)Codegen::DecodeSha256(bytes.data(), bytes.size()); }, "Truncated SHA-256 memory operand was accepted"); requireFailure([] { const Bytes bytes = {0x66, 0x0F, 0x38, 0xCB, 0xCA}; (void)Codegen::DecodeSha256(bytes.data(), bytes.size()); }, "Prefixed 0F 38 CB was decoded as SHA-256"); requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xC9, 0xCA}; (void)Codegen::DecodeSha256(bytes.data(), bytes.size()); }, "SHA-1 was decoded as SHA-256"); @@ -236,8 +278,14 @@ void sha1Operands() { const Bytes message = {0x0F, 0x38, 0xC9, 0x08}; const auto decodedMessage = Codegen::DecodeSha1(message.data(), message.size()); require(decodedMessage.Operation == Codegen::Sha1Operation::Msg1 && decodedMessage.Destination == 1 && decodedMessage.Memory && decodedMessage.Memory->Mod == 0 && decodedMessage.Memory->Rm == 0, "SHA1MSG1 memory operand was decoded incorrectly"); - requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xC9, 0x15, 0, 0, 0, 0}; (void)Codegen::DecodeSha1(bytes.data(), bytes.size()); }, "SHA-1 RIP-relative operand was accepted"); - requireFailure([] { const Bytes bytes = {0x0F, 0x3A, 0xCC, 0x15, 0, 0, 0, 0, 0x00}; (void)Codegen::DecodeSha1(bytes.data(), bytes.size()); }, "SHA1RNDS4 RIP-relative operand was accepted"); + const Bytes ripMessage = {0x0F, 0x38, 0xC9, 0x15, 0x10, 0x00, 0x00, 0x00}; + const auto ripDecoded = Codegen::DecodeSha1(ripMessage.data(), ripMessage.size()); + require(ripDecoded.Operation == Codegen::Sha1Operation::Msg1 && ripDecoded.Destination == 2 && ripDecoded.Memory && ripDecoded.Memory->RipRelative && ripDecoded.Memory->Displacement == 0x10 && ripDecoded.Memory->InstructionLength == 8, "SHA-1 RIP-relative operand was decoded incorrectly"); + const Bytes ripRounds = {0x0F, 0x3A, 0xCC, 0x15, 0xF0, 0xFF, 0xFF, 0xFF, 0x03}; + const auto ripRoundsDecoded = Codegen::DecodeSha1(ripRounds.data(), ripRounds.size()); + require(ripRoundsDecoded.Operation == Codegen::Sha1Operation::Rnds4 && ripRoundsDecoded.Function == 3 && ripRoundsDecoded.Memory && ripRoundsDecoded.Memory->RipRelative && ripRoundsDecoded.Memory->Displacement == -16 && ripRoundsDecoded.Memory->InstructionLength == 9, "SHA1RNDS4 RIP-relative operand was decoded incorrectly"); + requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xC9, 0x15, 0, 0, 0, 0, 0x90}; (void)Codegen::DecodeSha1(bytes.data(), bytes.size()); }, "SHA-1 RIP-relative operand that does not end the instruction was accepted"); + requireFailure([] { const Bytes bytes = {0x0F, 0x3A, 0xCC, 0x15, 0, 0, 0, 0, 0x00, 0x90}; (void)Codegen::DecodeSha1(bytes.data(), bytes.size()); }, "SHA1RNDS4 RIP-relative operand that does not end the instruction was accepted"); requireFailure([] { const Bytes bytes = {0x0F, 0x38, 0xC9, 0x54, 0x24}; (void)Codegen::DecodeSha1(bytes.data(), bytes.size()); }, "Truncated SHA-1 memory operand was accepted"); requireFailure([] { const Bytes bytes = {0x0F, 0x3A, 0xCC, 0xCA}; (void)Codegen::DecodeSha1(bytes.data(), bytes.size()); }, "SHA1RNDS4 without its immediate was accepted"); requireFailure([] { const Bytes bytes = {0x0F, 0x3A, 0xCC, 0x50, 0x02}; (void)Codegen::DecodeSha1(bytes.data(), bytes.size()); }, "SHA1RNDS4 memory form without its immediate was accepted"); @@ -359,6 +407,49 @@ void goldenBodies() { require(insertqRegisterForm.Bytes[0] == 0x48 && insertqRegisterForm.Bytes.size() % 16 == 0 && insertqRegisterForm.ReturnBranchOffset < insertqRegisterForm.Bytes.size(), "INSERTQ register form body does not start with the red-zone skip"); } +void ripRelativeBodies() { + const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(); + const auto relocated = [](const Codegen::Amd64OnlyMatch& match, const std::size_t displacementOffset, const std::size_t instructionEnd, const std::int64_t siteTarget) { + return match.Relocations.size() == 1 && match.Relocations[0].DisplacementOffset == displacementOffset && match.Relocations[0].InstructionEnd == instructionEnd && match.Relocations[0].SiteTarget == siteTarget; + }; + const Bytes message = {0x0F, 0x38, 0xCC, 0x15, 0x78, 0x56, 0x34, 0x12}; + const auto sha256 = matcher->Match(message.data(), message.size()); + require(sha256 && sha256->StubBody.size() > 21 && Bytes(sha256->StubBody.begin() + 13, sha256->StubBody.begin() + 21) == Bytes{0xF3, 0x0F, 0x6F, 0x0D, 0x00, 0x00, 0x00, 0x00}, "SHA-256 RIP-relative operand was not reloaded through a RIP-relative MOVDQU"); + require(relocated(*sha256, 17, 21, 8 + 0x12345678), "SHA-256 RIP-relative operand has the wrong relocation"); + const Bytes prefixed = {0x65, 0x67, 0x0F, 0x38, 0xCC, 0x15, 0x78, 0x56, 0x34, 0x12}; + const auto segmentOverride = matcher->Match(prefixed.data(), prefixed.size()); + require(segmentOverride && segmentOverride->StubBody.size() > 23 && Bytes(segmentOverride->StubBody.begin() + 13, segmentOverride->StubBody.begin() + 23) == Bytes{0x65, 0x67, 0xF3, 0x0F, 0x6F, 0x0D, 0x00, 0x00, 0x00, 0x00}, "RIP-relative reload lost the segment or address-size prefix"); + require(relocated(*segmentOverride, 19, 23, 10 + 0x12345678), "Prefixed RIP-relative operand has the wrong relocation"); + const Bytes rounds = {0x0F, 0x3A, 0xCC, 0x15, 0xF0, 0xFF, 0xFF, 0xFF, 0x03}; + const auto sha1 = matcher->Match(rounds.data(), rounds.size()); + require(sha1 && sha1->StubBody.size() > 21 && Bytes(sha1->StubBody.begin() + 13, sha1->StubBody.begin() + 21) == Bytes{0xF3, 0x0F, 0x6F, 0x05, 0x00, 0x00, 0x00, 0x00}, "SHA1RNDS4 RIP-relative operand was not reloaded through a RIP-relative MOVDQU"); + require(relocated(*sha1, 17, 21, 9 - 16), "SHA1RNDS4 RIP-relative operand does not count its immediate"); + const Bytes extrq = {0x66, 0x0F, 0x79, 0xCA}; + const Bytes followers = {0x90, 0x48, 0x8D, 0x05, 0x10, 0x00, 0x00, 0x00}; + const auto moved = matcher->Match(extrq.data(), extrq.size(), followers); + require(moved && moved->ReturnBranchOffset >= followers.size(), "EXTRQ with followers was not lowered through a stub"); + const auto start = moved->ReturnBranchOffset - followers.size(); + require(Bytes(moved->StubBody.begin() + static_cast(start), moved->StubBody.begin() + static_cast(moved->ReturnBranchOffset)) == Bytes{0x90, 0x48, 0x8D, 0x05, 0x00, 0x00, 0x00, 0x00}, "Moved RIP-relative LEA was not copied with a cleared displacement"); + require(relocated(*moved, start + 4, start + 8, 4 + 8 + 0x10), "Moved RIP-relative LEA has the wrong relocation"); + const std::vector> sequence = {extrq, message}; + const auto pair = matcher->MatchSequence(sequence, {}); + require(pair && pair->Relocations.size() == 1 && pair->Relocations[0].SiteTarget == 4 + 8 + 0x12345678, "RIP-relative operand in a sequence is not relative to the site start"); +} + +void stubRelocations() { + Bytes body(8, 0xCC); + const std::vector relocations = {{2, 6, 0x1040}}; + Codegen::ApplyStubRelocations(body, relocations, 0x401000, 0x400F00, 0); + require(read(body, 2) == 0x401000 + 0x1040 - (0x400F00 + 6) && body[0] == 0xCC && body[6] == 0xCC, "Stub relocation was not resolved against the site address"); + Codegen::ApplyStubRelocations(body, relocations, 0x1000, 0x7FFF0000, 0); + require(read(body, 2) == 0x1000 + 0x1040 - (0x7FFF0000 + 6), "Stub placed above its site was not relocated backwards"); + requireFailure([&] { Codegen::ApplyStubRelocations(body, relocations, 0x1000, 0x1000 + 0x90000000ull, 0); }, "RIP-relative operand beyond rel32 range was accepted"); + const std::vector outside = {{6, 10, 0}}; + requireFailure([&] { Codegen::ApplyStubRelocations(body, outside, 0x1000, 0x2000, 0); }, "Stub relocation outside the body was accepted"); + const std::vector narrow = {{4, 6, 0}}; + requireFailure([&] { Codegen::ApplyStubRelocations(body, narrow, 0x1000, 0x2000, 0); }, "Stub relocation narrower than a displacement was accepted"); +} + Bytes segmentFixture() { Bytes file(0x300, 0xCC); const Bytes text = { @@ -425,7 +516,16 @@ void converterRipRelativeFollower() { }; for (const auto& following : kRipRelativeVectorLoads) { const auto [file, header] = withFollower(following); - require(failureOffset([&] { (void)converter->Convert(file, {header}); }, "RIP-relative vector load was moved into an EXTRQ stub") == 0x204, "RIP-relative follower failure does not carry its file offset"); + const auto result = converter->Convert(file, {header}); + require(result.Trampolines.size() == 1 && result.Trampolines[0].Length == 4 + following.size(), "RIP-relative vector load was not moved into the EXTRQ stub"); + const auto& site = result.Trampolines[0]; + const auto info = Codegen::X64InstructionDecoder{}.DecodeInstruction(following.data(), following.size()); + const auto start = site.ReturnBranchOffset - following.size(); + require(site.Relocations.size() == 1 && site.Relocations[0].DisplacementOffset == start + info.RipRelativeDispOffset && site.Relocations[0].InstructionEnd == site.ReturnBranchOffset && site.Relocations[0].SiteTarget == static_cast(4 + following.size()) + 0x10, "Moved RIP-relative vector load has the wrong relocation"); + } + for (const Bytes& following : {Bytes{0xFF, 0x15, 0x10, 0x00, 0x00, 0x00}, Bytes{0xFF, 0x25, 0x10, 0x00, 0x00, 0x00}}) { + const auto [file, header] = withFollower(following); + require(failureOffset([&] { (void)converter->Convert(file, {header}); }, "RIP-relative indirect branch was moved into an EXTRQ stub") == 0x204, "RIP-relative branch failure does not carry its file offset"); } for (const auto& following : kRegisterVectorOperations) { const auto [file, header] = withFollower(following); @@ -460,9 +560,12 @@ void converterSha256() { const auto memoryResult = converter->Convert(memoryForm, {segmentHeader(text.size())}); require(memoryResult.Trampolines.size() == 2 && memoryResult.Reports[1].InstructionName == "SHA256MSG2", "SHA-256 memory form was not lowered through a stub"); auto ripRelative = file; - const Bytes ripMessage = {0x0F, 0x38, 0xCD, 0x2D, 0x00, 0x00, 0x00, 0x00, 0xC3}; + const Bytes ripMessage = {0x0F, 0x38, 0xCD, 0x2D, 0x40, 0x00, 0x00, 0x00, 0xC3}; std::copy(ripMessage.begin(), ripMessage.end(), ripRelative.begin() + 0x20C); - require(failureOffset([&] { (void)converter->Convert(ripRelative, {segmentHeader(text.size() + 3)}); }, "SHA-256 RIP-relative form was accepted") == 0x20C, "SHA-256 operand failure does not carry the file offset"); + const auto ripResult = converter->Convert(ripRelative, {segmentHeader(text.size() + 3)}); + require(ripResult.Trampolines.size() == 2 && ripResult.Bytes == ripRelative && ripResult.Reports[1].InstructionName == "SHA256MSG2", "SHA-256 RIP-relative form was not lowered through a stub"); + const auto& ripSite = ripResult.Trampolines[1]; + require(ripSite.Offset == 0x20C && ripSite.Length == 8 && ripSite.Relocations.size() == 1 && ripSite.Relocations[0].SiteTarget == 8 + 0x40 && ripSite.Relocations[0].InstructionEnd <= ripSite.ReturnBranchOffset, "SHA-256 RIP-relative site was recorded without its relocation"); } void converterSha1() { @@ -492,9 +595,12 @@ void converterSha1() { const auto memoryResult = converter->Convert(memoryForm, {segmentHeader(text.size())}); require(memoryResult.Trampolines.size() == 3 && memoryResult.Reports[2].InstructionName == "SHA1MSG2", "SHA-1 memory form was not lowered through a stub"); auto ripRelative = file; - const Bytes ripMessage = {0x0F, 0x38, 0xCA, 0x2D, 0x00, 0x00, 0x00, 0x00, 0xC3}; + const Bytes ripMessage = {0x0F, 0x38, 0xCA, 0x2D, 0xF0, 0xFF, 0xFF, 0xFF, 0xC3}; std::copy(ripMessage.begin(), ripMessage.end(), ripRelative.begin() + 0x211); - require(failureOffset([&] { (void)converter->Convert(ripRelative, {segmentHeader(text.size() + 3)}); }, "SHA-1 RIP-relative form was accepted") == 0x211, "SHA-1 operand failure does not carry the file offset"); + const auto ripResult = converter->Convert(ripRelative, {segmentHeader(text.size() + 3)}); + require(ripResult.Trampolines.size() == 3 && ripResult.Bytes == ripRelative && ripResult.Reports[2].InstructionName == "SHA1MSG2", "SHA-1 RIP-relative form was not lowered through a stub"); + const auto& ripSite = ripResult.Trampolines[2]; + require(ripSite.Offset == 0x211 && ripSite.Length == 8 && ripSite.Relocations.size() == 1 && ripSite.Relocations[0].SiteTarget == 8 - 16 && ripSite.Relocations[0].InstructionEnd <= ripSite.ReturnBranchOffset, "SHA-1 RIP-relative site was recorded without its relocation"); } void converterMonitorWait() { @@ -580,8 +686,10 @@ void converterReciprocal() { auto ripRelative = file; const Bytes load = {0xC5, 0xF8, 0x52, 0xD5, 0x8B, 0x05, 0x10, 0x00, 0x00, 0x00, 0xC3}; std::copy(load.begin(), load.end(), ripRelative.begin() + 0x200); - const auto kept = converter->Convert(ripRelative, {segmentHeader(load.size())}); - require(kept.Trampolines.empty() && kept.KeptCount == 1 && kept.Bytes == ripRelative, "A following RIP-relative instruction was moved into a stub"); + const auto moved = converter->Convert(ripRelative, {segmentHeader(load.size())}); + require(moved.Trampolines.size() == 1 && moved.KeptCount == 0 && moved.Bytes == ripRelative && moved.Trampolines[0].Length == 10, "VRSQRTPS did not absorb the following RIP-relative load"); + const auto& movedSite = moved.Trampolines[0]; + require(movedSite.Relocations.size() == 1 && movedSite.Relocations[0].DisplacementOffset == movedSite.ReturnBranchOffset - 4 && movedSite.Relocations[0].InstructionEnd == movedSite.ReturnBranchOffset && movedSite.Relocations[0].SiteTarget == 10 + 0x10, "Absorbed RIP-relative load has the wrong relocation"); } void converterStrayRex() { @@ -618,6 +726,19 @@ void rewriterStrayRex() { require(rewritten.Bytes == expected, "Branch with a stray REX was not adjusted by a length-changing rewrite"); } +void rewriterReferenceSites() { + const std::vector> cases = { + {{0x90, 0x48, 0xB8, 0x05, 0xF0, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0xC3}, 0, + {0x66, 0x90, 0x48, 0xB8, 0x05, 0xF0, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0xC3}}, + {{0xC5, 0xF9, 0x6F, 0x05, 0x01, 0x00, 0x00, 0x00, 0x90, 0xC3}, 8, + {0xC5, 0xF9, 0x6F, 0x05, 0x02, 0x00, 0x00, 0x00, 0x66, 0x90, 0xC3}}, + {{0x66, 0x0F, 0x38, 0x00, 0x05, 0x01, 0x00, 0x00, 0x00, 0x90, 0xC3}, 9, + {0x66, 0x0F, 0x38, 0x00, 0x05, 0x02, 0x00, 0x00, 0x00, 0x66, 0x90, 0xC3}}, + {{0xE2, 0x01, 0x90, 0xC3}, 2, {0xE2, 0x02, 0x66, 0x90, 0xC3}}}; + for (const auto& [code, offset, expected] : cases) + require(Codegen::X64InstructionRewriter{}.Rewrite(code, {offset, {0x66, 0x90}}).Bytes == expected, "Length-changing rewrite adjusted an immediate or missed a VEX, three-byte or LOOP reference"); +} + void converterFailureOffsets() { const auto converter = Codegen::MakeAmd64OnlyConverter(); const auto file = segmentFixture(); @@ -675,35 +796,40 @@ std::vector elfHeaders() { return {{1, 5, 0x200, 0x1000, 0, 0x100, 0x100, 0x1000}, {1, 6, 0x300, 0x2000, 0, 0x100, 0x100, 0x1000}}; } -void linuxPlacement() { - const auto source = elfFixture({0xEB, 0x06, 0xF2, 0x0F, 0x78, 0xDB, 0x08, 0x08, 0xC3}); - const auto headers = elfHeaders(); - const auto converted = Codegen::MakeAmd64OnlyConverter()->Convert(source, {headers[0]}); - require(converted.Trampolines.size() == 1 && converted.Bytes == source, "Linux fixture conversion produced unexpected results"); +Elfpatcher::Linux::LinuxElfPatcher linuxPatcher() { const auto byteWriter = std::make_shared(); - Elfpatcher::Linux::LinuxElfPatcher patcher( + return Elfpatcher::Linux::LinuxElfPatcher( std::make_shared(), std::make_shared(std::make_shared(), byteWriter), std::make_shared(byteWriter), byteWriter); - const auto output = patcher.Patch(converted.Bytes, headers, {}, 0, "$ORIGIN/libs", true, false, converted.Trampolines); - require(output[0x202] == 0xE9 && output[0x207] == 0x90, "Linux site was not replaced by a jump"); - const auto target = 0x1002 + 5 + static_cast(read(output, 0x203)); - require(target % 16 == 0 && target > 0x2100, "Linux stub is misaligned or inside the original image"); +} + +std::uint64_t executableFileOffset(const Bytes& output, const std::uint64_t address) { const auto phNum = read(output, 56); - std::uint64_t bodyOffset = 0; - bool found = false; for (std::uint16_t index = 0; index < phNum; ++index) { const auto header = 64 + index * 56; if (read(output, header) != 1) continue; const auto vaddr = read(output, header + 16); const auto memSize = read(output, header + 40); - if (static_cast(target) < vaddr || static_cast(target) >= vaddr + memSize) continue; + if (address < vaddr || address >= vaddr + memSize) continue; require((read(output, header + 4) & 1) != 0, "Linux stub segment is not executable"); - bodyOffset = read(output, header + 8) + (static_cast(target) - vaddr); - found = true; + return read(output, header + 8) + (address - vaddr); } - require(found, "Linux stub is not inside a PT_LOAD segment"); + throw std::runtime_error("Linux stub is not inside a PT_LOAD segment"); +} + +void linuxPlacement() { + const auto source = elfFixture({0xEB, 0x06, 0xF2, 0x0F, 0x78, 0xDB, 0x08, 0x08, 0xC3}); + const auto headers = elfHeaders(); + const auto converted = Codegen::MakeAmd64OnlyConverter()->Convert(source, {headers[0]}); + require(converted.Trampolines.size() == 1 && converted.Bytes == source, "Linux fixture conversion produced unexpected results"); + auto patcher = linuxPatcher(); + const auto output = patcher.Patch(converted.Bytes, headers, {}, 0, "$ORIGIN/libs", true, false, converted.Trampolines); + require(output[0x202] == 0xE9 && output[0x207] == 0x90, "Linux site was not replaced by a jump"); + const auto target = 0x1002 + 5 + static_cast(read(output, 0x203)); + require(target % 16 == 0 && target > 0x2100, "Linux stub is misaligned or inside the original image"); + const auto bodyOffset = executableFileOffset(output, static_cast(target)); auto expectedBody = kInsertqSelfBody; write(expectedBody, 10, static_cast(0x1008 - (target + 9 + 5))); const Bytes actualBody(output.begin() + static_cast(bodyOffset), output.begin() + static_cast(bodyOffset + expectedBody.size())); @@ -713,6 +839,23 @@ void linuxPlacement() { requireFailure([&] { (void)patcher.Patch(altered, headers, {}, 0, "$ORIGIN/libs", true, false, converted.Trampolines); }, "Changed Linux site bytes were accepted"); } +void linuxRipRelativePlacement() { + Bytes text = {0x0F, 0x38, 0xCC, 0x15, 0x00, 0x00, 0x00, 0x00, 0xC3}; + write(text, 4, 0x2040 - 0x1008); + const auto source = elfFixture(text); + const auto headers = elfHeaders(); + const auto converted = Codegen::MakeAmd64OnlyConverter()->Convert(source, {headers[0]}); + require(converted.Trampolines.size() == 1 && converted.Trampolines[0].Relocations.size() == 1, "RIP-relative Linux fixture was not lowered with a relocation"); + auto patcher = linuxPatcher(); + const auto output = patcher.Patch(converted.Bytes, headers, {}, 0, "$ORIGIN/libs", true, false, converted.Trampolines); + require(output[0x200] == 0xE9, "RIP-relative Linux site was not replaced by a jump"); + const auto stub = 0x1000 + 5 + static_cast(read(output, 0x201)); + const auto& relocation = converted.Trampolines[0].Relocations[0]; + const auto bodyOffset = executableFileOffset(output, static_cast(stub)); + const auto operand = stub + static_cast(relocation.InstructionEnd) + read(output, bodyOffset + relocation.DisplacementOffset); + require(operand == 0x2040, "Relocated RIP-relative operand does not reach the original address"); +} + } #if defined(__linux__) && defined(__x86_64__) @@ -912,19 +1055,24 @@ struct StubRun { std::uint64_t Rcx; std::uint64_t Flags; std::uint64_t Xmm[16][2]; + std::uint64_t SiteAddress; }; -StubRun runStubBody(const Codegen::Amd64OnlyMatch& match, const std::uint64_t rax, const std::uint64_t (&xmmIn)[16][2]) { +StubRun runStubBody(const Codegen::Amd64OnlyMatch& match, const std::uint64_t rax, const std::uint64_t (&xmmIn)[16][2], const Bytes& data = {}) { require(match.Lowering == Codegen::Amd64OnlyLowering::Trampoline, "Stub was not produced"); auto body = match.StubBody; const auto ret = body.size(); body.push_back(0xC3); const auto displacement = static_cast(ret - (match.ReturnBranchOffset + 5)); std::memcpy(body.data() + match.ReturnBranchOffset + 1, &displacement, sizeof(displacement)); - void* code = mmap(nullptr, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + require(body.size() <= 4096 && data.size() <= 4096, "Stub or its data does not fit in a page"); + auto* code = static_cast(mmap(nullptr, 3 * 4096, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); require(code != MAP_FAILED, "cannot map executable memory for the stub"); - std::memcpy(code, body.data(), body.size()); StubRun run{}; + run.SiteAddress = reinterpret_cast(code) + 4096; + Codegen::ApplyStubRelocations(std::span(body).first(match.ReturnBranchOffset), match.Relocations, run.SiteAddress, reinterpret_cast(code), 0); + std::memcpy(code, body.data(), body.size()); + std::copy(data.begin(), data.end(), code + 2 * 4096); asm volatile( "movdqu 0x00(%[xmmIn]), %%xmm0\n\t" "movdqu 0x10(%[xmmIn]), %%xmm1\n\t" @@ -972,7 +1120,7 @@ StubRun runStubBody(const Codegen::Amd64OnlyMatch& match, const std::uint64_t ra : [flags] "=&r"(run.Flags), [raxOut] "=&r"(run.Rax), [rcxOut] "=&r"(run.Rcx) : [xmmIn] "r"(xmmIn), [xmmOut] "r"(run.Xmm), [raxIn] "r"(rax), [code] "r"(code) : "rax", "rcx", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "memory", "cc"); - munmap(code, 4096); + munmap(code, 3 * 4096); return run; } @@ -1092,6 +1240,57 @@ void sha1Execution() { } } } + +void ripRelativeExecution() { + const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(); + std::uint64_t xmmIn[16][2]; + for (unsigned reg = 0; reg < 16; ++reg) { + xmmIn[reg][0] = 0x9e3779b97f4a7c15ull * (reg + 1); + xmmIn[reg][1] = 0xc2b2ae3d27d4eb4full * (reg + 3); + } + const std::uint64_t words[2] = {0x510e527f9b05688cull, 0x1f83d9ab5be0cd19ull}; + Bytes data(0x40, 0xA5); + std::memcpy(data.data() + 0x20, words, sizeof(words)); + const std::uint64_t rax = 0x5A5A5A5A5A5A5A5Aull; + const auto unchanged = [&](const StubRun& run, const std::uint32_t changed, const char* message) { + for (unsigned reg = 0; reg < 16; ++reg) + if ((changed & (1u << reg)) == 0) + require(run.Xmm[reg][0] == xmmIn[reg][0] && run.Xmm[reg][1] == xmmIn[reg][1], message); + require(run.Rcx == 0x1122334455667788ull && (run.Flags & 0x8D5) == (0x8D7 & 0x8D5), message); + }; + for (const std::uint8_t opcode : {std::uint8_t{0xCB}, std::uint8_t{0xCC}, std::uint8_t{0xCD}}) { + Bytes site = {0x0F, 0x38, opcode, 0x15, 0x00, 0x00, 0x00, 0x00}; + write(site, 4, 4096 + 0x20 - 8); + const auto match = matcher->Match(site.data(), site.size()); + require(match && match->Relocations.size() == 1, "SHA-256 RIP-relative stub was not produced"); + const auto run = runStubBody(*match, rax, xmmIn, data); + require(std::array{run.Xmm[2][0], run.Xmm[2][1]} == sha256Reference(opcode, xmmIn[2], words, xmmIn[0]), "SHA-256 RIP-relative stub did not read the original operand"); + require(run.Rax == rax, "SHA-256 RIP-relative stub changed rax"); + unchanged(run, 1u << 2, "SHA-256 RIP-relative stub clobbered a register or RFLAGS"); + } + Bytes rounds = {0x0F, 0x3A, 0xCC, 0x15, 0x00, 0x00, 0x00, 0x00, 0x02}; + write(rounds, 4, 4096 + 0x20 - 9); + const auto roundsMatch = matcher->Match(rounds.data(), rounds.size()); + require(roundsMatch && roundsMatch->Relocations.size() == 1, "SHA1RNDS4 RIP-relative stub was not produced"); + std::uint32_t x[4]; + std::uint32_t y[4]; + std::memcpy(x, xmmIn[2], sizeof(x)); + std::memcpy(y, words, sizeof(y)); + const auto expected = sha1Reference(Codegen::DecodeSha1(rounds.data(), rounds.size()), x, y); + const auto roundsRun = runStubBody(*roundsMatch, rax, xmmIn, data); + require(std::memcmp(roundsRun.Xmm[2], expected.data(), 16) == 0, "SHA1RNDS4 RIP-relative stub did not read the original operand"); + unchanged(roundsRun, 1u << 2, "SHA1RNDS4 RIP-relative stub clobbered a register or RFLAGS"); + const Bytes extrq = {0x66, 0x0F, 0x79, 0xCA}; + Bytes followers = {0xF3, 0x0F, 0x6F, 0x1D, 0x00, 0x00, 0x00, 0x00, 0x48, 0x8D, 0x05, 0x00, 0x00, 0x00, 0x00}; + write(followers, 4, 4096 + 0x20 - 12); + write(followers, 11, 4096 + 0x30 - 19); + const auto moved = matcher->Match(extrq.data(), extrq.size(), followers); + require(moved && moved->Relocations.size() == 2, "EXTRQ with RIP-relative followers was not lowered with two relocations"); + const auto movedRun = runStubBody(*moved, rax, xmmIn, data); + require(movedRun.Xmm[3][0] == words[0] && movedRun.Xmm[3][1] == words[1], "Moved RIP-relative MOVDQU did not load from the original address"); + require(movedRun.Rax == movedRun.SiteAddress + 4096 + 0x30, "Moved RIP-relative LEA did not compute the original address"); + unchanged(movedRun, (1u << 1) | (1u << 3), "Stub with moved RIP-relative instructions clobbered a register or RFLAGS"); +} #else void reciprocalExecution() {} void registerFormExecution() {} @@ -1099,6 +1298,7 @@ void sha1Execution() {} void sha256Execution() {} void clzeroExecution() {} void immediateFormExecution() {} +void ripRelativeExecution() {} #endif void scannerZeroTail() { @@ -1113,6 +1313,7 @@ int main() { try { decoderLengths(); decoderRipRelative(); + decoderTwoByteOpcodeLengths(); sse4aOperands(); sha256Operands(); sha1Operands(); @@ -1120,12 +1321,15 @@ int main() { reciprocalOperands(); matcherSubstitutions(); goldenBodies(); + ripRelativeBodies(); + stubRelocations(); registerFormExecution(); immediateFormExecution(); sha256Execution(); sha1Execution(); clzeroExecution(); reciprocalExecution(); + ripRelativeExecution(); converterSegment(); converterRipRelativeFollower(); converterSha256(); @@ -1135,8 +1339,10 @@ int main() { converterReciprocal(); converterStrayRex(); rewriterStrayRex(); + rewriterReferenceSites(); converterFailureOffsets(); linuxPlacement(); + linuxRipRelativePlacement(); scannerZeroTail(); std::cout << "AMD64-only converter tests passed\n"; } catch (const std::exception& error) { diff --git a/core/relinker/elfpatcher/include/elfpatcher/general/ElfConstants.hpp b/core/relinker/elfpatcher/include/elfpatcher/general/ElfConstants.hpp index c67ededd1..51857f69b 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/general/ElfConstants.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/general/ElfConstants.hpp @@ -62,6 +62,7 @@ inline constexpr std::uint32_t PT_DYNAMIC = 2; inline constexpr std::uint32_t PT_INTERP = 3; inline constexpr std::uint32_t PT_NOTE = 4; inline constexpr std::uint32_t PT_PHDR = 6; +inline constexpr std::uint32_t PT_GNU_EH_FRAME = 0x6474e550; inline constexpr std::uint32_t PT_GNU_RELRO = 0x6474e552; inline constexpr std::uint32_t PF_X = 0x1; inline constexpr std::uint32_t PF_W = 0x2; diff --git a/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp b/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp index b12f1ad90..d844e8176 100644 --- a/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp +++ b/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include namespace Elfpatcher { @@ -131,10 +132,19 @@ std::uint16_t ProgramHeaderLayoutBuilder::WriteLayout( const ProgramHeaderLayoutRequest& request ) const { std::uint16_t keptCount = 0; + std::uint16_t frameCount = 0; for (const auto& ph : request.OriginalHeaders) { if (_segmentFilter->ShouldSkip(ph)) continue; keptCount++; + if (ph.Type == PT_GNU_EH_FRAME) + frameCount++; + } + + const bool keepFrames = keptCount + kSyntheticProgramHeaderCount < request.PhNum; + if (!keepFrames && frameCount > 0) { + keptCount -= frameCount; + std::cerr << "WARNING: No free program header slot for PT_GNU_EH_FRAME; C++ exceptions thrown in the executable cannot be caught.\n"; } const std::uint16_t neededPh = keptCount + kSyntheticProgramHeaderCount; @@ -175,7 +185,7 @@ std::uint16_t ProgramHeaderLayoutBuilder::WriteLayout( writtenPh++; for (const auto& ph : request.OriginalHeaders) { - if (_segmentFilter->ShouldSkip(ph)) + if (_segmentFilter->ShouldSkip(ph) || (!keepFrames && ph.Type == PT_GNU_EH_FRAME)) continue; const std::size_t phEntOff = static_cast(request.PhOff) + writtenPh * request.PhEntSize; if (ph.Type == PT_LOAD) { diff --git a/core/relinker/elfpatcher/src/general/SegmentFilter.cpp b/core/relinker/elfpatcher/src/general/SegmentFilter.cpp index 07425c37e..ff8f24e7f 100644 --- a/core/relinker/elfpatcher/src/general/SegmentFilter.cpp +++ b/core/relinker/elfpatcher/src/general/SegmentFilter.cpp @@ -12,6 +12,7 @@ bool SegmentFilter::_isSceSpecificSegment(const std::uint32_t type) const { bool SegmentFilter::ShouldSkip(const Domain::ProgramHeader& ph) const { if (ph.Type == PT_OS_PROCPARAM) return false; + if (ph.Type == PT_GNU_EH_FRAME) return false; if (_isSceSpecificSegment(ph.Type)) return true; if (ph.Type == PT_DYNAMIC) return true; if (ph.Type == PT_NOTE && ph.MappedAddress == 0 && ph.FileSize > 0) return true; diff --git a/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp b/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp index 90ce2bc28..24746a4a1 100644 --- a/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp +++ b/core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp @@ -3,8 +3,10 @@ #include #include #include +#include #include #include +#include #include namespace Elfpatcher::Linux { @@ -47,6 +49,7 @@ void LinuxElfPatcher::_appendTrampoline( const auto bodyVaddr = vaddrOfExtraBlockOffset(bodyOff); for (const std::uint8_t b : site.Body) buf.push_back(b); + Codegen::ApplyStubRelocations(std::span(buf.data() + bodyOff, site.ReturnBranchOffset), site.Relocations, site.Address, bodyVaddr, site.Offset); const auto inRange = [](const std::int64_t displacement) { return displacement >= std::numeric_limits::min() && displacement <= std::numeric_limits::max(); @@ -101,6 +104,7 @@ std::vector LinuxElfPatcher::Patch( for (char c : runPath) buf.push_back(static_cast(c)); buf.push_back(0); + const std::uint64_t dynStrSize = static_cast(buf.size()) - dynStrOff; alignBuf(buf, kDynStrAlignment); const auto dynSymOff = static_cast(buf.size()); @@ -131,7 +135,7 @@ std::vector LinuxElfPatcher::Patch( for (std::uint8_t b : dynSection.DynamicSegmentData) dynSegBuf.push_back(b); _appendDynEntry(dynSegBuf, DT_STRTAB, vaddrOfExtraBlockOffset(dynStrOff)); - _appendDynEntry(dynSegBuf, DT_STRSZ, dynSection.DynStrData.size()); + _appendDynEntry(dynSegBuf, DT_STRSZ, dynStrSize); _appendDynEntry(dynSegBuf, DT_SYMTAB, vaddrOfExtraBlockOffset(dynSymOff)); _appendDynEntry(dynSegBuf, DT_SYMENT, kSymEntrySize); if (!dynSection.RelaData.empty()) { @@ -191,7 +195,7 @@ std::vector LinuxElfPatcher::Patch( SectionHeaderTableRequest sectionRequest{}; sectionRequest.DynStrOffset = dynStrOff; - sectionRequest.DynStrSize = dynSection.DynStrData.size(); + sectionRequest.DynStrSize = dynStrSize; sectionRequest.DynSymOffset = dynSymOff; sectionRequest.DynSymSize = dynSection.DynSymData.size(); sectionRequest.DynamicSegmentOffset = dynSegOff; diff --git a/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp index 46fc14810..f7744ddfa 100644 --- a/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp +++ b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp @@ -1,8 +1,10 @@ #include #include +#include #include #include #include +#include namespace Elfpatcher { @@ -153,6 +155,7 @@ std::vector GuestModuleWriter::WriteLinux(const Relinker::GuestIma const auto returnDisplacement = displacement(returnAddress, stubReturn, site.Offset); const auto siteDisplacement = displacement(stubAddress, siteNext, site.Offset); bytes.insert(bytes.end(), site.Body.begin(), site.Body.end()); + Codegen::ApplyStubRelocations(std::span(bytes.data() + stubOffset, site.ReturnBranchOffset), site.Relocations, site.Address, stubAddress, site.Offset); Io::WriteU32(bytes, stubOffset + site.ReturnBranchOffset + 1, static_cast(returnDisplacement)); std::fill_n(bytes.begin() + static_cast(site.Offset), site.Length, kNop1.Bytes[0]); diff --git a/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp index 230d805a8..068954f28 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp @@ -105,6 +105,13 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con for (std::size_t index = guestModules.size(); index < libraries.size(); ++index) { if (libraries[index] == "libc.prx" && import.Library != "libc.prx") order.push_back(CheckedRva(index)); } + if (import.Library == "libSceLibcInternal.prx") { + for (std::size_t index = 0; index < guestModules.size(); ++index) { + const auto& names = guestModules[index].Names; + if (std::find(names.begin(), names.end(), "libc.prx") != names.end()) + order.push_back(CheckedRva(index)); + } + } } Io::AlignBuffer(data, 4); platformTlsResolvers.push_back(std::none_of(order.begin(), order.end(), [&](auto index) { @@ -162,33 +169,32 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con code.Emit({0x0f, 0x0b}); }; - const auto requireDiagnosticSuccess = [&] { - code.Emit({0x48, 0x85, 0xc0}); - const auto success = code.Branch({0x0f, 0x85}); - raise(0xc0000001u); - code.PatchBranch(success, code.GetRva()); - }; - const auto writeString = [&](const std::uint32_t stringRva, const bool isError = false) { code.Rip({0x48, 0x8d, 0x0d}, stringRva); call("lstrlenA"); + code.Emit({0x48, 0x85, 0xc0}); + const auto empty = code.Branch({0x0f, 0x84}); code.Emit({0x89, 0x44, 0x24, 0x3c, 0xb9}); code.U32(isError ? 0xfffffff4u : 0xfffffff5u); call("GetStdHandle"); - requireDiagnosticSuccess(); + code.Emit({0x48, 0x85, 0xc0}); + const auto noHandle = code.Branch({0x0f, 0x84}); code.Emit({0x48, 0x83, 0xf8, 0xff}); - const auto validHandle = code.Branch({0x0f, 0x85}); - raise(0xc0000001u); - code.PatchBranch(validHandle, code.GetRva()); + const auto invalidHandle = code.Branch({0x0f, 0x84}); code.Emit({0x48, 0x89, 0xc1}); code.Rip({0x48, 0x8d, 0x15}, stringRva); code.Emit({0x44, 0x8b, 0x44, 0x24, 0x3c, 0x4c, 0x8d, 0x4c, 0x24, 0x38, 0x48, 0xc7, 0x44, 0x24, 0x20, 0, 0, 0, 0}); call("WriteFile"); - requireDiagnosticSuccess(); + code.Emit({0x48, 0x85, 0xc0}); + const auto failedWrite = code.Branch({0x0f, 0x84}); code.Emit({0x8b, 0x44, 0x24, 0x38, 0x3b, 0x44, 0x24, 0x3c}); - const auto complete = code.Branch({0x0f, 0x84}); - raise(0xc0000001u); - code.PatchBranch(complete, code.GetRva()); + const auto incompleteWrite = code.Branch({0x0f, 0x85}); + const auto done = code.GetRva(); + code.PatchBranch(empty, done); + code.PatchBranch(noHandle, done); + code.PatchBranch(invalidHandle, done); + code.PatchBranch(failedWrite, done); + code.PatchBranch(incompleteWrite, done); }; const auto writeLastError = [&] { @@ -214,8 +220,10 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con code.U32(ErrorMessageCapacity); code.Emit({0x48, 0xc7, 0x44, 0x24, 0x30, 0, 0, 0, 0}); call("FormatMessageA"); - requireDiagnosticSuccess(); + code.Emit({0x48, 0x85, 0xc0}); + const auto noMessage = code.Branch({0x0f, 0x84}); writeString(errorMessage, true); + code.PatchBranch(noMessage, code.GetRva()); }; const auto captureLastError = [&] { diff --git a/core/relinker/elfpatcher/src/windows/WindowsLoadImage.cpp b/core/relinker/elfpatcher/src/windows/WindowsLoadImage.cpp index ea4158313..f82ee3d93 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsLoadImage.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsLoadImage.cpp @@ -28,15 +28,16 @@ WindowsLoadImage::WindowsLoadImage(const std::vector& source, cons throw Domain::RelinkerException("No PT_LOAD segments found"); std::sort(segments.begin(), segments.end(), [](const auto& left, const auto& right) { return left.MappedAddress < right.MappedAddress; }); firstAddress = segments.front().MappedAddress & ~static_cast(SectionAlignment - 1); - const auto end = segments.back().MappedAddress + segments.back().MemorySize; + std::uint64_t end = firstAddress; + for (const auto& segment : segments) { + if (segment.MappedAddress < end) + throw Domain::RelinkerException("Overlapping PT_LOAD memory ranges", segment.MappedAddress); + end = segment.MappedAddress + segment.MemorySize; + } data.resize(AlignRva(end - firstAddress)); CheckedRva(LoadRva + static_cast(data.size())); pageFlags.resize(data.size() / SectionAlignment); - std::uint64_t previousEnd = firstAddress; for (const auto& segment : segments) { - if (segment.MappedAddress < previousEnd) - throw Domain::RelinkerException("Overlapping PT_LOAD memory ranges", segment.MappedAddress); - previousEnd = segment.MappedAddress + segment.MemorySize; const auto offset = static_cast(segment.MappedAddress - firstAddress); std::copy_n(source.begin() + static_cast(segment.Offset), static_cast(segment.FileSize), data.begin() + static_cast(offset)); std::uint32_t flags = 0; diff --git a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp index cec6aec7e..3fb5379dc 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,14 @@ namespace Elfpatcher::Windows { namespace { +constexpr std::size_t kJumpSize = 5; + +struct MovedInstruction { + std::uint32_t Rva; + std::vector Bytes; + std::optional RipDisplacement; +}; + struct TlsAccess { std::uint32_t Rva; Domain::FileByteOffset FileOffset; @@ -25,17 +34,48 @@ struct TlsAccess { std::uint8_t Register; std::uint32_t Displacement; std::uint8_t AluOpcode; + std::vector Address = {}; + bool Wide = true; + std::vector Moved = {}; }; bool isAluReadOpcode(std::uint8_t opcode) { return opcode == 0x03 || opcode == 0x0b || opcode == 0x13 || opcode == 0x1b || opcode == 0x23 || opcode == 0x2b || opcode == 0x33 || opcode == 0x3b; } +std::optional> registerAddress(const std::uint8_t* bytes, const std::size_t position, const std::size_t length, const std::uint8_t rex) { + if (length - position < 2) + return std::nullopt; + const auto modrm = bytes[position + 1]; + const auto mod = modrm >> 6; + if (mod == 3 || (mod == 0 && (modrm & 7) == 5)) + return std::nullopt; + std::size_t size = 2; + if ((modrm & 7) == 4) { + if (length - position < 3) + return std::nullopt; + const auto sib = bytes[position + 2]; + const bool noBase = mod == 0 && (sib & 7) == 5; + const bool noIndex = ((sib >> 3) & 7) == 4 && (rex & 2) == 0; + if ((noBase && noIndex) || ((sib & 7) == 4 && (rex & 1) == 0)) + return std::nullopt; + size += noBase ? 5 : 1; + } + size += mod == 1 ? 1 : mod == 2 ? 4 : 0; + if (length - position != size) + return std::nullopt; + std::vector address{static_cast(0x48 | (rex & 3)), 0x8d, static_cast(modrm & 0xc7)}; + address.insert(address.end(), bytes + position + 2, bytes + length); + return address; +} + void patchAccess(std::vector& sections, const TlsAccess& access, const std::uint32_t target) { for (auto& section : sections) { if (access.Rva < section.Rva || access.Rva - section.Rva >= section.Data.size()) continue; const auto offset = access.Rva - section.Rva; + if (access.Length < kJumpSize) + throw Domain::RelinkerException("TLS instruction is shorter than a jump", access.Rva); if (access.Length > section.Data.size() - offset) throw Domain::RelinkerException("TLS instruction crosses a PE section boundary", access.Rva); WindowsStubEmitter jump(access.Rva); @@ -66,11 +106,14 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co } if (header.Type != 1 || (header.Flags & 1) == 0) continue; + std::uint64_t decodedEnd = 0; for (auto instruction = instructions.lower_bound(header.MappedAddress); instruction != instructions.end() && *instruction - header.MappedAddress < header.FileSize; ++instruction) { const auto offset = *instruction - header.MappedAddress; const auto* bytes = source.data() + header.Offset + offset; const auto info = decoder.DecodeInstruction(bytes, header.FileSize - offset); const auto rva = image.GetRva(header.MappedAddress + offset, info.Length); + const bool insidePrevious = *instruction < decodedEnd; + decodedEnd = std::max(decodedEnd, *instruction + info.Length); if (info.HasBranchTarget && !info.HasRipRelativeDisp) { const auto target = static_cast(rva) + static_cast(info.Length) + info.BranchDisp; @@ -78,7 +121,7 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co branchTargets.insert(static_cast(target)); } - if (info.SegmentPrefix != 0) { + if (info.SegmentPrefix != 0 && !(insidePrevious && info.SegmentPrefix != 0x64)) { const auto position = info.OpcodeOffset; bool hasOperandSizePrefix = false; bool supportedPrefixes = info.SegmentPrefix == 0x64; @@ -91,6 +134,24 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co const bool loadValue = supportedPrefixes && (info.RexPrefix == 0x48 || info.RexPrefix == 0x4c) && loadRegister != 4 && bytes[position] == 0x8b && (bytes[position + 1] & 0xc7) == 0x04 && bytes[position + 2] == 0x25; const bool storeImmediate = supportedPrefixes && !hasOperandSizePrefix && (info.RexPrefix == 0 || info.RexPrefix == 0x40) && info.Length - position == 11 && bytes[position] == 0xc7 && bytes[position + 1] == 0x04 && bytes[position + 2] == 0x25 && Io::ReadU32(source, header.Offset + offset + position + 3) == 0x28; const bool aluRead = supportedPrefixes && (info.RexPrefix == 0x48 || info.RexPrefix == 0x4c) && loadRegister != 4 && info.Length - position == 7 && isAluReadOpcode(bytes[position]) && (bytes[position + 1] & 0xc7) == 0x04 && bytes[position + 2] == 0x25; + const bool wide = (info.RexPrefix & 8) != 0; + const auto address = supportedPrefixes && !loadValue && !aluRead && bytes[position] == 0x8b && (wide || !hasOperandSizePrefix) ? registerAddress(bytes, position, info.Length, info.RexPrefix) : std::nullopt; + const auto addressRegister = static_cast(((bytes[position + 1] >> 3) & 7) | ((info.RexPrefix & 4) << 1)); + if (address && addressRegister != 4) { + std::vector moved; + auto length = info.Length; + while (length < kJumpSize) { + if (length >= header.FileSize - offset) + throw Domain::RelinkerException("Short guest TLS instruction ends its segment", header.Offset + offset); + const auto next = decoder.DecodeInstruction(bytes + length, header.FileSize - offset - length); + if (next.FlowKind != Codegen::ControlFlowKind::Sequential || next.HasBranchTarget || next.SegmentPrefix != 0) + throw Domain::RelinkerException("Short guest TLS instruction is followed by an instruction that cannot move", header.Offset + offset + length); + moved.push_back({CheckedRva(rva + length), std::vector(bytes + length, bytes + length + next.Length), next.HasRipRelativeDisp ? std::optional(next.RipRelativeDispOffset) : std::nullopt}); + length += next.Length; + } + accesses.push_back({rva, header.Offset + offset, length, false, 0, addressRegister, 0, 0, *address, wide, std::move(moved)}); + continue; + } if (!loadValue && !storeImmediate && !aluRead) { std::ostringstream message; message << "Unsupported Windows guest TLS instruction (bytes:" << std::hex << std::setfill('0'); @@ -162,7 +223,29 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co throw Domain::RelinkerException("Branch enters a guest TLS instruction", *target); patchAccess(sections, access, code.GetRva()); code.Emit({0x48, 0x8d, 0x64, 0x24, 0x80}); - if (access.AluOpcode != 0 && access.Register == 0) { + if (!access.Address.empty()) { + code.Emit({0x51}); + code.Emit({0x50}); + for (const auto byte : access.Address) + code.Emit({byte}); + code.Emit({0x50}); + loadPointer(); + code.Emit({0x59}); + const auto rex = static_cast((access.Wide ? 0x48 : 0x40) | ((access.Register >> 3) << 2)); + if (rex != 0x40) + code.Emit({rex}); + code.Emit({0x8b, static_cast(0x04 | ((access.Register & 7) << 3)), 0x08}); + if (access.Register == 0) { + code.Emit({0x48, 0x8d, 0x64, 0x24, 0x08}); + code.Emit({0x59}); + } else if (access.Register == 1) { + code.Emit({0x58}); + code.Emit({0x48, 0x8d, 0x64, 0x24, 0x08}); + } else { + code.Emit({0x58}); + code.Emit({0x59}); + } + } else if (access.AluOpcode != 0 && access.Register == 0) { code.Emit({0x51}); code.Emit({0x52}); code.Emit({0x50}); @@ -207,6 +290,18 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co if (preserveCounter) code.Emit({0x59}); } code.Emit({0x48, 0x8d, 0xa4, 0x24, 0x80, 0, 0, 0}); + for (const auto& instruction : access.Moved) { + auto bytes = instruction.Bytes; + if (instruction.RipDisplacement) { + const auto target = static_cast(instruction.Rva) + static_cast(bytes.size()) + std::bit_cast(Io::ReadU32(bytes, *instruction.RipDisplacement)); + const auto displacement = target - (static_cast(code.GetRva()) + static_cast(bytes.size())); + if (displacement < std::numeric_limits::min() || displacement > std::numeric_limits::max()) + throw Domain::RelinkerException("Moved RIP-relative instruction exceeds rel32 range from its TLS stub", access.FileOffset); + Io::WriteU32(bytes, *instruction.RipDisplacement, static_cast(static_cast(displacement))); + } + for (const auto byte : bytes) + code.Emit({byte}); + } code.Rip({0xe9}, CheckedRva(access.Rva + access.Length)); } diff --git a/core/relinker/elfpatcher/src/windows/WindowsTrampolineBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsTrampolineBuilder.cpp index 0dc522242..f8d06aa53 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsTrampolineBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsTrampolineBuilder.cpp @@ -1,9 +1,11 @@ #include #include #include +#include #include #include #include +#include namespace Elfpatcher::Windows { @@ -40,6 +42,7 @@ void WindowsTrampolineBuilder::Build(const std::vector& const auto stubRva = CheckedRva(sectionRva + bytes.size()); const auto bodyOffset = bytes.size(); bytes.insert(bytes.end(), site.Body.begin(), site.Body.end()); + Codegen::ApplyStubRelocations(std::span(bytes.data() + bodyOffset, site.ReturnBranchOffset), site.Relocations, siteRva, stubRva, site.Offset); const auto returnRva = CheckedRva(siteRva + site.Length); const auto displacement = static_cast(returnRva) - (static_cast(stubRva) + static_cast(site.ReturnBranchOffset + kJmpRel32.Size)); if (displacement < std::numeric_limits::min() || displacement > std::numeric_limits::max()) diff --git a/core/relinker/elfpatcher/tests/WindowsIconResourceTests.cpp b/core/relinker/elfpatcher/tests/WindowsIconResourceTests.cpp index 52af30318..f91bf45f6 100644 --- a/core/relinker/elfpatcher/tests/WindowsIconResourceTests.cpp +++ b/core/relinker/elfpatcher/tests/WindowsIconResourceTests.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -85,7 +86,7 @@ void run(const std::filesystem::path& root) { } int main() { - const auto root = std::filesystem::temp_directory_path() / "anyps5-icon-resource-tests"; + const auto root = std::filesystem::temp_directory_path() / ("anyps5-icon-resource-tests-" + std::to_string(std::random_device{}())); std::filesystem::remove_all(root); try { run(root); diff --git a/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp b/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp index e065dc76e..b346b598c 100644 --- a/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp +++ b/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp @@ -209,9 +209,59 @@ std::set CodeInstructionCollector::Collect(const std::ve address += info.Length; } } + for (const auto& header : headers) { + if (header.Type != 1 || (header.Flags & 1) == 0 || header.FileSize < 16) continue; + range(header.Offset, header.FileSize); + const auto* text = bytes.data() + header.Offset; + const auto size = static_cast(header.FileSize); + const auto paddedStart = [&](std::size_t address) { return address % 16 == 0 && address >= 2 && text[address - 1] == 0xcc && text[address - 2] == 0xcc; }; + const auto nextPaddedStart = [&](std::size_t address) { + address = (address + 16) & ~std::size_t{15}; + while (address < size && !paddedStart(address)) address += 16; + return address; + }; + for (std::size_t position = nextPaddedStart(0); position < size;) { + if (position + 1 < size && text[position] == 0 && text[position + 1] == 0) { + position = nextPaddedStart(position); + continue; + } + Codegen::DecodedInstructionInfo info; + try { + info = decoder.DecodeInstruction(text + position, size - position); + } catch (const Codegen::CodegenException&) { + position = nextPaddedStart(position); + continue; + } + if (info.Length == 0 || info.Length > size - position) { + position = nextPaddedStart(position); + continue; + } + const auto next = position + info.Length; + const auto crossed = (position & ~std::size_t{15}) + 16; + if (crossed < next && paddedStart(crossed)) { + position = crossed; + continue; + } + const auto opcode = position + info.OpcodeOffset; + if (info.SegmentPrefix == 0x64 && opcode + 2 < next && (text[opcode + 1] & 0xc7) == 0x04 && text[opcode + 2] == 0x25) addRoot(header.MappedAddress + position); + position = next; + } + } + const auto decodesInSegment = [&](std::uint64_t address) { + for (const auto& header : headers) { + if (header.Type != 1 || (header.Flags & 1) == 0 || address < header.MappedAddress || address - header.MappedAddress >= header.FileSize) continue; + const auto offset = address - header.MappedAddress; + try { + return decoder.DecodeInstruction(bytes.data() + header.Offset + offset, header.FileSize - offset).Length != 0; + } catch (const Codegen::CodegenException&) { + return false; + } + } + return false; + }; for (const auto target : staticTargets) { for (const auto& [skipBegin, skipEnd] : skipped) { - if (skipBegin <= target && target < skipEnd) + if (skipBegin <= target && target < skipEnd && !decodesInSegment(target)) throw Domain::RelinkerException("Code analysis: branch into skipped range tail", target); } } diff --git a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp index 99368fb49..b2cd06d9a 100644 --- a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp +++ b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp @@ -12,6 +12,25 @@ namespace Relinker { +namespace { + +std::vector ReadNeededNames(const Domain::SysVDynamicSection& dynamic) { + if (dynamic.DynamicSegmentData.size() % 16 != 0) throw Domain::RelinkerException("Invalid executable dependency table"); + std::vector names; + for (std::size_t offset = 0; offset < dynamic.DynamicSegmentData.size(); offset += 16) { + if (Io::ReadU64(dynamic.DynamicSegmentData, offset) != 1) throw Domain::RelinkerException("Unexpected executable dependency tag"); + const auto nameOffset = Io::ReadU64(dynamic.DynamicSegmentData, offset + 8); + if (nameOffset >= dynamic.DynStrData.size()) throw Domain::RelinkerException("Invalid dependency string offset"); + const auto start = dynamic.DynStrData.begin() + static_cast(nameOffset); + const auto end = std::find(start, dynamic.DynStrData.end(), 0); + if (end == dynamic.DynStrData.end()) throw Domain::RelinkerException("Unterminated dependency string"); + names.emplace_back(start, end); + } + return names; +} + +} + std::vector GuestModuleBuilder::Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, const bool windows, const bool toIntel, ISyscallScanner& syscallScanner, const bool lazyBinding, const std::string& runPath, const std::set& excludedModules) const { const auto root = std::filesystem::absolute(inputPath).parent_path(); const auto singular = root / "sce_module"; @@ -47,16 +66,9 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path if (isElf(entry.path())) paths.push_back(entry.path()); } } - if (dynamic.DynamicSegmentData.size() % 16 != 0) throw Domain::RelinkerException("Invalid executable dependency table"); + const auto neededNames = ReadNeededNames(dynamic); std::set missingNeeded; - for (std::size_t offset = 0; offset < dynamic.DynamicSegmentData.size(); offset += 16) { - if (Io::ReadU64(dynamic.DynamicSegmentData, offset) != 1) continue; - const auto nameOffset = Io::ReadU64(dynamic.DynamicSegmentData, offset + 8); - if (nameOffset >= dynamic.DynStrData.size()) throw Domain::RelinkerException("Invalid dependency string offset"); - const auto start = dynamic.DynStrData.begin() + static_cast(nameOffset); - const auto end = std::find(start, dynamic.DynStrData.end(), 0); - if (end == dynamic.DynStrData.end()) throw Domain::RelinkerException("Unterminated dependency string"); - const std::string name(start, end); + for (const auto& name : neededNames) { if (excludedModules.contains(name)) continue; if (std::none_of(paths.begin(), paths.end(), [&](const auto& path) { return path.filename().string() == name; })) missingNeeded.insert(name); } @@ -115,13 +127,25 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path images.push_back(std::move(image)); } std::map guestNames; + std::map windowsGuestFiles; + const auto foldFilename = [](std::string name) { + for (auto& character : name) if (character >= 'A' && character <= 'Z') character = static_cast(character + ('a' - 'A')); + return name; + }; for (std::size_t index = 0; index < images.size(); ++index) { for (const auto& name : {images[index].SourcePath.filename().string(), images[index].Soname}) { if (name.empty()) continue; const auto [found, inserted] = guestNames.emplace(name, index); if (!inserted && found->second != index) throw Domain::RelinkerException("Ambiguous guest dependency name: " + name); } + if (windows) windowsGuestFiles.emplace(foldFilename(images[index].SourcePath.filename().string()), index); } + const auto findGuest = [&](const std::string& name) { + const auto exact = guestNames.find(name); + if (exact != guestNames.end() || !windows) return exact; + const auto file = windowsGuestFiles.find(foldFilename(name)); + return file == windowsGuestFiles.end() ? guestNames.end() : guestNames.emplace(name, file->second).first; + }; const auto rejectSharedImport = [&](const std::string& name, const std::string& importer) { const auto shared = sharedExports.find(name); if (shared == sharedExports.end()) return; @@ -154,7 +178,7 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path } for (std::size_t index = 0; index < images.size(); ++index) { for (const auto& name : images[index].Dependencies) { - const auto found = guestNames.find(name); + const auto found = findGuest(name); if (found != guestNames.end() && found->second != index) dependencies[index].insert(found->second); } for (const auto& symbol : images[index].Symbols) { @@ -164,8 +188,8 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path std::vector providers; if (found != exports.end()) { for (const auto provider : found->second) { - const auto& candidate = images[provider]; - if (!windows || symbol.Library.empty() || symbol.Library == candidate.SourcePath.filename().string() || symbol.Library == candidate.Soname) providers.push_back(provider); + const auto declared = findGuest(symbol.Library); + if (!windows || symbol.Library.empty() || (declared != guestNames.end() && declared->second == provider)) providers.push_back(provider); } } if (providers.size() > 1) throw Domain::RelinkerException("Ambiguous guest import after stripping #: " + symbol.Name); @@ -200,20 +224,11 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path std::vector hostLibraries; std::set uniqueHosts; const auto addHost = [&](const std::string& name) { - if (guestNames.contains(name)) return; + if (findGuest(name) != guestNames.end()) return; if (name.empty() || name.find_first_of("/\\:$") != std::string::npos) throw Domain::RelinkerException("Invalid host dependency: " + name); if (uniqueHosts.insert(name).second) hostLibraries.push_back(name); }; - if (dynamic.DynamicSegmentData.size() % 16 != 0) throw Domain::RelinkerException("Invalid executable dependency table"); - for (std::size_t offset = 0; offset < dynamic.DynamicSegmentData.size(); offset += 16) { - if (Io::ReadU64(dynamic.DynamicSegmentData, offset) != 1) throw Domain::RelinkerException("Unexpected executable dependency tag"); - const auto nameOffset = Io::ReadU64(dynamic.DynamicSegmentData, offset + 8); - if (nameOffset >= dynamic.DynStrData.size()) throw Domain::RelinkerException("Invalid dependency string offset"); - const auto start = dynamic.DynStrData.begin() + nameOffset; - const auto end = std::find(start, dynamic.DynStrData.end(), 0); - if (end == dynamic.DynStrData.end()) throw Domain::RelinkerException("Unterminated dependency string"); - addHost(std::string(start, end)); - } + for (const auto& name : neededNames) addHost(name); for (const auto& image : images) for (const auto& dependency : image.Dependencies) addHost(dependency); if (uniqueHosts.contains("libSceLibcInternal.prx") && uniqueHosts.insert("libc.prx").second) hostLibraries.push_back("libc.prx"); dynamic.DynamicSegmentData.clear(); @@ -254,6 +269,11 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path needed.insert(needed.end(), hostLibraries.begin(), hostLibraries.end()); output = Elfpatcher::GuestModuleWriter().WriteLinux(image, needed, guestRunPath); } + if (windows) { + for (const auto& [name, provider] : guestNames) { + if (provider == index && std::find(runtime.Names.begin(), runtime.Names.end(), name) == runtime.Names.end()) runtime.Names.push_back(name); + } + } dynamic.GuestModules.push_back(std::move(runtime)); artifacts.push_back({target, std::move(output)}); } diff --git a/core/relinker/relinker/src/output/CallRegistryWriter.cpp b/core/relinker/relinker/src/output/CallRegistryWriter.cpp index 086a844f9..39ca268aa 100644 --- a/core/relinker/relinker/src/output/CallRegistryWriter.cpp +++ b/core/relinker/relinker/src/output/CallRegistryWriter.cpp @@ -13,7 +13,13 @@ static std::string _jsonString(const std::string& s) { else if (c == '\n') out += "\\n"; else if (c == '\r') out += "\\r"; else if (c == '\t') out += "\\t"; - else out.push_back(c); + else if (static_cast(c) < 0x20) { + constexpr char digits[] = "0123456789abcdef"; + const auto value = static_cast(c); + out += "\\u00"; + out.push_back(digits[value >> 4]); + out.push_back(digits[value & 0x0f]); + } else out.push_back(c); } out.push_back('"'); return out; diff --git a/core/relinker/relinker/src/parsing/ElfReader.cpp b/core/relinker/relinker/src/parsing/ElfReader.cpp index 8401513b1..84b845c19 100644 --- a/core/relinker/relinker/src/parsing/ElfReader.cpp +++ b/core/relinker/relinker/src/parsing/ElfReader.cpp @@ -74,7 +74,8 @@ ElfHeader ElfReader::ReadHeader() const { throw RelinkerException("File too small for ELF header"); } - if (_fileBuffer[0] == 0x4f && _fileBuffer[1] == 0x15 && _fileBuffer[2] == 0x3d && _fileBuffer[3] == 0x1d) { + const std::uint32_t magic = _readU32At(0); + if (magic == 0x1d3d154f || magic == 0xeef51454) { throw RelinkerException("The input is a SELF container, not an ELF"); } @@ -102,6 +103,9 @@ ElfHeader ElfReader::ReadHeader() const { std::vector ElfReader::ReadProgramHeaders() const { const ElfHeader header = ReadHeader(); + if (header.ProgramHeaderCount != 0 && header.ProgramHeaderEntrySize != 56) { + throw RelinkerException("Invalid ELF program header entry size: expected 56 bytes", 0x36); + } std::vector headers; FileByteOffset offset = header.ProgramHeaderOffset; diff --git a/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp b/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp index 1d1407d05..63d311a13 100644 --- a/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp +++ b/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp @@ -145,6 +145,8 @@ RelinkResult RelinkerPipeline::Relink(const std::vector& sourceElf constexpr std::size_t relaEntSize = 24; if (readAsSize(DT_OS_RELAENT, DT_RELAENT, "DT_RELAENT") != relaEntSize) throw RelinkerException("Unsupported DT_RELAENT value"); + if (dynRelaSize % relaEntSize != 0) + throw RelinkerException("Invalid DT_RELASZ value"); std::vector> neededLibraryNamesByStrOffset; for (const auto& tag : dynTags) @@ -160,6 +162,9 @@ RelinkResult RelinkerPipeline::Relink(const std::vector& sourceElf if (dynStrTabOffset > raw.size() || dynStrTabSize > raw.size() - dynStrTabOffset) throw RelinkerException("Dynamic string table is out of bounds", dynStrTabOffset); + if (hasPltRelocations && (dynJmpRelOffset > raw.size() || dynJmpRelSize > raw.size() - dynJmpRelOffset)) + throw RelinkerException("Jump relocation table is out of bounds", dynJmpRelOffset); + auto readCStr = [&](FileByteOffset strOff) -> std::string { if (strOff >= dynStrTabSize) throw RelinkerException("Dynamic string offset is outside DT_STRSZ", strOff); @@ -185,11 +190,15 @@ RelinkResult RelinkerPipeline::Relink(const std::vector& sourceElf throw RelinkerException("Duplicate import module ID"); } + auto relaEntryPos = [&](const FileByteOffset relaOff, const ByteCount off) -> FileByteOffset { + if (relaOff > raw.size() || off > raw.size() - relaOff || relaEntSize > raw.size() - relaOff - off) + throw RelinkerException("Relocation entry out of bounds", relaOff); + return relaOff + off; + }; + auto extractRela = [&](const FileByteOffset relaOff, const ByteCount relaSize) { - for (ByteCount off = 0; off + relaEntSize <= relaSize; off += relaEntSize) { - const FileByteOffset pos = relaOff + off; - if (pos + relaEntSize > raw.size()) - throw RelinkerException("Relocation entry out of bounds", pos); + for (ByteCount off = 0; relaSize >= relaEntSize && off <= relaSize - relaEntSize; off += relaEntSize) { + const FileByteOffset pos = relaEntryPos(relaOff, off); std::uint64_t rOffset = 0, rInfo = 0; std::int64_t rAddend = 0; @@ -291,8 +300,8 @@ RelinkResult RelinkerPipeline::Relink(const std::vector& sourceElf }; auto extractRelative = [&](const FileByteOffset relaOff, const ByteCount relaSize) { - for (ByteCount off = 0; off + relaEntSize <= relaSize; off += relaEntSize) { - const FileByteOffset pos = relaOff + off; + for (ByteCount off = 0; relaSize >= relaEntSize && off <= relaSize - relaEntSize; off += relaEntSize) { + const FileByteOffset pos = relaEntryPos(relaOff, off); std::uint64_t rOffset = 0, rInfo = 0; std::int64_t rAddend = 0; std::memcpy(&rOffset, raw.data() + pos, 8); diff --git a/core/relinker/relinker/tests/ElfFixture.hpp b/core/relinker/relinker/tests/ElfFixture.hpp index 9962b9d85..8c44f1179 100644 --- a/core/relinker/relinker/tests/ElfFixture.hpp +++ b/core/relinker/relinker/tests/ElfFixture.hpp @@ -18,6 +18,9 @@ inline constexpr std::uint64_t kCodeOffset = 0x4000; inline constexpr std::uint64_t kCodeSize = 0x1000; inline constexpr std::uint64_t kDynamicVaddr = 0x600; inline constexpr std::uint16_t kSpareProgramHeaders = 3; +inline constexpr std::uint64_t kModuleDataOffset = 0x5000; +inline constexpr std::uint64_t kModuleDataVaddr = 0x2000; +inline constexpr std::uint64_t kModuleDataSize = 0x400; template void Write(Bytes& bytes, const std::size_t offset, const TValue value) { @@ -72,6 +75,42 @@ inline Bytes MakeExecutable(const std::span code) { return image; } +inline Bytes MakeModule(const std::span code) { + if (code.size() > kCodeSize) throw std::runtime_error("ELF fixture code exceeds the code segment"); + Bytes image(0x8000); + const std::uint8_t ident[16] = {0x7F, 'E', 'L', 'F', 2, 1, 1}; + std::memcpy(image.data(), ident, sizeof(ident)); + Write(image, 16, 3); + Write(image, 18, 62); + Write(image, 20, 1); + Write(image, 24, 0); + Write(image, 32, 64); + Write(image, 52, 64); + Write(image, 54, 56); + Write(image, 56, 3 + kSpareProgramHeaders); + Write(image, 58, 64); + const std::pair tags[] = { + {5, kModuleDataVaddr + 0x200}, {10, 1}, + {6, kModuleDataVaddr + 0x220}, {11, 24}, + {4, kModuleDataVaddr + 0x240}, + {7, kModuleDataVaddr + 0x300}, {8, 0}, {9, 24}, + {0, 0}, + }; + WriteProgramHeader(image, 0, 1, 5, kCodeOffset, 0, kCodeSize, 0x4000); + WriteProgramHeader(image, 1, 1, 6, kModuleDataOffset, kModuleDataVaddr, kModuleDataSize, 8); + WriteProgramHeader(image, 2, 2, 6, kModuleDataOffset, kModuleDataVaddr, sizeof(tags) / sizeof(tags[0]) * 16, 8); + for (std::uint16_t index = 3; index < 3 + kSpareProgramHeaders; ++index) WriteProgramHeader(image, index, 0x6FFFFF01, 0, 0, 0, 0, 1); + std::fill(image.begin() + kCodeOffset, image.begin() + kCodeOffset + kCodeSize, 0xCC); + std::copy(code.begin(), code.end(), image.begin() + kCodeOffset); + for (std::size_t index = 0; index < sizeof(tags) / sizeof(tags[0]); ++index) { + Write(image, kModuleDataOffset + index * 16, tags[index].first); + Write(image, kModuleDataOffset + index * 16 + 8, tags[index].second); + } + Write(image, kModuleDataOffset + 0x240, 1); + Write(image, kModuleDataOffset + 0x244, 1); + return image; +} + class MappedImage { public: explicit MappedImage(const Bytes& elf) { diff --git a/core/relinker/relinker/tests/GuestIntelTrampolinesTests.cpp b/core/relinker/relinker/tests/GuestIntelTrampolinesTests.cpp new file mode 100644 index 000000000..ff50df293 --- /dev/null +++ b/core/relinker/relinker/tests/GuestIntelTrampolinesTests.cpp @@ -0,0 +1,73 @@ +#include "ElfFixture.hpp" +#include "RelinkerProcess.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace RelinkerTests; + +void require(const bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +const Bytes kMainCode = {0xC3}; + +const Bytes kModuleCode = { + 0x66, 0x48, 0x0F, 0x6E, 0xCF, + 0x66, 0x48, 0x0F, 0x6E, 0xD6, + 0x66, 0x0F, 0x78, 0xC1, 0x08, 0x08, + 0xF2, 0x0F, 0x78, 0xD1, 0x08, 0x10, + 0x66, 0x48, 0x0F, 0x7E, 0xD0, + 0xC3, +}; +constexpr std::uint64_t kSites[] = {10, 16}; +constexpr std::size_t kSiteLength = 6; + +std::uint64_t expected(const std::uint64_t value, const std::uint64_t destination) { + return (destination & ~0xFF0000ull) | (((value >> 8) & 0xFF) << 16); +} + +} + +int main(const int argc, char** argv) { + try { + require(argc == 2, "usage: guest_intel_trampolines_tests "); + const TempDirectory directory; + const auto input = directory.Path() / "input.elf"; + const auto output = directory.Path() / "output.elf"; + const auto modules = directory.Path() / "sce_module"; + std::filesystem::create_directory(modules); + WriteFile(input, MakeExecutable(kMainCode)); + WriteFile(modules / "sample.prx", MakeModule(kModuleCode)); + + const auto run = RunRelinker(argv[1], {"--to-intel", input.string(), output.string()}, directory.Path() / "relinker.log"); + require(run.ExitCode == 0, "Relinker failed:\n" + run.Output); + + const auto module = directory.Path() / "app0" / "sce_module" / "sample.prx.guest.prx"; + require(std::filesystem::exists(module), "The relinker did not write the converted guest module:\n" + run.Output); + + const MappedImage image(ReadFile(module)); + for (const auto site : kSites) { + const auto* bytes = image.At(site); + require(bytes[0] == 0xE9, "SSE4a site in the guest module was not replaced by a jump"); + for (std::size_t index = 5; index < kSiteLength; ++index) require(bytes[index] == 0x90, "SSE4a site tail in the guest module is not padded with NOPs"); + std::int32_t displacement; + std::memcpy(&displacement, bytes + 1, sizeof(displacement)); + require(site + 5 + displacement >= kCodeSize, "Guest module SSE4a stub is inside the original code segment"); + } + + const auto function = image.At(0); + for (const auto [value, destination] : {std::pair{0x1122334455667788ull, 0x0123456789ABCDEFull}, {0ull, ~0ull}, {~0ull, 0ull}, {0x9E3779B97F4A7C15ull, 0x0F1E2D3C4B5A6978ull}}) + require(function(value, destination) == expected(value, destination), "Relinked guest module trampolines computed the wrong value"); + std::cout << "Guest Intel trampoline execution tests passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/relinker/relinker/tests/RelaBoundsTests.cpp b/core/relinker/relinker/tests/RelaBoundsTests.cpp new file mode 100644 index 000000000..a445c4f69 --- /dev/null +++ b/core/relinker/relinker/tests/RelaBoundsTests.cpp @@ -0,0 +1,68 @@ +#include "ElfFixture.hpp" +#include "RelinkerProcess.hpp" +#include +#include +#include +#include + +namespace { + +using namespace RelinkerTests; + +void require(const bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +const Bytes kCode = {0xC3}; + +constexpr std::size_t kTagStride = 16; +constexpr std::size_t kTagBase = kCodeOffset + kDynamicVaddr; +constexpr std::size_t kRelaTag = 4; +constexpr std::size_t kRelaSzTag = 5; +constexpr std::int64_t kDtOsRela = 0x6100002f; +constexpr std::int64_t kDtRelaSz = 8; +constexpr std::size_t kImageSize = 0x8000; + +void WriteTag(Bytes& image, const std::size_t index, const std::int64_t tag, const std::uint64_t value) { + Write(image, kTagBase + index * kTagStride, tag); + Write(image, kTagBase + index * kTagStride + 8, value); +} + +void RunAndRequireRejected(const std::string& binary, const Bytes& image, const std::string& what) { + const TempDirectory directory; + const auto input = directory.Path() / "input.elf"; + const auto output = directory.Path() / "output.elf"; + WriteFile(input, image); + const auto run = RunRelinker(binary, {"--skip-sce-module", "--to-intel", input.string(), output.string()}, directory.Path() / "relinker.log"); + require(run.ExitCode != 0, "Relinker accepted a relocation table " + what + " and exited 0:\n" + run.Output); + require(run.Output.find("Relocation entry out of bounds") != std::string::npos, + "Relinker did not report the entry as out of bounds for a table " + what + ":\n" + run.Output); +} + +} + +int main(const int argc, char** argv) { + try { + require(argc == 2, "usage: rela_bounds_tests "); + const std::string binary = argv[1]; + + { + Bytes image = MakeExecutable(kCode); + WriteTag(image, kRelaTag, kDtOsRela, 0xFFFFFFFFFFFFFFF8ull); + WriteTag(image, kRelaSzTag, kDtRelaSz, 24); + RunAndRequireRejected(binary, image, "whose offset is near UINT64_MAX"); + } + + { + Bytes image = MakeExecutable(kCode); + WriteTag(image, kRelaTag, kDtOsRela, kImageSize - 16); + WriteTag(image, kRelaSzTag, kDtRelaSz, 48); + RunAndRequireRejected(binary, image, "that does not fit in the file"); + } + + return 0; + } catch (const std::exception& error) { + std::cerr << "rela_bounds_tests: " << error.what() << '\n'; + return 1; + } +} diff --git a/core/relinker/relinker/tests/StrictNidFilterTests.cpp b/core/relinker/relinker/tests/StrictNidFilterTests.cpp index 63fbdac61..77d77d3f7 100644 --- a/core/relinker/relinker/tests/StrictNidFilterTests.cpp +++ b/core/relinker/relinker/tests/StrictNidFilterTests.cpp @@ -68,6 +68,17 @@ void deadTailAndExplicitEntry() { require(live.ImportSlots == std::set{0x2000}, "Explicit entry after UD2 lost its import"); } +void ud1DeadTail() { + auto input = fixture(); + emit(input, 0, {0x0F, 0xB9, 0xC0}); + importThunk(input, 16, 0x2000); + require(AnalyzeStrictReachability(input).ImportSlots.empty(), "UD1 fallthrough retained a dead import"); + input = fixture(); + ripOperand(input, 0, {0x0F, 0xB9, 0x05}, 0x3000); + importThunk(input, 16, 0x2000); + require(AnalyzeStrictReachability(input).ImportSlots.empty(), "UD1 with a RIP-relative operand retained a dead import"); +} + void conditionalTailCall() { auto input = fixture(); emit(input, 0, {0x75, 0x0E, 0xC3}); @@ -283,6 +294,7 @@ void filterAndPltCompaction() { int main() { try { deadTailAndExplicitEntry(); + ud1DeadTail(); conditionalTailCall(); callbackAndRelocationRoots(); registerImportCall(); diff --git a/core/relinker/relinker/tests/test_call_registry.py b/core/relinker/relinker/tests/test_call_registry.py new file mode 100644 index 000000000..1c56670c4 --- /dev/null +++ b/core/relinker/relinker/tests/test_call_registry.py @@ -0,0 +1,43 @@ +import json +from pathlib import Path +import struct +import subprocess +import sys +import tempfile + +from test_string_table_bounds import fixture + + +def main(): + relinker = Path(sys.argv[1]).resolve() + names = ["symbol", 'quote"slash\\', "caf\u00e9-\u007f"] + names.extend("symbol" + chr(value) for value in range(1, 32)) + with tempfile.TemporaryDirectory(prefix="anyps5-call-registry-") as directory: + work = Path(directory) + for index, name in enumerate(names): + strings = b"\0lib.so\0" + name.encode("utf-8") + b"\0" + image = fixture(str_size=len(strings), table_bytes=strings) + struct.pack_into("= len(strings) or end < 0 or strings[run_path:end] != expected: + raise AssertionError((name, "DT_RUNPATH is outside DT_STRSZ", run_path, tags[DT_STRSZ])) + if dynstr_section_size(elf) != tags[DT_STRSZ]: + raise AssertionError((name, ".dynstr size differs from DT_STRSZ", dynstr_section_size(elf), tags[DT_STRSZ])) + print("Linux dynamic string tests passed") + + +if __name__ == "__main__": + main() diff --git a/core/relinker/relinker/tests/test_linux_eh_frame_header.py b/core/relinker/relinker/tests/test_linux_eh_frame_header.py new file mode 100644 index 000000000..047a3c56b --- /dev/null +++ b/core/relinker/relinker/tests/test_linux_eh_frame_header.py @@ -0,0 +1,72 @@ +from pathlib import Path +import struct +import subprocess +import sys +import tempfile + +PT_LOAD = 1 +PT_DYNAMIC = 2 +PT_GNU_EH_FRAME = 0x6474E550 +PT_SCE_VERSION = 0x6FFFFF01 +EH_FRAME_HEADER = (PT_GNU_EH_FRAME, 4, 0x4800, 0x800, 0x800, 16, 16, 4) + + +def fixture(header_count): + image = bytearray(0x8000) + image[:16] = b"\x7fELF\x02\x01\x01" + bytes(9) + struct.pack_into("> 3), 0xb8 | (target & 7)]) + struct.pack("= 8 else b"") + bytes([0x50 | (target & 7)])) + emit(bytes.fromhex("48 8d a4 24 00 ff ff ff")) + emit(fs_load(0, 0)) + for displacement, value in REGISTER_VALUES.items(): + immediate(1, value) + emit(bytes([0x48, 0x89, 0x48, displacement & 255])) + markers = [0x1020304050607000 + target * 0x101 for target in range(16)] + expected = list(markers) + for target in range(16): + if target != 4: + immediate(target, markers[target]) + for target, value in address.items(): + immediate(target, value) + expected[target] = value & mask + value = REGISTER_VALUES[offset] + expected[register] = value if wide else value & 0xffffffff + emit(b"\x68" + struct.pack("> 3) << 2, 0x89, 0x84 | (target & 7) << 3, 0x24]) + struct.pack("= 8 else b"") + bytes([0x58 | (target & 7)])) + emit(b"\xc3") + if rip_offset is not None: + struct.pack_into("> 3) << 2]) if wide or register >= 8 else b"") + bytes([0x8b, 0x04 | (register & 7) << 3, 0x08]) + restore = {0: bytes.fromhex("48 8d 64 24 08 59"), 1: bytes.fromhex("58 48 8d 64 24 08")}.get(register, bytes.fromhex("58 59")) + return bytes.fromhex("48 8d 64 24 80 51 50") + address + b"\x50", b"\x59" + move + restore + bytes.fromhex("48 8d a4 24 80 00 00 00") + + def make_image(transfer, metadata, extent=None, body=None): image = fixture() image.extend(b"\x90" * 0x1000) @@ -272,8 +382,8 @@ def pe_bytes_at(pe, rva, size): raise AssertionError(f"Unmapped PE RVA {rva:#x}") -def add_alias(image, size): - struct.pack_into("IIBBBBB", 32, 32, 8, 6, 1, 0, 0)) + valid[33:]) - for image in malformed: + for index, image in enumerate(malformed): icon.write_bytes(image) - result = relink() + result, output = relink(f"malformed-{index}") assert result.returncode != 0 and not output.exists(), (result.stdout, result.stderr) assert "Invalid Windows icon PNG" in result.stderr and str(icon) in result.stderr, result.stderr icon.unlink() - result = relink() + result, output = relink("missing-final") assert result.returncode == 0 and resources(output.read_bytes()) is None print("Windows icon integration tests passed (5 images, 11 malformed files, missing icon)") diff --git a/core/relinker/relinker/tests/test_windows_import_modules.py b/core/relinker/relinker/tests/test_windows_import_modules.py index af5ea7894..2ad1d9399 100644 --- a/core/relinker/relinker/tests/test_windows_import_modules.py +++ b/core/relinker/relinker/tests/test_windows_import_modules.py @@ -1,12 +1,14 @@ -from pathlib import Path +import ctypes import os import struct import subprocess import sys import tempfile +from ctypes import wintypes +from pathlib import Path from test_guest_intel_trampolines import main_fixture -from test_guest_module_directories import module_with_symbol +from test_guest_module_directories import module_with_symbol, needed_libraries def dynamic(image, offset, phoff, tags): @@ -29,6 +31,38 @@ def provider(value, soname=None): return image +def check_internal_guest_libc(convert, work, relinker): + result, dummy = convert('empty-native-provider', 'c.prx') + assert result.returncode == 0, result.stderr + result, shared = convert('shared-native-provider', 'a.prx') + assert result.returncode == 0, result.stderr + empty_provider = dummy.parent / 'app0' / 'prx' / 'c.prx.guest.prx' + shared_provider = shared.parent / 'app0' / 'prx' / 'a.prx.guest.prx' + for name, native_owner, symbol, expected in ( + ('guest-only', None, 'shared#A#B', 22), + ('internal-first', 'libSceLibcInternal.prx', 'shared#A#B', 11), + ('native-libc-first', 'libc.prx', 'shared#A#B', 11), + ('unresolved', None, 'absent#A#B', None)): + case = work / ('internal-guest-libc-' + name) + (case / 'sce_module').mkdir(parents=True) + (case / 'sce_module' / 'libc.prx').write_bytes(provider(22)) + source = case / 'input.elf' + source.write_bytes(executable('libSceLibcInternal.prx', symbol, extra_dependencies=('libc.prx',))) + output = case / 'output.exe' + result = subprocess.run([str(relinker), '--windows', str(source), str(output)], capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stderr + (case / 'libs').mkdir() + for owner in ('libSceLibcInternal.prx', 'libc.prx'): + selected = shared_provider if owner == native_owner else empty_provider + (case / 'libs' / owner).write_bytes(selected.read_bytes()) + if os.name == 'nt': + run = subprocess.run([str(output)], capture_output=True, text=True, timeout=30) + if expected is None: + assert run.returncode != 0 and 'unresolved ELF import absent' in run.stderr, (run.returncode, run.stdout, run.stderr) + else: + assert run.returncode == expected, (run.returncode, run.stdout, run.stderr) + + def consumer(owner, symbol='shared#A#B', expected=22, module_name=None): image = module_with_symbol(False) strings = b'\0' + symbol.encode() + b'\0' + owner.encode() + b'\0' @@ -73,12 +107,72 @@ def executable(owner, symbol='shared#A#B', module_name=None, extra_dependencies= return image +def shell_execute_exit_code(executable_path): + see_mask_nocloseprocess = 0x00000040 + wait_object_0 = 0 + wait_timeout = 0x00000102 + + class ShellExecuteInfo(ctypes.Structure): + _fields_ = [ + ('cbSize', wintypes.DWORD), + ('fMask', wintypes.DWORD), + ('hwnd', wintypes.HWND), + ('lpVerb', wintypes.LPCWSTR), + ('lpFile', wintypes.LPCWSTR), + ('lpParameters', wintypes.LPCWSTR), + ('lpDirectory', wintypes.LPCWSTR), + ('nShow', ctypes.c_int), + ('hInstApp', wintypes.HANDLE), + ('lpIDList', wintypes.LPVOID), + ('lpClass', wintypes.LPCWSTR), + ('hkeyClass', wintypes.HANDLE), + ('dwHotKey', wintypes.DWORD), + ('hIconOrMonitor', wintypes.HANDLE), + ('hProcess', wintypes.HANDLE), + ] + + shell32 = ctypes.WinDLL('shell32', use_last_error=True) + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + shell32.ShellExecuteExW.argtypes = [ctypes.POINTER(ShellExecuteInfo)] + shell32.ShellExecuteExW.restype = wintypes.BOOL + kernel32.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD] + kernel32.WaitForSingleObject.restype = wintypes.DWORD + kernel32.GetExitCodeProcess.argtypes = [wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)] + kernel32.GetExitCodeProcess.restype = wintypes.BOOL + kernel32.TerminateProcess.argtypes = [wintypes.HANDLE, wintypes.UINT] + kernel32.TerminateProcess.restype = wintypes.BOOL + kernel32.CloseHandle.argtypes = [wintypes.HANDLE] + kernel32.CloseHandle.restype = wintypes.BOOL + info = ShellExecuteInfo() + info.cbSize = ctypes.sizeof(info) + info.fMask = see_mask_nocloseprocess + info.lpVerb = 'open' + info.lpFile = str(executable_path) + info.nShow = 1 + if not shell32.ShellExecuteExW(ctypes.byref(info)): + raise OSError(ctypes.get_last_error(), 'ShellExecuteExW failed') + try: + result = kernel32.WaitForSingleObject(info.hProcess, 30_000) + if result == wait_timeout: + kernel32.TerminateProcess(info.hProcess, 0xffffffff) + kernel32.WaitForSingleObject(info.hProcess, 0xffffffff) + raise TimeoutError('ShellExecuteExW child did not exit') + if result != wait_object_0: + raise OSError(f'WaitForSingleObject returned {result}') + exit_code = wintypes.DWORD() + if not kernel32.GetExitCodeProcess(info.hProcess, ctypes.byref(exit_code)): + raise OSError(ctypes.get_last_error(), 'GetExitCodeProcess failed') + return exit_code.value + finally: + kernel32.CloseHandle(info.hProcess) + + def main(): relinker = Path(sys.argv[1]).resolve() with tempfile.TemporaryDirectory(prefix='anyps5-import-modules-') as directory: work = Path(directory) - def convert(name, owner, symbol='shared#A#B', guest_owner=None, module_name=None, provider_name='b.prx', extra_dependencies=()): + def convert(name, owner, symbol='shared#A#B', guest_owner=None, module_name=None, provider_name='b.prx', extra_dependencies=(), windows_gui=False): case = work / name modules = case / 'prx' modules.mkdir(parents=True) @@ -94,7 +188,11 @@ def main(): source = case / 'input.elf' source.write_bytes(executable(owner, symbol, module_name, extra_dependencies)) output = case / 'output.exe' - result = subprocess.run([str(relinker), '--windows', str(source), str(output)], capture_output=True, text=True, timeout=30) + arguments = [str(relinker), '--windows'] + if windows_gui: + arguments.append('--windows-gui') + arguments.extend([str(source), str(output)]) + result = subprocess.run(arguments, capture_output=True, text=True, timeout=30) return result, output for owner in ('a.prx', 'b.prx', 'alias.prx'): @@ -104,6 +202,8 @@ def main(): run = subprocess.run([str(output)], capture_output=True, text=True, timeout=30) assert run.returncode == (11 if owner == 'a.prx' else 22), (run.returncode, run.stdout, run.stderr) + check_internal_guest_libc(convert, work, relinker) + for filename, module_name in [('foo.native.prx', 'foo_native'), ('libSceFont-module.prx', 'libSceFont')]: result, output = convert(module_name, filename, guest_owner=filename, @@ -113,6 +213,45 @@ def main(): run = subprocess.run([str(output)], capture_output=True, text=True, timeout=30) assert run.returncode == 22, (run.returncode, run.stdout, run.stderr) + for index, filename in enumerate(('party.prx', 'party.PRX')): + result, output = convert('case-' + str(index), 'Party.prx', guest_owner='Party.prx', provider_name=filename) + assert result.returncode == 0, (result.stdout, result.stderr) + if os.name == 'nt': + run = subprocess.run([str(output)], capture_output=True, text=True, timeout=30) + assert run.returncode == 22, (run.returncode, run.stdout, run.stderr) + + case = work / 'exact-soname-before-filename-case' + modules = case / 'prx' + modules.mkdir(parents=True) + (modules / 'party.prx').write_bytes(provider(11)) + (modules / 'other.prx').write_bytes(provider(22, 'Party.prx')) + (modules / 'consumer.prx').write_bytes(consumer('Party.prx')) + source = case / 'input.elf' + source.write_bytes(executable('Party.prx')) + output = case / 'output.exe' + result = subprocess.run([str(relinker), '--windows', str(source), str(output)], capture_output=True, text=True, timeout=30) + assert result.returncode == 0, (result.stdout, result.stderr) + if os.name == 'nt': + run = subprocess.run([str(output)], capture_output=True, text=True, timeout=30) + assert run.returncode == 22, (run.returncode, run.stdout, run.stderr) + + result, output = convert('wrong-case-soname', 'ALIAS.prx') + assert result.returncode == 0, result.stderr + if os.name == 'nt': + run = subprocess.run([str(output)], capture_output=True, text=True, timeout=30) + assert run.returncode != 0 and 'Failed to load module:' in run.stderr and 'ALIAS.prx' in run.stderr, (run.returncode, run.stderr) + + case = work / 'linux-filename-case' + modules = case / 'prx' + modules.mkdir(parents=True) + (modules / 'party.prx').write_bytes(provider(22)) + source = case / 'input.elf' + source.write_bytes(executable('Party.prx')) + output = case / 'output.elf' + result = subprocess.run([str(relinker), str(source), str(output)], capture_output=True, text=True, timeout=30) + assert result.returncode == 0, (result.stdout, result.stderr) + assert 'Party.prx' in needed_libraries(output.read_bytes()) + result, output = convert('ambiguous-alias', 'foo.native.prx', module_name='foo_native', provider_name='foo.native.prx', extra_dependencies=('foo.native-module.prx',)) assert result.returncode == 2 and 'Ambiguous import module dependency' in result.stderr, result.stderr @@ -143,6 +282,15 @@ def main(): assert result.returncode == 2 and 'Ambiguous guest import' in result.stderr, result.stderr assert not output.exists() + if os.name == 'nt': + for name, owner, symbol, expected in [ + ('missing-symbol-gui', 'a.prx', 'absent#A#B', 0xc0000139), + ('missing-dependency-gui', 'missing.prx', 'shared#A#B', 0xc0000135)]: + result, output = convert(name, owner, symbol, windows_gui=True) + assert result.returncode == 0, result.stderr + run_status = shell_execute_exit_code(output) + assert run_status == expected, (name, hex(run_status), hex(expected)) + result, output = convert('missing-symbol', 'a.prx', 'absent#A#B') assert result.returncode == 0, result.stderr if os.name == 'nt': @@ -158,6 +306,7 @@ def main(): if os.name == 'nt': run = subprocess.run([str(output)], capture_output=True, text=True, timeout=30) assert run.returncode != 0 and 'Failed to load module:' in run.stderr and 'missing.prx' in run.stderr, (run.returncode, run.stderr) + print('Windows module-scoped import tests passed') diff --git a/core/relinker/relinker/tests/windows_image_runner.py b/core/relinker/relinker/tests/windows_image_runner.py new file mode 100644 index 000000000..89afb1d93 --- /dev/null +++ b/core/relinker/relinker/tests/windows_image_runner.py @@ -0,0 +1,48 @@ +import ctypes +import mmap +from pathlib import Path +import struct +import sys + +SYS_ARCH_PRCTL = 158 +ARCH_SET_GS = 0x1001 +THREAD_LOCAL_STORAGE_POINTER = 0x58 +DLL_PROCESS_ATTACH = 1 + + +def run(path, entry): + pe = Path(path).read_bytes() + header = struct.unpack_from(" #include @@ -13,6 +14,7 @@ class RecompileCacheKey { public: static void Build(const RecompileRequest& request, std::vector& key) { key.clear(); + append(key, RuntimeAbi::Version); append(key, request.shader.stage); // The code enters as a hash rather than word by word: the key is built, hashed and compared // on every dispatch and draw. The cache verifies a match against the code it stored. @@ -24,17 +26,14 @@ public: } else { append(key, request.shader.code); } - append(key, request.context.waveSize); - append(key, request.context.userDataBaseRegister); - append(key, request.context.userData.size()); - append(key, request.context.compute); - append(key, request.context.pixel); - append(key, request.context.vertex); - appendMesh(key, request); - append(key, request.target); - append(key, DebugProbeActive()); - append(key, RayTracingStrict()); - append(key, RayTracingMiss()); + appendInterface(key, request); + } + + static void BuildInterface(const RecompileRequest& request, std::vector& key) { + key.clear(); + append(key, RuntimeAbi::Version); + append(key, request.shader.stage); + appendInterface(key, request); } // A hash over every field Build appends except the code, the target and the probe flag: the @@ -44,6 +43,7 @@ public: struct ContextKeyStorage {}; auto& key = HostThreadLocal, ContextKeyStorage>(); key.clear(); + append(key, RuntimeAbi::Version); append(key, request.shader.stage); append(key, request.context.waveSize); append(key, request.context.userDataBaseRegister); @@ -51,7 +51,9 @@ public: append(key, request.context.compute); append(key, request.context.pixel); append(key, request.context.vertex); + append(key, request.context.floatMode); appendMesh(key, request); + appendTessellation(key, request); std::uint64_t hash = 0xcbf29ce484222325ull; for (const auto value : key) { hash ^= value; @@ -74,6 +76,22 @@ public: } private: + static void appendInterface(std::vector& key, const RecompileRequest& request) { + append(key, request.context.waveSize); + append(key, request.context.userDataBaseRegister); + append(key, request.context.userData.size()); + append(key, request.context.compute); + append(key, request.context.pixel); + append(key, request.context.vertex); + append(key, request.context.floatMode); + appendMesh(key, request); + appendTessellation(key, request); + append(key, request.target); + append(key, DebugProbeActive()); + append(key, RayTracingStrict()); + append(key, RayTracingMiss()); + } + static void appendMesh(std::vector& key, const RecompileRequest& request) { if (request.shader.stage != ShaderStage::Mesh) return; const auto* mesh = request.graphics && request.graphics->mesh ? &*request.graphics->mesh : nullptr; @@ -82,6 +100,14 @@ private: for (const auto value : {mesh->inputPrimitive, mesh->primitivesPerGroup, mesh->verticesPerGroup, mesh->maxVertices, mesh->maxPrimitives, mesh->threadsPerGroup, mesh->ldsSizeDwords, mesh->provokingVertex, mesh->esgsItemSize}) append(key, value); } + static void appendTessellation(std::vector& key, const RecompileRequest& request) { + if (request.shader.stage != ShaderStage::Local && request.shader.stage != ShaderStage::TessellationControl && request.shader.stage != ShaderStage::TessellationEvaluation) return; + const auto* tessellation = request.graphics && request.graphics->tessellation ? &*request.graphics->tessellation : nullptr; + append(key, tessellation != nullptr); + if (tessellation == nullptr) return; + for (const auto value : {tessellation->inputControlPoints, tessellation->outputControlPoints, tessellation->domain, tessellation->partitioning, tessellation->outputTopology}) append(key, value); + } + template static void append(std::vector& key, TValue value) requires (std::is_integral_v || std::is_enum_v) { key.push_back(static_cast(value)); @@ -109,6 +135,13 @@ private: for (const unsigned char byte : value) append(key, byte); } + static void append(std::vector& key, const ShaderFloatMode& value) { + append(key, value.floatMode); + append(key, value.dx10Clamp); + append(key, value.ieeeMode); + append(key, value.fp16Overflow); + } + static void append(std::vector& key, const ShaderComputeStageInfo& value) { append(key, value.numThreads); append(key, value.ldsSizeDwords); @@ -116,6 +149,7 @@ private: append(key, value.tgSizeEnable); append(key, value.threadIdComponentCount); append(key, value.PartialGroups()); + append(key, value.scratchDwords); } static void append(std::vector& key, const ShaderPixelStageInfo& value) { @@ -141,8 +175,8 @@ private: append(key, value.earlyZ); append(key, value.executeOnNoop); append(key, value.conservativeZExport); + append(key, value.orderedPixelShader); append(key, value.targetOutputMode); - append(key, value.targetExportMapping); } static void append(std::vector& key, const ShaderVertexResourceDestination& value) { @@ -153,15 +187,16 @@ private: } static void append(std::vector& key, const ShaderVertexStageInfo& value) { - append(key, value.resourcesNum); append(key, value.fetchAttribReg); append(key, value.fetchBufferReg); append(key, value.fetchEmbedded); if (value.resourcesNum > value.resources.size()) throw std::runtime_error("Shader cache: invalid vertex resource count"); + append(key, value.resourcesNum); for (std::uint32_t i = 0; i < value.resourcesNum; ++i) { - append(key, value.resources[i].fields[1] & 0xffff0000u); - append(key, value.resources[i].fields[3]); - append(key, value.resourcesDst[i]); + if (!value.fetchEmbedded) append(key, VertexInputNumericClass(static_cast((value.resources[i].fields[3] >> 12u) & 0x7fu))); + auto destination = value.resourcesDst[i]; + if (value.fetchEmbedded) destination.fetchIndex = 0; + append(key, destination); } } @@ -202,6 +237,7 @@ private: append(key, value.tessellation); append(key, value.nonConstantImageOffsets); append(key, value.srgbDecodeFormats); + append(key, value.narrowSubgroupClock); } }; diff --git a/core/shader/recompiler/CompiledVariant.hpp b/core/shader/recompiler/CompiledVariant.hpp index c2439554e..00b780914 100644 --- a/core/shader/recompiler/CompiledVariant.hpp +++ b/core/shader/recompiler/CompiledVariant.hpp @@ -4,16 +4,14 @@ #include "Recompiler.hpp" #include "IntermediateRepresentation/IrMetadata/CompiledShaderInfo.hpp" #include "Optimization/include/Optimization/BindingAllocator.hpp" -#include "Optimization/include/Optimization/ResourceMaterializer.hpp" namespace ShaderRecompiler { struct CompiledVariant { - ResourceSpecialization specialization; BindingLayout layout; CompiledShaderInfo info; - BindingAllocationResult bindings; - RecompileResult result; + CompiledBindingLayout bindings; + CompiledShaderArtifact artifact; }; } diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp index 3891e7860..d93905ba9 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp @@ -108,6 +108,8 @@ struct ControlFlowGraph { std::vector codeTableLoadProgramCounters; std::vector codeTableLoads; std::uint32_t entryBlock = InvalidControlFlowId; + bool hasFetchCall = false; + std::uint32_t fetchCallProgramCounter = 0; bool irreducible = false; bool unsupported = false; FailureKind failureKind = FailureKind::None; diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp index 58934a077..bca0da8c9 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp @@ -3,17 +3,34 @@ #include "ControlFlow/ControlFlowGraph.hpp" #include "RdnaDecoder/RdnaProgram.hpp" +#include #include namespace ShaderRecompiler { +struct SwappcInfo { + bool fetchCallAllowed = false; + std::uint32_t userDataBaseRegister = 0; + std::uint32_t userDataCount = 0; +}; + +struct SwappcCall { + std::uint32_t callIndex = 0; + std::uint32_t linkRegister = 0; + bool fetch = false; + std::uint32_t targetIndex = 0; + std::uint32_t targetProgramCounter = 0; + std::uint32_t returnIndex = 0; + std::uint32_t returnTargetProgramCounter = 0; +}; + class GraphBuilder { public: - [[nodiscard]] ControlFlowGraph Build(const RdnaProgram& program) const; + [[nodiscard]] ControlFlowGraph Build(const RdnaProgram& program, const SwappcInfo* swappc = nullptr) const; private: - [[nodiscard]] std::vector splitIntoBlocks(const RdnaProgram& program) const; - void linkBlocks(std::vector& blocks, const RdnaProgram& program) const; + [[nodiscard]] std::vector splitIntoBlocks(const RdnaProgram& program, const std::vector& calls) const; + void linkBlocks(std::vector& blocks, const RdnaProgram& program, const std::vector& calls) const; }; } diff --git a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp index cfcb917ff..1bd3c7776 100644 --- a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp +++ b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp @@ -146,7 +146,7 @@ bool resolveLongSetpcTarget(const RdnaProgram& program, std::uint32_t setpcIndex const bool adds = low.op == RdnaOpcode::SAddU32 && high.op == RdnaOpcode::SAddcU32; const bool subtracts = low.op == RdnaOpcode::SSubU32 && high.op == RdnaOpcode::SSubbU32; std::uint32_t lowImmediate = 0, highImmediate = 0; - if (setpc.op != RdnaOpcode::SSetpcB64 || pcRegister == NoScalarRegister || pcRegister % 2u != 0u || pc.op != RdnaOpcode::SGetpcB64 || !isScalar(pc.destination, pcRegister)) return false; + if (!(setpc.op == RdnaOpcode::SSetpcB64 || setpc.op == RdnaOpcode::SSwappcB64) || pcRegister == NoScalarRegister || pcRegister % 2u != 0u || pc.op != RdnaOpcode::SGetpcB64 || !isScalar(pc.destination, pcRegister)) return false; if (adds ? !addsImmediateTo(low, pcRegister, lowImmediate) || !addsImmediateTo(high, pcRegister + 1u, highImmediate) : !subtracts || !subtractsImmediateFrom(low, pcRegister, lowImmediate) || !subtractsImmediateFrom(high, pcRegister + 1u, highImmediate)) return false; const auto offset = (static_cast(highImmediate) << 32u) | lowImmediate; @@ -166,7 +166,7 @@ bool resolveSetpcTarget(const RdnaProgram& program, std::uint32_t setpcIndex, st } const auto& setpc = program.instructions[setpcIndex]; - if (setpc.op != RdnaOpcode::SSetpcB64 || setpc.source0.kind != RdnaOperandKind::ScalarRegister) { + if (!(setpc.op == RdnaOpcode::SSetpcB64 || setpc.op == RdnaOpcode::SSwappcB64) || setpc.source0.kind != RdnaOperandKind::ScalarRegister) { return false; } @@ -392,6 +392,239 @@ bool resolveBoundedJumpTable(const RdnaProgram& program, std::uint32_t index, Bo return resolveDwordJumpTable(program, index, result); } +bool pairOverlapsRegister(const RdnaOperand& operand, std::uint32_t linkRegister, std::uint32_t count = 1u) { + const std::uint32_t slot = scalarIndex(operand); + return slot != NoScalarRegister && slot <= linkRegister + 1u && linkRegister < slot + count; +} + +bool touchesLinkRegister(const RdnaInstruction& instruction, std::uint32_t linkRegister) { + if (instruction.op == RdnaOpcode::SMovrelsB32 || instruction.op == RdnaOpcode::SMovrelsB64) return true; + std::uint32_t source0Count = 1u, source1Count = 1u, source2Count = 1u; + switch (instruction.family) { + case RdnaInstructionFamily::SMEM: + source0Count = 2u; + switch (instruction.op) { + case RdnaOpcode::SBufferLoadDword: + case RdnaOpcode::SBufferLoadDwordx2: + case RdnaOpcode::SBufferLoadDwordx4: + case RdnaOpcode::SBufferLoadDwordx8: + case RdnaOpcode::SBufferLoadDwordx16: source0Count = 4u; break; + default: break; + } + break; + case RdnaInstructionFamily::MUBUF: + case RdnaInstructionFamily::MTBUF: source1Count = 4u; break; + case RdnaInstructionFamily::MIMG: + source1Count = instruction.imageR128 ? 4u : 8u; + source2Count = 4u; + break; + case RdnaInstructionFamily::FLAT: + source0Count = 2u; + source1Count = instruction.memorySegment == 2u ? 2u : 1u; + break; + default: break; + } + return writesScalar(instruction, linkRegister) || writesScalar(instruction, linkRegister + 1u) || + pairOverlapsRegister(instruction.destination, linkRegister) || pairOverlapsRegister(instruction.destination2, linkRegister) || + pairOverlapsRegister(instruction.source0, linkRegister, source0Count) || pairOverlapsRegister(instruction.source1, linkRegister, source1Count) || + pairOverlapsRegister(instruction.source2, linkRegister, source2Count) || pairOverlapsRegister(instruction.source3, linkRegister); +} + +bool isFetchCallPrefixOpcode(const RdnaInstruction& instruction) { + return IsScalarAluOpcode(instruction.op) || instruction.op == RdnaOpcode::SNop || instruction.op == RdnaOpcode::SWaitcnt || + instruction.op == RdnaOpcode::SWaitcntDepctr || instruction.op == RdnaOpcode::SWaitIdle || instruction.op == RdnaOpcode::SIcacheInv || + instruction.op == RdnaOpcode::SSetprio || instruction.op == RdnaOpcode::SClause; +} + +std::uint32_t instructionIndexOfProgramCounter(const RdnaProgram& program, std::uint32_t programCounter) { + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + if (program.instructions[index].programCounter == programCounter) { + return index; + } + } + return InvalidControlFlowId; +} + +const SwappcCall* findCallAt(const std::vector& calls, std::uint32_t index) { + for (const auto& call : calls) { + if (call.callIndex == index) { + return &call; + } + } + return nullptr; +} + +const SwappcCall* findReturnAt(const std::vector& calls, std::uint32_t index) { + for (const auto& call : calls) { + if (!call.fetch && call.returnIndex == index) { + return &call; + } + } + return nullptr; +} + +std::vector analyzeSwappcCalls(const RdnaProgram& program, const SwappcInfo* swappc) { + std::vector calls; + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + const auto& instruction = program.instructions[index]; + if (instruction.op != RdnaOpcode::SSwappcB64 && instruction.op != RdnaOpcode::SCallB64) { + continue; + } + const std::uint32_t linkRegister = scalarIndex(instruction.destination); + if (linkRegister == NoScalarRegister || linkRegister % 2u != 0u || linkRegister > 104u) { + throw std::invalid_argument("unsupported scalar call link register at program counter " + toHexString(instruction.programCounter) + ": the link must be an ordinary aligned scalar register pair"); + } + SwappcCall call; + call.callIndex = index; + call.linkRegister = linkRegister; + for (const auto& other : calls) { + if (call.linkRegister <= other.linkRegister + 1u && other.linkRegister <= call.linkRegister + 1u) { + throw std::invalid_argument("unclosable scalar call/return pairing at program counter " + toHexString(instruction.programCounter) + + ": link register s[" + std::to_string(call.linkRegister) + "] is shared with another call"); + } + } + std::uint32_t target = instruction.branchTarget; + const bool staticTarget = instruction.op == RdnaOpcode::SCallB64 || resolveSetpcTarget(program, index, target); + call.targetIndex = staticTarget ? instructionIndexOfProgramCounter(program, target) : InvalidControlFlowId; + if (instruction.op == RdnaOpcode::SCallB64 && call.targetIndex == InvalidControlFlowId) { + throw std::invalid_argument("s_call_b64 at program counter " + toHexString(instruction.programCounter) + " targets invalid instruction boundary " + toHexString(target)); + } + const bool positional = std::all_of(program.instructions.begin(), program.instructions.begin() + index, isFetchCallPrefixOpcode); + const std::uint32_t targetRegister = scalarIndex(instruction.source0); + const bool userDataPair = swappc != nullptr && targetRegister != NoScalarRegister && + targetRegister >= swappc->userDataBaseRegister && targetRegister + 1u < swappc->userDataBaseRegister + swappc->userDataCount && + std::none_of(program.instructions.begin(), program.instructions.end(), [&](const RdnaInstruction& other) { + return writesScalar(other, targetRegister) || writesScalar(other, targetRegister + 1u); + }); + const bool outsideProgram = staticTarget && call.targetIndex == InvalidControlFlowId; + if (instruction.op == RdnaOpcode::SSwappcB64 && swappc != nullptr && swappc->fetchCallAllowed && positional && (userDataPair || outsideProgram)) { + call.fetch = true; + call.returnTargetProgramCounter = instructionEndProgramCounter(instruction); + calls.push_back(call); + continue; + } + if (!staticTarget || call.targetIndex == InvalidControlFlowId) { + throw std::invalid_argument("computed/data-dependent s_swappc_b64 call target at program counter " + toHexString(instruction.programCounter) + " is not statically resolvable"); + } + call.targetProgramCounter = target; + calls.push_back(call); + } + + for (auto& call : calls) { + std::uint32_t returnIndex = InvalidControlFlowId; + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + if (index == call.callIndex) { + continue; + } + const auto& instruction = program.instructions[index]; + if (instruction.op == RdnaOpcode::SSetpcB64 && isScalar(instruction.source0, call.linkRegister)) { + if (returnIndex != InvalidControlFlowId) { + throw std::invalid_argument("unclosable scalar call/return pairing at program counter " + toHexString(program.instructions[call.callIndex].programCounter) + + ": multiple s_setpc_b64 returns read link register s[" + std::to_string(call.linkRegister) + "]"); + } + returnIndex = index; + continue; + } + if (touchesLinkRegister(instruction, call.linkRegister)) { + throw std::invalid_argument("scalar call link register s[" + std::to_string(call.linkRegister) + "] written at program counter " + + toHexString(program.instructions[call.callIndex].programCounter) + " escapes the constant-offset call/return model (used at program counter " + + toHexString(instruction.programCounter) + ")"); + } + } + if (call.fetch) { + if (returnIndex != InvalidControlFlowId) { + throw std::invalid_argument("unclosable s_swappc_b64 fetch-shader call at program counter " + toHexString(program.instructions[call.callIndex].programCounter) + + ": its s_setpc_b64 return must live in the fetch-shader code"); + } + continue; + } + if (returnIndex == InvalidControlFlowId) { + throw std::invalid_argument("unclosable scalar call/return pairing at program counter " + toHexString(program.instructions[call.callIndex].programCounter) + + ": no paired s_setpc_b64 return reads link register s[" + std::to_string(call.linkRegister) + "]"); + } + if (returnIndex < call.targetIndex) { + throw std::invalid_argument("unclosable scalar call/return pairing: return precedes the call target at program counter " + + toHexString(program.instructions[call.callIndex].programCounter)); + } + call.returnIndex = returnIndex; + call.returnTargetProgramCounter = instructionEndProgramCounter(program.instructions[call.callIndex]); + } + + for (std::size_t first = 0; first < calls.size(); ++first) { + if (!calls[first].fetch && calls[first].callIndex >= calls[first].targetIndex && calls[first].callIndex <= calls[first].returnIndex) { + throw std::invalid_argument("recursive scalar call at program counter " + toHexString(program.instructions[calls[first].callIndex].programCounter) + + ": the call executes inside its own call region"); + } + for (std::size_t second = first + 1u; second < calls.size(); ++second) { + if (calls[first].fetch || calls[second].fetch) { + continue; + } + const bool disjoint = calls[first].returnIndex < calls[second].targetIndex || calls[second].returnIndex < calls[first].targetIndex; + const bool firstContainsSecond = calls[first].targetIndex <= calls[second].targetIndex && calls[second].returnIndex <= calls[first].returnIndex; + const bool secondContainsFirst = calls[second].targetIndex <= calls[first].targetIndex && calls[first].returnIndex <= calls[second].returnIndex; + if (!disjoint && !firstContainsSecond && !secondContainsFirst) { + throw std::invalid_argument("recursive scalar calls at program counter " + toHexString(program.instructions[calls[first].callIndex].programCounter) + + " and program counter " + toHexString(program.instructions[calls[second].callIndex].programCounter) + ": call regions interleave"); + } + } + } + + const bool anyStaticCall = std::any_of(calls.begin(), calls.end(), [](const SwappcCall& call) { return !call.fetch; }); + if (anyStaticCall) { + struct Transfer { + std::uint32_t index; + std::uint32_t targetIndex; + }; + std::vector transfers; + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + const auto& instruction = program.instructions[index]; + if (IsDirectBranchOpcode(instruction.op)) { + transfers.push_back(Transfer{index, instructionIndexOfProgramCounter(program, instruction.branchTarget)}); + if (IsConditionalBranchOpcode(instruction.op)) transfers.push_back(Transfer{index, index + 1u}); + continue; + } + if (instruction.op == RdnaOpcode::SSetpcB64) { + if (findReturnAt(calls, index) != nullptr) { + continue; + } + std::uint32_t target = 0; + if (!resolveSetpcTarget(program, index, target)) { + throw std::invalid_argument("unclosable scalar call region in a program with calls: dynamic s_setpc_b64 at program counter " + + toHexString(instruction.programCounter) + " cannot be confined to a call region"); + } + transfers.push_back(Transfer{index, instructionIndexOfProgramCounter(program, target)}); + continue; + } + if (instruction.op == RdnaOpcode::SSwappcB64 || instruction.op == RdnaOpcode::SCallB64) { + const SwappcCall* call = findCallAt(calls, index); + transfers.push_back(Transfer{index, call->fetch ? index + 1u : call->targetIndex}); + continue; + } + if (instruction.op != RdnaOpcode::SEndpgm && instruction.op != RdnaOpcode::SCodeEnd) transfers.push_back(Transfer{index, index + 1u}); + } + for (const auto& call : calls) { + if (call.fetch) { + continue; + } + for (const auto& transfer : transfers) { + const bool sourceInside = transfer.index >= call.targetIndex && transfer.index <= call.returnIndex; + const bool targetInside = transfer.targetIndex != InvalidControlFlowId && transfer.targetIndex >= call.targetIndex && transfer.targetIndex <= call.returnIndex; + if (!sourceInside && targetInside && transfer.index != call.callIndex) { + throw std::invalid_argument("unclosable scalar call/return pairing at program counter " + + toHexString(program.instructions[call.callIndex].programCounter) + ": control flow enters the call region from program counter " + + toHexString(program.instructions[transfer.index].programCounter)); + } + if (sourceInside && !targetInside && transfer.index != call.returnIndex) { + throw std::invalid_argument("unclosable scalar call/return pairing at program counter " + + toHexString(program.instructions[call.callIndex].programCounter) + ": control flow leaves the call region at program counter " + + toHexString(program.instructions[transfer.index].programCounter)); + } + } + } + } + return calls; +} + BranchCondition conditionForOpcode(RdnaOpcode opcode) { switch (opcode) { case RdnaOpcode::SBranch: return BranchCondition::Always; @@ -472,7 +705,7 @@ void pruneUnreachableBlocks(ControlFlowGraph& graph) { } -std::vector GraphBuilder::splitIntoBlocks(const RdnaProgram& program) const { +std::vector GraphBuilder::splitIntoBlocks(const RdnaProgram& program, const std::vector& calls) const { if (program.instructions.empty()) { throw std::invalid_argument("cannot build a control flow graph for an empty program"); } @@ -503,7 +736,9 @@ std::vector GraphBuilder::splitIntoBlocks(const RdnaProgram& program } } else if (instruction.op == RdnaOpcode::SSetpcB64) { std::uint32_t target = 0; - if (!resolveSetpcTarget(program, index, target)) { + if (const SwappcCall* returnCall = findReturnAt(calls, index); returnCall != nullptr) { + target = returnCall->returnTargetProgramCounter; + } else if (!resolveSetpcTarget(program, index, target)) { BoundedJumpTable table; if (!resolveBoundedJumpTable(program, index, table)) throw std::invalid_argument("unsupported dynamic s_setpc_b64 at program counter " + toHexString(instruction.programCounter)); for (const auto tableTarget : table.targets) { @@ -520,6 +755,14 @@ std::vector GraphBuilder::splitIntoBlocks(const RdnaProgram& program if (nextProgramCounter <= endProgramCounter) { labels.insert(nextProgramCounter); } + } else if (instruction.op == RdnaOpcode::SSwappcB64 || instruction.op == RdnaOpcode::SCallB64) { + const SwappcCall* call = findCallAt(calls, index); + if (call != nullptr && !call->fetch) { + labels.insert(call->targetProgramCounter); + if (nextProgramCounter <= endProgramCounter) { + labels.insert(nextProgramCounter); + } + } } else if (instruction.op == RdnaOpcode::SEndpgm) { labels.insert(nextProgramCounter); } @@ -550,7 +793,7 @@ std::vector GraphBuilder::splitIntoBlocks(const RdnaProgram& program return blocks; } -void GraphBuilder::linkBlocks(std::vector& blocks, const RdnaProgram& program) const { +void GraphBuilder::linkBlocks(std::vector& blocks, const RdnaProgram& program, const std::vector& calls) const { std::map programCounterToBlock; for (const auto& block : blocks) { programCounterToBlock.emplace(block.startProgramCounter, block.id); @@ -570,7 +813,11 @@ void GraphBuilder::linkBlocks(std::vector& blocks, const RdnaProgram block.terminator.kind = TerminatorKind::Return; } else if (last.op == RdnaOpcode::SSetpcB64) { std::uint32_t target = 0; - if (!resolveSetpcTarget(program, block.instructionEnd - 1u, target)) { + if (const SwappcCall* returnCall = findReturnAt(calls, block.instructionEnd - 1u); returnCall != nullptr) { + block.terminator.kind = TerminatorKind::Branch; + block.terminator.condition = BranchCondition::Always; + block.terminator.trueBlock = programCounterToBlock.at(returnCall->returnTargetProgramCounter); + } else if (!resolveSetpcTarget(program, block.instructionEnd - 1u, target)) { BoundedJumpTable table; if (!resolveBoundedJumpTable(program, block.instructionEnd - 1u, table)) throw std::invalid_argument("unsupported dynamic s_setpc_b64 at program counter " + toHexString(last.programCounter)); block.terminator.kind = TerminatorKind::IndirectBranch; @@ -582,11 +829,18 @@ void GraphBuilder::linkBlocks(std::vector& blocks, const RdnaProgram addUnique(block.successors, programCounterToBlock.at(tableTarget)); } continue; + } else { + if (block.instructionEnd - block.instructionBegin < 4u && resolveLongSetpcTarget(program, block.instructionEnd - 1u, target)) throw std::invalid_argument("long branch at program counter " + toHexString(last.programCounter) + " does not start in its block"); + block.terminator.kind = TerminatorKind::Branch; + block.terminator.condition = BranchCondition::Always; + block.terminator.trueBlock = programCounterToBlock.at(target); } - if (block.instructionEnd - block.instructionBegin < 4u && resolveLongSetpcTarget(program, block.instructionEnd - 1u, target)) throw std::invalid_argument("long branch at program counter " + toHexString(last.programCounter) + " does not start in its block"); + } else if (const SwappcCall* call = findCallAt(calls, block.instructionEnd - 1u); call != nullptr && !call->fetch) { + std::uint32_t ignoredTarget = 0; + if (block.instructionEnd - block.instructionBegin < 4u && resolveLongSetpcTarget(program, block.instructionEnd - 1u, ignoredTarget)) throw std::invalid_argument("long branch at program counter " + toHexString(last.programCounter) + " does not start in its block"); block.terminator.kind = TerminatorKind::Branch; block.terminator.condition = BranchCondition::Always; - block.terminator.trueBlock = programCounterToBlock.at(target); + block.terminator.trueBlock = programCounterToBlock.at(call->targetProgramCounter); } else if (last.op == RdnaOpcode::SBranch) { block.terminator.kind = TerminatorKind::Branch; block.terminator.condition = BranchCondition::Always; @@ -636,10 +890,17 @@ void GraphBuilder::linkBlocks(std::vector& blocks, const RdnaProgram } } -ControlFlowGraph GraphBuilder::Build(const RdnaProgram& program) const { +ControlFlowGraph GraphBuilder::Build(const RdnaProgram& program, const SwappcInfo* swappc) const { + const std::vector calls = analyzeSwappcCalls(program, swappc); ControlFlowGraph graph; - graph.blocks = splitIntoBlocks(program); - linkBlocks(graph.blocks, program); + graph.blocks = splitIntoBlocks(program, calls); + linkBlocks(graph.blocks, program, calls); + for (const auto& call : calls) { + if (call.fetch) { + graph.hasFetchCall = true; + graph.fetchCallProgramCounter = program.instructions[call.callIndex].programCounter; + } + } const auto originalSize = graph.blocks.size(); for (std::uint32_t id = 0; id < originalSize; ++id) { const auto term = graph.blocks[id].terminator; diff --git a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp index a6e1b095c..48386cd20 100644 --- a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp +++ b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp @@ -305,6 +305,7 @@ void writePixelInfo(Writer& writer, const ShaderPixelStageInfo& info) { writer.WriteBool(info.earlyZ); writer.WriteBool(info.executeOnNoop); writer.WriteU8(static_cast(info.conservativeZExport)); + writer.WriteBool(info.orderedPixelShader); for (const std::uint8_t value : info.targetOutputMode) { writer.WriteU8(value); } @@ -341,6 +342,7 @@ ShaderPixelStageInfo readPixelInfo(Reader& reader, std::uint32_t version) { info.earlyZ = reader.ReadBool(); info.executeOnNoop = reader.ReadBool(); if (version >= 7u) info.conservativeZExport = readConservativeZExport(reader); + if (version >= 11u) info.orderedPixelShader = reader.ReadBool(); for (std::uint8_t& value : info.targetOutputMode) { value = reader.ReadU8(); } @@ -517,6 +519,13 @@ void writeGuestContext(Writer& writer, const GuestContext& context) { writer.WriteU64(region.guestAddress); writer.WriteBytes(region.bytes); } + writer.WriteBool(context.floatMode.has_value()); + if (context.floatMode.has_value()) { + writer.WriteU32(context.floatMode->floatMode); + writer.WriteBool(context.floatMode->dx10Clamp); + writer.WriteBool(context.floatMode->ieeeMode); + writer.WriteBool(context.floatMode->fp16Overflow); + } } GuestContext readGuestContext(Reader& reader, DeserializedRequest& result, std::uint32_t version) { @@ -549,6 +558,14 @@ GuestContext readGuestContext(Reader& reader, DeserializedRequest& result, std:: result.memory.push_back(region); } context.memory = result.memory; + if (version >= 13u && reader.ReadBool()) { + ShaderFloatMode mode; + mode.floatMode = reader.ReadU32(); + mode.dx10Clamp = reader.ReadBool(); + mode.ieeeMode = reader.ReadBool(); + mode.fp16Overflow = reader.ReadBool(); + context.floatMode = mode; + } return context; } @@ -697,7 +714,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const std::string buffer; Writer writer(buffer); writer.WriteU32(0x41505335u); - writer.WriteU32(9u); + writer.WriteU32(13u); writeShaderBinary(writer, request.shader); writeGuestContext(writer, request.context); writeSpirvTarget(writer, request.target); @@ -714,6 +731,8 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const } writer.WriteBool(request.target.nonConstantImageOffsets); writer.WriteU32(request.target.srgbDecodeFormats); + if (request.context.compute.has_value()) writer.WriteU32(request.context.compute->scratchDwords); + writer.WriteBool(request.target.narrowSubgroupClock); return base64Encode(buffer); } @@ -722,7 +741,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const Reader reader(decoded); if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature"); const auto version = reader.ReadU32(); - if (version < 1u || version > 9u) throw std::runtime_error("unsupported recompile request serialization version"); + if (version < 1u || version > 13u) throw std::runtime_error("unsupported recompile request serialization version"); DeserializedRequest result{}; result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader); result.request.context = readGuestContext(reader, result, version); @@ -740,6 +759,8 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const } if (version >= 6u) result.request.target.nonConstantImageOffsets = reader.ReadBool(); if (version >= 9u) result.request.target.srgbDecodeFormats = reader.ReadU32(); + if (version >= 10u && result.request.context.compute.has_value()) result.request.context.compute->scratchDwords = reader.ReadU32(); + if (version >= 12u) result.request.target.narrowSubgroupClock = reader.ReadBool(); return result; } diff --git a/core/shader/recompiler/Deserialize.cpp b/core/shader/recompiler/Deserialize.cpp index 59f3c075d..c7379f1a7 100644 --- a/core/shader/recompiler/Deserialize.cpp +++ b/core/shader/recompiler/Deserialize.cpp @@ -166,19 +166,9 @@ int main(int argc, char** argv) { translateOptions.waveSize = request.context.waveSize; translateOptions.userDataBaseRegister = request.context.userDataBaseRegister; translateOptions.userDataCount = static_cast(request.context.userData.size()); + translateOptions.scratchDwords = request.context.compute.has_value() ? request.context.compute->scratchDwords : 0u; translateOptions.inputInfo = inputInfo; - EmbeddedFetchPlan embeddedFetch; - - if ((stageKind == ShaderStageKind::Vertex || stageKind == ShaderStageKind::Local) && inputInfo.vertex != nullptr && inputInfo.vertex->fetchEmbedded) { - EmbeddedVertexFetchAnalyzer embeddedFetchAnalyzer; - embeddedFetch = embeddedFetchAnalyzer.Analyze(decoded, inputInfo.vertex->fetchAttribReg, inputInfo.vertex->fetchBufferReg, request.context.userDataBaseRegister, static_cast(request.context.userData.size()), request.context.waveSize); - } - - translateOptions.embeddedFetch = embeddedFetch.loads.empty() ? nullptr : &embeddedFetch; - - std::cout << "embedded fetch loads: " << embeddedFetch.loads.size() << "\n"; - InstructionTranslator translator; auto program = translator.Translate(decoded, cfg, translateOptions); diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index 5447bef4e..eed5b9551 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -50,6 +50,14 @@ struct SrtRead { bool operator==(const SrtRead& other) const = default; }; +struct SrtReadPoison { + std::uint32_t slot = 0; + std::uint32_t pc = 0; + std::uint64_t address = 0; + + bool operator==(const SrtReadPoison& other) const = default; +}; + struct ResourceBlock { IrValue* condition = nullptr; std::vector successors; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp index bd2c34d3b..91952766e 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp @@ -1,6 +1,7 @@ #ifndef CORE_SHADER_RECOMPILIER_INTERMEDIATEREPRESENTATION_INCLUDE_INTERMEDIATEREPRESENTATION_IRMETADATA_DESCRIPTORBINDING_HPP #define CORE_SHADER_RECOMPILIER_INTERMEDIATEREPRESENTATION_INCLUDE_INTERMEDIATEREPRESENTATION_IRMETADATA_DESCRIPTORBINDING_HPP +#include "RuntimeAbi.hpp" #include #include #include @@ -8,24 +9,15 @@ namespace ShaderRecompiler { -inline constexpr std::uint32_t FirstImageBinding = 1u; -inline constexpr std::uint32_t FirstComparisonImageBinding = 22u; -inline constexpr std::uint32_t FirstStorageImageBinding = 29u; -inline constexpr std::uint32_t ImageBindingCount = 48u; +inline constexpr std::uint32_t FirstImageBinding = RuntimeAbi::FirstImageBinding; +inline constexpr std::uint32_t FirstComparisonImageBinding = RuntimeAbi::FirstComparisonImageBinding; +inline constexpr std::uint32_t FirstStorageImageBinding = RuntimeAbi::FirstStorageImageBinding; +inline constexpr std::uint32_t ImageBindingCount = RuntimeAbi::ImageBindingCount; -enum class DescriptorBindingKind : std::uint32_t { - Buffers = 0u, - Samplers = FirstImageBinding + ImageBindingCount, - Gds, - BdaPagetable, - FaultBuffer, - FlattenedSrt, - ShaderData, - Count, -}; +using DescriptorBindingKind = RuntimeAbi::Binding; struct PushData { - static constexpr std::uint32_t DwordCount = 32; + static constexpr std::uint32_t DwordCount = RuntimeAbi::PushConstantDwords; static constexpr std::uint32_t MeshDrawDwordCount = 6; static constexpr std::uint32_t NoStart = std::numeric_limits::max(); std::array dwords {}; @@ -50,15 +42,19 @@ struct IrBindingLayout { std::uint32_t memoryOffsetDword = 0; std::uint32_t memoryOffsetCount = 0; bool dispatchThreadLimit = false; + std::uint32_t runtimeImageCount = 0; std::vector userDataRegisters; std::vector descriptors; [[nodiscard]] std::uint32_t DispatchThreadLimitDword() const { return memoryOffsetDword + (memoryOffsetCount + 3u) / 4u; } - [[nodiscard]] std::uint32_t ShaderDataDwords() const { + [[nodiscard]] std::uint32_t ImageMetadataDword() const { return DispatchThreadLimitDword() + (dispatchThreadLimit ? 3u : 0u); } + [[nodiscard]] std::uint32_t ShaderDataDwords() const { + return ImageMetadataDword() + runtimeImageCount * (sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)); + } [[nodiscard]] bool UsesPushData() const { return pushDataStartDword != PushData::NoStart; } diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp index ee85759d2..a8216cc16 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp @@ -44,6 +44,7 @@ struct MemoryInfo { bool imageHasMip = false; bool imageR128 = false; bool imagePacked = false; + std::uint32_t imageByElements = 0; bool idxen = false; bool offen = false; bool planningOnly = false; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp index a59dbab67..953dacc16 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp @@ -41,6 +41,7 @@ struct ResourceSnapshot { std::vector flattenedSrt; std::vector userData; UniformFill uniformFill; + std::vector srtPoison; }; struct UniformFillPlan { @@ -63,6 +64,8 @@ struct IrResourcePlan { std::vector controlFlow; std::vector materializationSources; std::vector srtReads; + std::vector guardedSrtSlots; + std::uint32_t srtGuardOffset = 0; std::vector cleanFlatSlots; // One byte per srtReads slot, 1 when the CPU walk never consumes the slot's value (see // Detail::ComputePureFlatSlots): a driver may reuse a capture whose words differ only there. diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp index a25930f11..ec2d0c79f 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp @@ -17,16 +17,15 @@ struct BufferResource { std::uint32_t source = 0; std::uint32_t firstUsePc = 0; std::uint32_t maxByteExtent = 0; - std::uint32_t packedStride = 0; - IrBufferFormat descriptorFormat = IrBufferFormat::Invalid; - std::uint32_t descriptorSwizzle = 0x00000facu; std::uint32_t imageAlias = NoImageAlias; bool read = false; bool written = false; bool atomic = false; bool formatted = false; + bool descriptorFormatted = false; + std::uint32_t formattedReadMask = 0; bool scalar = false; - bool empty = false; + std::uint8_t typedAlignment = 1; bool operator==(const BufferResource& other) const = default; }; @@ -34,6 +33,9 @@ struct BufferResource { enum class ImageMipMode { None, DynamicStorage }; namespace EmulatedCompare { +inline constexpr std::uint32_t NativeOffsetUnsupported = 1u << 29u; +inline constexpr std::uint32_t Unsupported = 1u << 31u; +inline constexpr std::uint32_t RequiresSingleLevel = 1u << 30u; inline constexpr std::uint32_t Enabled = 1u << 0u; inline constexpr std::uint32_t FunctionShift = 1u; inline constexpr std::uint32_t Linear = 1u << 4u; @@ -74,9 +76,15 @@ struct ImageResource { bool cube = false; bool r128 = false; bool srgbDecode = false; + bool srgbDecodeCompatible = true; + std::uint32_t srgbDecodeFormats = 0u; bool depthBits = false; bool depthUnorm16 = false; bool packed = false; + bool fmaskCompatible = true; + bool depthBitsCompatible = true; + std::uint32_t byElements = 0; + std::uint32_t byComponents = 0; IrBufferFormat packedFormat = IrBufferFormat::Invalid; std::uint32_t emulatedCompare = 0; std::uint32_t indirectRoot = NoIndirectImage; @@ -103,8 +111,11 @@ inline bool ImageSampleExplicitLod(std::uint32_t flags, IrShaderStage stage) { } struct SamplerResource { + static constexpr std::uint32_t NoCopy = std::numeric_limits::max(); + std::uint32_t source = 0; std::uint32_t firstUsePc = 0; + std::uint32_t copyOf = NoCopy; bool forcePointFiltering = false; bool depthCompare = false; std::uint8_t uses = 0; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp index aa6ee6b96..ca81c0fec 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp @@ -18,6 +18,7 @@ struct ShaderInfo { std::vector buffers; std::vector images; + std::vector> runtimeImageModes; std::vector samplers; std::vector sampledPairs; std::vector inputs; @@ -32,6 +33,7 @@ struct ShaderInfo { bool hasBitwiseXor = false; bool usesDma = false; bool bdaWrites = false; + bool usesFaultBuffer = false; bool dispatchThreadLimit = false; bool operator==(const ShaderInfo& other) const = default; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp index 484c34519..d223ca696 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp @@ -262,6 +262,8 @@ enum class IrOpcode : std::uint16_t { FPSqrt64, FPTrigPreop64, FPDot2F32F16, + FPInterpolateF32, + FPInterpolateF16, ConvertF32F64, ConvertF64F32, ConvertF64S32, @@ -457,6 +459,7 @@ enum class IrOpcode : std::uint16_t { ImageAtomicCmpSwap64, GetAttribute, GetInterpolationParameter, + GetInterpolationParameterF16, SetAttribute, ControlNop, Waitcnt, diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp index 5919c08e6..a06c8e9f4 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp @@ -303,6 +303,8 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("FPSqrt64", U64, U64), makeMeta("FPTrigPreop64", U64, U64, U32), makeMeta("FPDot2F32F16", U32, U32, U32, U32), + makeMeta("FPInterpolateF32", F32, F32, F32, F32, U32), + makeMeta("FPInterpolateF16", F32, F32, F32, F32, U32), makeMeta("ConvertF32F64", F32, U64), makeMeta("ConvertF64F32", U64, F32), makeMeta("ConvertF64S32", U64, U32), @@ -498,6 +500,7 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("ImageAtomicCmpSwap64", U64, ImageResource, ImageAddress, U64, U64, U1), makeMeta("GetAttribute", U32, U32, U32), makeMeta("GetInterpolationParameter", U32, U32, U32, U32), + makeMeta("GetInterpolationParameterF16", F32, U32, U32, U32, U32), makeMeta("SetAttribute", Void, U32x4, U1), makeMeta("ControlNop", Void), makeMeta("Waitcnt", Void), diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp index 15aed9410..6a9d16ec0 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp @@ -827,7 +827,7 @@ PositionExportComponent DecodePositionExportComponent(std::uint32_t control, std std::uint32_t NativeBinding(IrShaderStage stage, DescriptorBindingKind kind) { const std::uint32_t group = stage == IrShaderStage::Pixel ? 1u : stage == IrShaderStage::TessellationControl ? 2u : stage == IrShaderStage::TessellationEvaluation ? 3u : 0u; - return static_cast(kind) + group * static_cast(DescriptorBindingKind::Count); + return RuntimeAbi::BindingNumber(static_cast(group), kind); } ImageResourceClass ImageBindingResourceClass(DescriptorBindingKind kind) { @@ -850,17 +850,15 @@ DescriptorBindingKind DescriptorBindingForImage(const ImageResource& image) { constexpr std::uint32_t sampledUintBinding = 8u; constexpr std::uint32_t sampledSintBinding = 15u; constexpr std::uint32_t storageFloatBinding = FirstStorageImageBinding; - constexpr std::uint32_t storageUintBinding = storageFloatBinding + 5u; - constexpr std::uint32_t atomicUintBinding = storageUintBinding + 5u; - constexpr std::uint32_t atomic64UintBinding = atomicUintBinding + 5u; + constexpr std::uint32_t storageUintBinding = storageFloatBinding + 7u; + constexpr std::uint32_t atomicUintBinding = storageUintBinding + 7u; + constexpr std::uint32_t atomic64UintBinding = atomicUintBinding + 7u; std::uint32_t base = 0u; - bool sampled = false; if (image.resourceClass == ImageResourceClass::Sampled) { if (image.atomic) { fail("DescriptorBindingForImage sampled image cannot be atomic"); } - sampled = true; switch (image.numericClass) { case IrTextureNumericClass::Float: base = image.depthCompare ? FirstComparisonImageBinding : sampledFloatBinding; @@ -916,19 +914,13 @@ DescriptorBindingKind DescriptorBindingForImage(const ImageResource& image) { dimension = 3u; break; case RdnaImageDimension::Dim2DMsaa: - if (!sampled) { - fail("DescriptorBindingForImage storage image cannot be multisampled"); - } dimension = 4u; break; case RdnaImageDimension::Dim2DMsaaArray: - if (!sampled) { - fail("DescriptorBindingForImage storage image cannot be multisampled"); - } dimension = 5u; break; case RdnaImageDimension::Dim3D: - dimension = sampled ? 6u : 4u; + dimension = 6u; break; case RdnaImageDimension::Unknown: default: diff --git a/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp b/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp index 407dfd6bd..120de5358 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp @@ -9,11 +9,15 @@ namespace ShaderRecompiler { -struct BindingAllocationResult { - std::vector bindings; +struct CompiledBindingLayout { IrBindingLayout layout; std::uint32_t pushConstantOffsetBytes = 0; std::uint32_t pushConstantSizeBytes = 0; +}; + +struct BindingAllocationResult : CompiledBindingLayout { + std::vector specialization; + std::vector bindings; std::vector pushConstants; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp b/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp index cae8055d8..01ece57e4 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp @@ -9,11 +9,35 @@ namespace ShaderRecompiler { std::uint32_t PointFilteredSamplerWord(std::uint32_t word0, std::uint32_t filter); +struct PreparedDescriptorBinding { + DescriptorBinding descriptor; + std::vector resources; + std::vector identityImageSwizzle; + std::vector samplerFilterElements; + std::size_t writtenBuffers = 0; + std::size_t readOnlyBuffers = 0; +}; + +struct PreparedImageMetadata { + std::uint32_t resource; + std::uint32_t offset; + RuntimeAbi::ResourceMetadata metadata; +}; + +struct DescriptorBindingPlan { + std::vector bindings; + std::vector imageMetadata; + std::vector specialization; + bool storageData = false; +}; class DescriptorBindingBuilder { public: - void Populate(BindingAllocationResult& allocation, const IrProgram& program, const ResourceSnapshot& snapshot, const std::array& partialThreads) const; - void Populate(BindingAllocationResult& allocation, const ShaderInfo& info, IrShaderStage stage, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads) const; + DescriptorBindingPlan Prepare(const IrBindingLayout& layout, const ShaderInfo& info, IrShaderStage stage, const ResourceSnapshot& snapshot, std::span exportMappings = {}) const; + DescriptorBindingPlan Select(const DescriptorBindingPlan& plan, std::span bindings) const; + void Populate(BindingAllocationResult& allocation, const CompiledBindingLayout& compiled, const DescriptorBindingPlan& plan, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads) const; + void Populate(BindingAllocationResult& allocation, const IrProgram& program, const ResourceSnapshot& snapshot, const std::array& partialThreads, std::span exportMappings = {}) const; + void Populate(BindingAllocationResult& allocation, const ShaderInfo& info, IrShaderStage stage, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads, std::span exportMappings = {}) const; }; } diff --git a/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp b/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp index 5abfd7503..321474d50 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp @@ -16,6 +16,7 @@ public: private: static bool ReadGuestMemory(void* userContext, std::uint64_t address, std::uint32_t* value); + static bool Covered(void* userContext, std::uint64_t address, std::uint64_t bytes); std::vector regions; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp index aed4b1d83..5bd14939e 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp @@ -8,56 +8,20 @@ namespace ShaderRecompiler { -struct ResourceSpecialization { - struct Buffer { - std::uint32_t packedStride = 0; - IrBufferFormat descriptorFormat = IrBufferFormat::Invalid; - std::uint32_t descriptorSwizzle = ShaderImageIdentitySwizzle; - bool empty = false; - - bool operator==(const Buffer& other) const; - }; - - struct Image { - IrTextureNumericClass numericClass = IrTextureNumericClass::Unsupported; - RdnaImageDimension dimension = RdnaImageDimension::Unknown; - std::uint32_t mipCount = 1; - IrBufferFormat conversionFormat = IrBufferFormat::Invalid; - std::uint32_t shaderSwizzle = ShaderImageIdentitySwizzle; - std::uint32_t indirectRoot = ImageResource::NoIndirectImage; - std::uint32_t indirectMappingOffset = 0; - std::uint32_t indirectSearchIterations = 0; - bool cube = false; - bool fmask = false; - bool depthBits = false; - bool depthUnorm16 = false; - IrBufferFormat packedFormat = IrBufferFormat::Invalid; - std::uint32_t emulatedCompare = 0; - bool srgbDecode = false; - - bool operator==(const Image& other) const; - }; - - std::vector buffers; - std::vector images; - - bool operator==(const ResourceSpecialization& other) const; - - std::vector boundDescriptors; -}; - // Why a bindless image table (a T# loaded from a table buffer at a runtime key) was not bound; // counted on the [bindless] line (APS5_PROFILE_DRAW). enum class BindlessRejection { Capacity, MaterialScan, NoEntry, Storage, NonUniform, ImageSlots, Count }; class ResourceMaterializer { public: - void Apply(IrProgram& program, const ResourceSpecialization& specialization) const; + static std::uint32_t EmulatedCompareState(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint32_t index); + void ApplyStaticInterface(IrProgram& program, bool nativeSampleOffsets = true) const; + static std::vector RuntimeImageModes(const ImageResource& image); + static std::uint32_t RuntimeImageMode(const ImageResource& image, const DescriptorValue& descriptor, std::span modes); + static void PrepareImageModes(ShaderInfo& info); [[nodiscard]] IrResourcePlan ExtractPlan(const IrProgram& program) const; - void Materialize(const IrResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot, ResourceSpecialization& specialization) const; - // APS5_PROFILE_DRAW: the time Materialize spent building specializations, over every call. + void Materialize(const IrResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot) const; static std::uint64_t SpecializationNanoseconds(); - // The slots every bindless image table binds (APS5_BINDLESS_SLOTS, default 16, 1..48). static std::uint32_t BindlessSlots(); static void CountBindlessRejection(BindlessRejection reason); }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp index 7e56f67e4..a0c470a7b 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp @@ -3,7 +3,7 @@ #include "Recompiler.hpp" #include "IntermediateRepresentation/IrProgram.hpp" -#include "Optimization/ResourceMaterializer.hpp" +#include "Optimization/SrtWalker.hpp" #include #include @@ -12,13 +12,10 @@ namespace ShaderRecompiler { [[nodiscard]] IrProgram PrepareResourceProgram(const RecompileRequest& request); [[nodiscard]] std::shared_ptr GetResourcePlan(const RecompileRequest& request); -// What a driver's capture of a request produces: the plan and the materialization of the words the -// runtime read through it. Recompile(request, capture) compiles from these without a second walk. struct SourceEntry; struct ResourceCapture { std::shared_ptr plan; ResourceSnapshot snapshot; - ResourceSpecialization specialization; // The cache entry the plan belongs to; null when the request bypasses the cache. std::shared_ptr source; // The walk's read addresses when the plan has pure flat slots (the leaf of each pure slot, @@ -37,9 +34,32 @@ struct ResourceCapture { // request with the same code and the same cache key fields (RecompileCacheKey::ContextHash plus // the target); the vertex stages' input validation is repeated per capture because it reads V# // fields the key does not cover. Null for a request that bypasses the cache (useCache false). +struct CompiledVariant; struct SourceHandle { std::shared_ptr source; + std::shared_ptr artifact; + std::vector staticKey; }; +class PreparedShaderInvocation { +public: + static std::optional TryCreate(const RecompileRequest& request, const std::shared_ptr& handle); + static std::optional TryCreate(const RecompileRequest& request, const std::shared_ptr& handle, std::span key); + const RecompileRequest& Request() const { return request; } + std::shared_ptr Capture(const SrtRuntime& runtime) const; + std::shared_ptr Materialize(const ResourceCapture& capture) const; + +private: + PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle); + RecompileRequest request; + std::shared_ptr handle; +}; +[[nodiscard]] std::span GetPreparedCode(const SourceHandle& handle); +[[nodiscard]] std::shared_ptr PrepareShader(const RecompileRequest& request); +[[nodiscard]] bool MatchesPreparedShader(const RecompileRequest& request, const SourceHandle& handle); +void BuildPreparedShaderKey(const RecompileRequest& request, std::vector& key); +[[nodiscard]] bool MatchesPreparedShader(const RecompileRequest& request, const SourceHandle& handle, std::span key); +[[nodiscard]] const CompiledShaderArtifact& GetPreparedArtifact(const SourceHandle& handle); +[[nodiscard]] std::shared_ptr MaterializeShader(const RecompileRequest& request, const ResourceCapture& capture, const SourceHandle& handle); [[nodiscard]] std::shared_ptr ResolveSource(const RecompileRequest& request); [[nodiscard]] std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime, const SourceHandle& handle); diff --git a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp index befe1cce7..996730b67 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp @@ -40,18 +40,6 @@ struct ShaderBufferResource { [[nodiscard]] std::uint8_t Type() const { return (fields[3] >> 30u) & 0x3u; } }; -struct ShaderColorComponentMapping { - static constexpr std::uint8_t Identity = 0xe4u; - std::uint8_t packed = Identity; - - [[nodiscard]] std::uint32_t Map(std::uint32_t component) const { - return (packed >> (component * 2u)) & 0x3u; - } - [[nodiscard]] bool IsIdentity() const { - return packed == Identity; - } -}; - struct ShaderVertexInputBuffer { static constexpr int MaxAttributes = 32; @@ -169,7 +157,6 @@ struct ShaderPixelInputInfo { return vgprs; }(); std::uint8_t targetOutputMode[8] = {}; - std::array targetExportMapping = {}; std::uint32_t scratchSizeDwords = 0; bool psPosX = false; bool psPosY = false; @@ -185,6 +172,7 @@ struct ShaderPixelInputInfo { bool psEarlyZ = false; bool psExecuteOnNoop = false; ConservativeZExport psConservativeZExport = ConservativeZExport::AnyZ; + bool psOrderedPixelShader = false; ShaderStageRuntime stage; bool HasPositionInput() const { @@ -213,6 +201,26 @@ struct ShaderPixelInputInfo { return one ? 0x3f800000u : 0u; } + [[nodiscard]] bool InputIsFp16(std::uint32_t input) const { + return input < inputNum && input < 32u && (interpolatorSettings[input] & 0x80000u) != 0u; + } + + [[nodiscard]] bool InputHalfIsDefault(std::uint32_t input, bool high) const { + if (!high) { + return InputIsDefault(input); + } + return input < inputNum && input < 32u && (interpolatorSettings[input] & 0x100000u) != 0u; + } + + [[nodiscard]] std::uint32_t InputHalfDefaultBits(std::uint32_t input, std::uint32_t component, bool high) const { + if (!high) { + return InputDefaultBits(input, component); + } + const auto value = input < 32u ? (interpolatorSettings[input] >> 21u) & 0x3u : 0u; + const bool one = component == 3u ? (value & 0x1u) != 0u : (value & 0x2u) != 0u; + return one ? 0x3f800000u : 0u; + } + [[nodiscard]] bool InputIsLinear(std::uint32_t input, std::uint32_t linearInputs, std::uint32_t perspectiveInputs) const { const auto bit = input < 32u ? 1u << input : 0u; if ((linearInputs & perspectiveInputs & bit) != 0u) { diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp index f5f5a4070..16cdedac5 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp @@ -11,6 +11,7 @@ namespace ShaderRecompiler { using SrtMemoryReader = bool (*)(void* userData, std::uint64_t address, std::uint32_t* value); +using SrtMemoryProbe = bool (*)(void* userData, std::uint64_t address, std::uint64_t bytes); // The guest addresses a walk dereferenced (Detail::Evaluator::EvaluateRawRead): the leaf read of // each pure flat slot (IrResourcePlan::pureFlatSlots) as (flat offset, address), set by the @@ -30,6 +31,7 @@ struct SrtRuntime { void* userContext = nullptr; SrtMemoryReader readSpecializationMemory = nullptr; SrtReadTrace* readTrace = nullptr; + SrtMemoryProbe accessible = nullptr; }; enum class RuntimeValueType { @@ -44,7 +46,7 @@ public: void EvaluateUniformValues(const IrResourcePlan& program, std::span values, const SrtRuntime& runtime, std::span results) const; void EvaluateDescriptorSource(const IrResourcePlan& program, std::uint32_t source, const SrtRuntime& runtime, DescriptorValue& result) const; void EvaluateDescriptorSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results) const; - void EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources) const; + void EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison = nullptr) const; void Walk(const IrResourcePlan& program, const SrtRuntime& runtime, std::vector& flat) const; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp index c1b008e9b..eb9c568ad 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp @@ -15,7 +15,7 @@ namespace ShaderRecompiler::Detail { // Why the last EvaluateRuntimeSourcesImpl call on this thread returned false. const std::string& RuntimeSourceFailureReason(); -bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources); +bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison = nullptr); } diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp index 7877c5928..269bfceb4 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp @@ -66,12 +66,18 @@ private: std::size_t _count = 0; }; +struct InaccessibleRead { + const IrValue* read = nullptr; + std::uint64_t address = 0; +}; + class Evaluator { public: Evaluator(const IrResourcePlan& program, const SrtRuntime& runtime, std::span cleanFlatSlots = {}, Evaluator* cleanEvaluator = nullptr, IrValue* activeMask = nullptr) : _program(program), _runtime(runtime), _cleanFlatSlots(cleanFlatSlots), _cleanEvaluator(cleanEvaluator), _activeMask(activeMask != nullptr ? activeMask->Resolve() : nullptr) {} bool Evaluate(IrValue* value, std::uint32_t& result); bool EvaluateWide(IrValue* raw, std::uint64_t& result); + void ReportInaccessibleReads(InaccessibleRead* sink) { _inaccessible = sink; } private: static float Float32(std::uint64_t bits); @@ -88,6 +94,7 @@ private: std::span _cleanFlatSlots; Evaluator* _cleanEvaluator = nullptr; IrValue* _activeMask = nullptr; + InaccessibleRead* _inaccessible = nullptr; EvaluatedValues _cache; std::vector _visiting; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp index f1f806f08..85bd0e7ab 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp @@ -16,6 +16,7 @@ namespace ShaderRecompiler::Detail { // fill) has no pure slot. Reachability over-approximates what the evaluators consume, so an // unknown opcode only makes more slots impure. std::vector ComputePureFlatSlots(const IrResourcePlan& plan); +std::vector ComputeGuardedFlatSlots(const IrResourcePlan& plan); } diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp index e4ab1ce06..00c598e3b 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp @@ -6,6 +6,7 @@ namespace ShaderRecompiler::Detail { bool IsRawRead(const IrResourcePlan& program, const IrValue& inst); +bool LoadedFromMemory(const IrValue& handle); bool IsDescriptorHandle(IrOpcode opcode); bool IsRuntimeSelect(IrOpcode opcode); bool IsRuntimeUniformOp(IrOpcode opcode); diff --git a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp index ed0b6987d..518ac228e 100644 --- a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp +++ b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp @@ -1,4 +1,5 @@ #include "Optimization/BindingAllocator.hpp" +#include "Optimization/ResourceMaterializer.hpp" #include #include #include @@ -73,7 +74,7 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin fail(metadata.shaderInfoComplete ? "shader binding layout failed: binding layout already allocated" : "shader binding layout failed: shader info is not ready"); } - if (layout.descriptorSet != 0u) { + if (layout.descriptorSet != RuntimeAbi::DescriptorSet) { fail("shader binding layout failed: descriptor set must be 0"); } if (layout.firstBinding != 0u) { @@ -96,10 +97,11 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin next.memoryOffsetDword = static_cast(next.userDataRegisters.size()); next.memoryOffsetCount = static_cast(info.buffers.size()); next.dispatchThreadLimit = info.dispatchThreadLimit; - const std::uint32_t pushDataStartDword = layout.pushConstantOffsetBytes / 4u; - const std::uint32_t pushConstantSizeDwords = layout.pushConstantSizeBytes / 4u; - const bool usesPushData = next.ShaderDataDwords() != 0u && next.ShaderDataDwords() <= pushConstantSizeDwords; - next.pushDataStartDword = usesPushData ? pushDataStartDword : PushData::NoStart; + if (info.buffers.size() > RuntimeAbi::BufferCapacity || info.images.size() > RuntimeAbi::ImageCapacity || info.samplers.size() > RuntimeAbi::SamplerHeapCapacity) fail("shader binding layout exceeds runtime metadata capacity"); + if (std::ranges::any_of(next.userDataRegisters, [](auto reg) { return reg >= RuntimeAbi::UserDataCapacity; })) fail("shader user data exceeds runtime ABI capacity"); + if (std::ranges::any_of(info.images, [](const auto& image) { return image.indirectRoot != ImageResource::NoIndirectImage; })) next.runtimeImageCount = static_cast(info.images.size()); + const auto pushSize = layout.pushConstantSizeBytes / 4u; + next.pushDataStartDword = next.runtimeImageCount == 0u && next.ShaderDataDwords() != 0u && next.ShaderDataDwords() <= pushSize ? layout.pushConstantOffsetBytes / 4u : PushData::NoStart; if (!info.buffers.empty()) { std::vector resources(info.buffers.size()); @@ -111,19 +113,15 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin std::array, ImageBindingCount> imageGroups; const auto place = [&](std::uint32_t i) { - const DescriptorBindingKind kind = DescriptorBindingForImage(info.images[i]); - const std::uint32_t group = ImageBindingIndex(kind); - if (group >= imageGroups.size()) { - fail("shader binding layout failed: image " + std::to_string(i) + " has an unmapped binding class"); - } - std::vector& resources = imageGroups[group]; const bool dynamic = info.images[i].mipMode == ImageMipMode::DynamicStorage; - const std::uint32_t count = dynamic ? info.images[i].mipCount : 1u; - if (count == 0u || (!dynamic && info.images[i].mipCount != 1u)) { - fail("shader binding layout failed: image " + std::to_string(i) + " has an invalid specialized mip count " + - std::to_string(info.images[i].mipCount)); + const std::uint32_t count = dynamic ? RuntimeAbi::StorageHeapCapacity : 1u; + std::array placed{}; + for (const auto& mode : ResourceMaterializer::RuntimeImageModes(info.images[i])) { + const auto group = ImageBindingIndex(DescriptorBindingForImage(mode)); + if (placed.at(group)) continue; + placed[group] = true; + imageGroups[group].insert(imageGroups[group].end(), count, i); } - resources.insert(resources.end(), count, i); }; // A bindless table's slots follow their root as consecutive elements: the SPIR-V indexes the // binding with element(root) + slot. @@ -145,14 +143,16 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin } for (std::uint32_t i = 0; i < imageGroups.size(); i++) { if (!imageGroups[i].empty()) { + if (imageGroups[i].size() > RuntimeAbi::HeapCapacity(static_cast(FirstImageBinding + i))) fail("shader image heap capacity exceeded"); addBinding(next, static_cast(FirstImageBinding + i), std::move(imageGroups[i])); } } if (!info.samplers.empty()) { - std::vector resources(info.samplers.size()); + if (info.samplers.size() > RuntimeAbi::SamplerHeapCapacity / 2u) fail("shader sampler pairs exceed runtime heap capacity"); + std::vector resources(info.samplers.size() * 2u); for (std::uint32_t i = 0; i < resources.size(); i++) { - resources[i] = i; + resources[i] = i / 2u; } addBinding(next, DescriptorBindingKind::Samplers, std::move(resources)); } @@ -161,6 +161,8 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin } if (info.usesDma) { addBinding(next, DescriptorBindingKind::BdaPagetable); + } + if (info.usesDma || info.usesFaultBuffer) { addBinding(next, DescriptorBindingKind::FaultBuffer); } diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index 6cd17b6b3..0005d4c46 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -1,11 +1,14 @@ #include "Optimization/DescriptorBindingBuilder.hpp" +#include "Optimization/ResourceMaterializer.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" +#include "SpirvBackend/SpirvBufferFormat.hpp" #include #include #include #include #include #include +#include #include #include #include @@ -95,21 +98,6 @@ DescriptorImageShape ImageShapeForResource(const ImageResource& image) { fail("DescriptorBindingBuilder::Populate image resource dimension has no descriptor image shape"); } -DescriptorImageShape ImageShapeFor(const std::vector& images, const std::vector& resources) { - if (resources.empty()) { - fail("DescriptorBindingBuilder::Populate guest image binding has no resources"); - } - std::optional shape; - for (const std::uint32_t r : resources) { - const DescriptorImageShape current = ImageShapeForResource(images.at(r)); - if (shape.has_value() && *shape != current) { - fail("DescriptorBindingBuilder::Populate guest image array elements disagree on image shape"); - } - shape = current; - } - return *shape; -} - std::vector GuestBuffersDescriptor(const std::vector& resources, const ResourceSnapshot& snapshot) { std::vector result; result.reserve(resources.size() * 4u); @@ -128,30 +116,6 @@ std::vector GuestBuffersDescriptor(const std::vector GuestImagesDescriptor(const std::vector& resources, const ResourceSnapshot& snapshot) { - std::vector result; - std::uint32_t dwordCount = 0; - for (std::size_t i = 0; i < resources.size(); i++) { - const std::uint32_t r = resources[i]; - if (r >= snapshot.images.size()) { - fail("DescriptorBindingBuilder::Populate guest image index is out of range"); - } - const DescriptorValue& value = snapshot.images[r]; - if (value.dwordCount == 0u) { - fail("DescriptorBindingBuilder::Populate guest image descriptor is empty"); - } - if (i == 0u) { - dwordCount = value.dwordCount; - } else if (value.dwordCount != dwordCount) { - fail("DescriptorBindingBuilder::Populate guest image descriptors have inconsistent widths"); - } - for (std::uint32_t dword = 0; dword < value.dwordCount; dword++) { - result.push_back(value.dwords[dword]); - } - } - return result; -} - std::vector GuestSamplersDescriptor(const std::vector& resources, const ResourceSnapshot& snapshot) { std::vector result; std::uint32_t dwordCount = 0; @@ -202,7 +166,8 @@ struct UnnormalizedProof { UnnormalizedProof ProveUnnormalized(const ShaderInfo& info, const ResourceSnapshot& snapshot) { UnnormalizedProof proof{std::vector(info.samplers.size()), std::vector(info.images.size())}; for (std::uint32_t r = 0; r < info.samplers.size(); r++) { - if ((GuestSamplersDescriptor({r}, snapshot)[0] & ForceUnnormalizedBit) == 0u) { + if (snapshot.samplers.at(r).dwordCount != 4u) fail("sampler descriptor must contain four dwords"); + if ((snapshot.samplers[r].dwords[0] & ForceUnnormalizedBit) == 0u) { continue; } const auto& sampler = info.samplers[r]; @@ -217,7 +182,8 @@ UnnormalizedProof ProveUnnormalized(const ShaderInfo& info, const ResourceSnapsh if (pair.sampler != r) { continue; } - const auto& image = info.images.at(pair.image); + const auto& base = info.images.at(pair.image); + const auto& image = info.runtimeImageModes.at(pair.image).at(ResourceMaterializer::RuntimeImageMode(base, snapshot.images.at(pair.image), info.runtimeImageModes.at(pair.image))); if (image.indirectRoot != ImageResource::NoIndirectImage) { failUnnormalized("samples an image selected at run time"); } @@ -237,49 +203,24 @@ UnnormalizedProof ProveUnnormalized(const ShaderInfo& info, const ResourceSnapsh return proof; } -std::vector SamplerElements(const IrBindingLayout& layout, const ShaderInfo& info) { - std::vector elements(info.samplers.size(), ShaderInfo::MaxSamplers); - for (const IrDescriptorBinding& logical : layout.descriptors) { - if (logical.kind != DescriptorBindingKind::Samplers) { - continue; - } - for (std::uint32_t element = 0; element < logical.resources.size() && element < ShaderInfo::MaxSamplers; element++) { - elements.at(logical.resources[element]) = element; - } - } - return elements; -} - -std::uint32_t ImageSamplerMask(const ShaderInfo& info, const std::vector& samplerElements, std::uint32_t resource) { - const std::uint32_t root = info.images.at(resource).indirectRoot; - std::uint32_t mask = 0; - for (const SampledResourcePair& pair : info.sampledPairs) { - if (pair.image != resource && pair.image != root) { - continue; - } - if (pair.sampler >= samplerElements.size() || samplerElements[pair.sampler] >= ShaderInfo::MaxSamplers) { - fail("DescriptorBindingBuilder::Populate sampled image pair names a sampler outside the first " + std::to_string(ShaderInfo::MaxSamplers) + " elements of the sampler binding"); - } - mask |= 1u << samplerElements[pair.sampler]; - } - return mask; -} - -std::vector ShaderDataDwordsFor(const IrBindingLayout& layout, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads) { +std::vector ShaderDataDwordsFor(const IrBindingLayout& layout, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads, const std::vector& imageMetadata) { std::vector result(layout.ShaderDataDwords(), 0u); - for (std::size_t i = 0; i < layout.userDataRegisters.size(); i++) { - const std::uint32_t reg = layout.userDataRegisters[i]; - if (reg < userDataBase || reg - userDataBase >= snapshot.userData.size()) { - fail("DescriptorBindingBuilder::Populate user-data register is out of range"); - } + for (std::size_t i = 0; i < layout.userDataRegisters.size(); ++i) { + const auto reg = layout.userDataRegisters[i]; + if (reg < userDataBase || reg - userDataBase >= snapshot.userData.size()) fail("shader user-data register is out of range"); result[i] = snapshot.userData[reg - userDataBase]; } if (layout.dispatchThreadLimit) { - if (partialThreads == std::array{}) { - fail("DescriptorBindingBuilder::Populate partial-group shader has no dispatch size"); - } + if (partialThreads == std::array{}) fail("partial-group shader has no dispatch size"); std::copy(partialThreads.begin(), partialThreads.end(), result.begin() + layout.DispatchThreadLimitDword()); } + for (const auto& entry : imageMetadata) { + auto metadata = entry.metadata; + const auto& descriptor = snapshot.images.at(entry.resource); + if (descriptor.dwordCount != metadata.descriptor.size()) fail("invalid runtime image descriptor width"); + std::copy_n(descriptor.dwords.begin(), metadata.descriptor.size(), metadata.descriptor.begin()); + std::memcpy(result.data() + entry.offset, &metadata, sizeof(metadata)); + } return result; } @@ -294,66 +235,256 @@ std::uint32_t PointFilteredSamplerWord(std::uint32_t word0, std::uint32_t filter return (filter & ~(0xffu << 20u)) | (1u << 24u) | (mipmapped ? 1u << 26u : 0u); } -void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, const IrProgram& program, const ResourceSnapshot& snapshot, const std::array& partialThreads) const { - Populate(allocation, program.Info(), program.Resources().stage, program.Resources().userDataBase, snapshot, partialThreads); +void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, const IrProgram& program, const ResourceSnapshot& snapshot, const std::array& partialThreads, std::span exportMappings) const { + Populate(allocation, program.Info(), program.Resources().stage, program.Resources().userDataBase, snapshot, partialThreads, exportMappings); } -void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, const ShaderInfo& info, IrShaderStage stage, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads) const { - const IrBindingLayout& layout = allocation.layout; - const std::vector shaderData = ShaderDataDwordsFor(layout, userDataBase, snapshot, partialThreads); - const UnnormalizedProof unnormalized = ProveUnnormalized(info, snapshot); - const std::vector samplerElements = SamplerElements(layout, info); +void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, const ShaderInfo& info, IrShaderStage stage, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads, std::span exportMappings) const { + auto plan = Prepare(allocation.layout, info, stage, snapshot, exportMappings); + Populate(allocation, allocation, plan, userDataBase, snapshot, partialThreads); + allocation.specialization = std::move(plan.specialization); +} - std::vector bindings; - bindings.reserve(layout.descriptors.size()); - std::size_t writtenHere = 0; - std::size_t readOnlyHere = 0; - for (const IrDescriptorBinding& logical : layout.descriptors) { - DescriptorBinding physical; - physical.descriptorSet = 0u; - physical.binding = NativeBinding(stage, logical.kind); - physical.count = logical.resources.empty() ? 1u : static_cast(logical.resources.size()); - physical.kind = PhysicalKindFor(logical.kind); - physical.role = RoleFor(logical.kind); - physical.readOnly = false; - - switch (physical.role) { - case DescriptorRole::GuestBuffers: - physical.guestDescriptor = GuestBuffersDescriptor(logical.resources, snapshot); - for (const std::uint32_t resource : logical.resources) { +DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& layout, const ShaderInfo& info, IrShaderStage stage, const ResourceSnapshot& snapshot, std::span exportMappings) const { + if (stage == IrShaderStage::Pixel && exportMappings.size() != 8u) fail("fragment runtime export mappings are missing"); + if (info.images.size() > ShaderInfo::MaxImages) fail("runtime image count exceeds the static capacity"); + DescriptorBindingPlan plan; + const auto unnormalized = ProveUnnormalized(info, snapshot); + std::vector compareStates(info.images.size()); + for (std::uint32_t index = 0; index < info.images.size(); ++index) { + if (!info.images[index].depthCompare) continue; + const auto compare = compareStates[index] = ResourceMaterializer::EmulatedCompareState(info, snapshot, index); + const auto first = PipelineSpecialization::CompareBase + index * PipelineSpecialization::CompareWords; + const std::array values{EmulatedCompare::Function(compare), (compare & EmulatedCompare::Linear) != 0u, EmulatedCompare::AddressX(compare), EmulatedCompare::AddressY(compare), EmulatedCompare::Reference(compare), (compare & EmulatedCompare::BorderWhite) != 0u}; + for (std::uint32_t word = 0; word < values.size(); ++word) plan.specialization.push_back({first + word, values[word]}); + } + std::vector imageModes(info.images.size()); + for (std::size_t index = 0; index < info.images.size(); ++index) imageModes[index] = ResourceMaterializer::RuntimeImageMode(info.images[index], snapshot.images.at(index), info.runtimeImageModes.at(index)); + for (std::uint32_t index = 0; index < info.buffers.size(); ++index) { + const auto& descriptor = snapshot.buffers.at(index); + if (descriptor.dwordCount != 4u) fail("buffer specialization requires four descriptor words"); + const auto& buffer = info.buffers[index]; + const auto format = (descriptor.dwords[3] >> 12u) & 0x7fu; + const bool present = descriptor.dwords[2] != 0u && (descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u); + if (buffer.descriptorFormatted && (format > static_cast(IrBufferFormat::Format32_32_32_32Float) || (present && format == 0u))) fail("buffer specialization has an unsupported format"); + const auto misalignment = present ? descriptor.dwords[0] & 3u : 0u; + if (buffer.atomic && misalignment != 0u) fail("buffer atomic on a V# whose base is not DWORD aligned"); + auto alignment = static_cast(buffer.typedAlignment); + if (buffer.descriptorFormatted && format != 0u) alignment = std::max(alignment, std::min(GetFormatInfo(static_cast(format)).byteSize, 4u)); + if (alignment == 0u || misalignment % alignment != 0u) fail("is a formatted buffer access through a V# whose base is not aligned to its element"); + const auto first = PipelineSpecialization::BufferBase + index * PipelineSpecialization::BufferWords; + plan.specialization.push_back({first, descriptor.dwords[1] & 0xffff0000u}); + plan.specialization.push_back({first + 1u, descriptor.dwords[3]}); + plan.specialization.push_back({first + 2u, present ? 1u : 0u}); + plan.specialization.push_back({first + 3u, misalignment}); + if (buffer.formattedReadMask != 0u) { + for (std::uint32_t component = 0; component < 4u; ++component) { + if ((buffer.formattedReadMask & (1u << component)) == 0u) continue; + const auto selector = (descriptor.dwords[3] >> (component * 3u)) & 7u; + if (selector == 2u || selector == 3u) fail("buffer specialization has a reserved component selector"); + } + } + } + for (std::uint32_t index = 0; index < info.images.size(); ++index) { + if (info.images[index].indirectRoot != ImageResource::NoIndirectImage) continue; + const auto first = PipelineSpecialization::ImageBase + index * PipelineSpecialization::ImageWords; + plan.specialization.push_back({first, imageModes[index]}); + for (std::uint32_t component = 0; component < 4u; ++component) plan.specialization.push_back({first + 1u + component, (snapshot.images.at(index).dwords[3] >> (component * 3u)) & 7u}); + } + for (std::uint32_t target = 0; target < exportMappings.size(); ++target) { + for (std::uint32_t component = 0; component < 4u; ++component) plan.specialization.push_back({PipelineSpecialization::ExportBase + target * 4u + component, (exportMappings[target] >> (component * 2u)) & 3u}); + } + std::vector samplerModes(info.samplers.size(), 0u); + const auto samplerMode = [](const ImageResource& image) { return image.numericClass == IrTextureNumericClass::Sint || image.conversionFormat != IrBufferFormat::Invalid || image.depthBits ? 2u : 1u; }; + for (const auto& pair : info.sampledPairs) { + const auto& image = info.images.at(pair.image); + if (image.indirectRoot == ImageResource::NoIndirectImage) samplerModes.at(pair.sampler) |= samplerMode(info.runtimeImageModes.at(pair.image).at(imageModes[pair.image])); + else { + const auto& root = info.images.at(image.indirectRoot); + for (const auto slot : root.indirectResources) samplerModes.at(pair.sampler) |= samplerMode(info.runtimeImageModes.at(slot).at(imageModes[slot])); + } + } + for (std::uint32_t index = 0; index < info.images.size(); ++index) { + if (info.images[index].mipMode != ImageMipMode::DynamicStorage) continue; + const auto word = snapshot.images.at(index).dwords[3]; + const auto first = (word >> 12u) & 0xfu; + const auto last = (word >> 16u) & 0xfu; + if (last < first || last - first >= RuntimeAbi::StorageHeapCapacity) fail("invalid dynamic storage mip range"); + plan.specialization.push_back({PipelineSpecialization::MipCountBase + index, last - first + 1u}); + } + for (std::uint32_t resource = 0; resource < info.images.size(); ++resource) { + const auto& image = info.images[resource]; + if (image.indirectRoot == ImageResource::NoIndirectImage) continue; + const auto& modes = info.runtimeImageModes.at(resource); + if (modes.size() > PipelineSpecialization::ImageModeStride) fail("runtime image mode specialization capacity exceeded"); + std::vector active(modes.size(), false); + for (const auto slot : info.images.at(image.indirectRoot).indirectResources) { + const auto mode = ResourceMaterializer::RuntimeImageMode(image, snapshot.images.at(slot), modes); + if (mode != imageModes[slot]) fail("indirect image mode numbering disagrees with its root"); + active[mode] = true; + } + for (std::uint32_t mode = 0; mode < modes.size(); ++mode) plan.specialization.push_back({PipelineSpecialization::ImageModeBase + resource * PipelineSpecialization::ImageModeStride + mode, active[mode] ? 1u : 0u}); + } + plan.bindings.reserve(layout.descriptors.size()); + for (const auto& logical : layout.descriptors) { + const bool imageHeap = ImageBindingResourceClass(logical.kind) != ImageResourceClass::None; + const bool samplerHeap = logical.kind == DescriptorBindingKind::Samplers; + IrDescriptorBinding compact{logical.kind, {}}; + std::vector originals; + std::uint32_t previous = UINT32_MAX; + std::uint32_t mip = 0; + for (std::uint32_t element = 0; element < logical.resources.size(); ++element) { + const auto resource = logical.resources[element]; + bool active = true; + if (imageHeap) { + const auto& image = info.images.at(resource); + const auto& mode = info.runtimeImageModes.at(resource).at(imageModes[resource]); + mip = previous == resource ? mip + 1u : 0u; + previous = resource; + { + const auto word = snapshot.images.at(resource).dwords[3]; + active = DescriptorBindingForImage(mode) == logical.kind && mode.packedFormat != IrBufferFormat::Fmask8_S2_F1 && (image.mipMode != ImageMipMode::DynamicStorage || mip <= ((word >> 16u) & 0xfu) - ((word >> 12u) & 0xfu)); + } + } else if (samplerHeap) { + active = (samplerModes.at(resource) & (1u << (element & 1u))) != 0u; + } + if (imageHeap || samplerHeap) plan.specialization.push_back({PipelineSpecialization::DescriptorIndex(static_cast(logical.kind), element), active ? static_cast(compact.resources.size()) : 0u}); + if (active) { + compact.resources.push_back(resource); + originals.push_back(element); + } + } + if (imageHeap || samplerHeap) plan.specialization.push_back({PipelineSpecialization::HeapCountBase + static_cast(logical.kind), std::max(1u, static_cast(compact.resources.size()))}); + if ((imageHeap || samplerHeap) && compact.resources.empty()) continue; + PreparedDescriptorBinding entry{}; + auto& physical = entry.descriptor; + physical.descriptorSet = RuntimeAbi::DescriptorSet; + physical.binding = NativeBinding(stage, compact.kind); + physical.count = compact.resources.empty() ? 1u : static_cast(compact.resources.size()); + physical.kind = PhysicalKindFor(compact.kind); + physical.role = RoleFor(compact.kind); + if (physical.role == DescriptorRole::GuestBuffers) { + for (const auto resource : compact.resources) { const auto& buffer = info.buffers.at(resource); physical.bufferAtomic.push_back(buffer.atomic); - // The tracker merges every buffer access of a source into its resource - // (ResourceTracker::Merge), so an element without a store or atomic is read-only - // over its whole extent; stores through pointers (BDA) never bind a V#. physical.bufferWritten.push_back(buffer.written || buffer.atomic); - if (buffer.written || buffer.atomic) ++writtenHere; - else ++readOnlyHere; + if (buffer.written || buffer.atomic) ++entry.writtenBuffers; + else ++entry.readOnlyBuffers; } + } else if (physical.role == DescriptorRole::GuestImages) { + const auto& modes = info.runtimeImageModes.at(compact.resources.front()); + const auto shape = std::ranges::find_if(modes, [&](const ImageResource& mode) { return DescriptorBindingForImage(mode) == compact.kind; }); + if (shape == modes.end()) fail("runtime image heap has no static image type"); + physical.imageShape = ImageShapeForResource(*shape); + for (std::size_t element = 0; element < compact.resources.size(); ++element) { + const auto resource = compact.resources[element]; + const auto& image = info.images.at(resource); + const auto& mode = info.runtimeImageModes.at(resource).at(imageModes[resource]); + entry.identityImageSwizzle.push_back(mode.conversionFormat != IrBufferFormat::Invalid || mode.depthBits); + physical.imageWritten.push_back(image.written || image.atomic); + physical.imageDepthCompare.push_back(mode.depthCompare); + physical.imageAtomic.push_back(image.atomic); + physical.imageUnnormalized.push_back(unnormalized.images.at(resource)); + physical.imageSamplers.push_back(0u); + physical.imageAtomic64.push_back(image.atomic64); + if (layout.runtimeImageCount == 0u || image.indirectRoot == ImageResource::NoIndirectImage) continue; + if (resource >= layout.runtimeImageCount) fail("runtime image metadata exceeds compact layout"); + const auto& descriptor = snapshot.images.at(resource); + RuntimeAbi::ResourceMetadata metadata{}; + if (descriptor.dwordCount != metadata.descriptor.size()) fail("invalid runtime image descriptor width"); + metadata.binding = static_cast(compact.kind); + metadata.firstElement = static_cast(element); + metadata.elementCount = 1u; + metadata.flags = imageModes[resource] << 1u; + if (descriptor.dwords[0] == 0u && (descriptor.dwords[1] & 0xffu) == 0u) metadata.flags |= 1u; + const auto offset = layout.ImageMetadataDword() + resource * static_cast(sizeof(metadata) / sizeof(std::uint32_t)); + plan.imageMetadata.push_back({resource, offset, metadata}); + } + } else if (physical.role == DescriptorRole::GuestSamplers) { + for (std::uint32_t element = 0; element < compact.resources.size(); ++element) { + const auto resource = compact.resources[element]; + bool compare = false; + for (const auto& pair : info.sampledPairs) if (pair.sampler == resource) compare |= info.images.at(pair.image).depthCompare && compareStates.at(pair.image) == 0u; + physical.samplerDepthCompare.push_back(compare); + physical.samplerUnnormalized.push_back(unnormalized.samplers.at(resource)); + if ((originals[element] & 1u) != 0u) entry.samplerFilterElements.push_back(element); + } + } else if (physical.role == DescriptorRole::ShaderData && layout.UsesPushData()) { + fail("DescriptorBindingBuilder::Populate shader-data binding must not exist when push data is used"); + } + plan.storageData |= physical.role == DescriptorRole::ShaderData; + entry.resources = std::move(compact.resources); + plan.bindings.push_back(std::move(entry)); + } + std::vector samplerMasks(info.samplers.size()); + for (const auto& binding : plan.bindings) { + if (binding.descriptor.role != DescriptorRole::GuestSamplers) continue; + if (binding.resources.size() > ShaderInfo::MaxSamplers) fail("prepared sampler mask capacity exceeded"); + for (std::uint32_t element = 0; element < binding.resources.size(); ++element) samplerMasks.at(binding.resources[element]) |= 1u << element; + } + for (auto& binding : plan.bindings) { + if (binding.descriptor.role != DescriptorRole::GuestImages) continue; + for (std::size_t element = 0; element < binding.resources.size(); ++element) { + const auto resource = binding.resources[element]; + for (const auto& pair : info.sampledPairs) { + if (pair.image == resource || pair.image == info.images.at(resource).indirectRoot) binding.descriptor.imageSamplers[element] |= samplerMasks.at(pair.sampler); + } + } + } + return plan; +} + +DescriptorBindingPlan DescriptorBindingBuilder::Select(const DescriptorBindingPlan& plan, std::span bindings) const { + DescriptorBindingPlan selected; + std::set remaining(bindings.begin(), bindings.end()); + selected.bindings.reserve(remaining.size()); + for (const auto& binding : plan.bindings) { + if (remaining.erase(binding.descriptor.binding) == 0u) continue; + selected.storageData |= binding.descriptor.role == DescriptorRole::ShaderData; + selected.bindings.push_back(binding); + } + if (!remaining.empty()) fail("specialized module bindings disagree with the prepared plan"); + const bool hasSamplers = std::any_of(selected.bindings.begin(), selected.bindings.end(), [](const auto& binding) { return binding.descriptor.role == DescriptorRole::GuestSamplers; }); + if (!hasSamplers) { + for (auto& binding : selected.bindings) { + std::fill(binding.descriptor.imageSamplers.begin(), binding.descriptor.imageSamplers.end(), 0u); + } + } + selected.imageMetadata = plan.imageMetadata; + return selected; +} + +void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, const CompiledBindingLayout& compiled, const DescriptorBindingPlan& plan, std::uint32_t userDataBase, const ResourceSnapshot& snapshot, const std::array& partialThreads) const { + const auto& layout = compiled.layout; + const bool needsShaderData = layout.UsesPushData() || plan.storageData; + auto shaderData = needsShaderData ? ShaderDataDwordsFor(layout, userDataBase, snapshot, partialThreads, plan.imageMetadata) : std::vector{}; + std::vector bindings; + bindings.reserve(plan.bindings.size()); + std::size_t writtenHere = 0; + std::size_t readOnlyHere = 0; + for (const auto& entry : plan.bindings) { + auto physical = entry.descriptor; + switch (physical.role) { + case DescriptorRole::GuestBuffers: + physical.guestDescriptor = GuestBuffersDescriptor(entry.resources, snapshot); + writtenHere += entry.writtenBuffers; + readOnlyHere += entry.readOnlyBuffers; break; case DescriptorRole::GuestImages: - physical.guestDescriptor = GuestImagesDescriptor(logical.resources, snapshot); - physical.imageShape = ImageShapeFor(info.images, logical.resources); - for (const std::uint32_t resource : logical.resources) { - const auto& image = info.images.at(resource); - physical.imageWritten.push_back(image.written || image.atomic); - physical.imageDepthCompare.push_back(image.depthCompare); - physical.imageAtomic.push_back(image.atomic); - physical.imageAtomic64.push_back(image.atomic64); - physical.imageUnnormalized.push_back(unnormalized.images.at(resource)); - physical.imageSamplers.push_back(ImageSamplerMask(info, samplerElements, resource)); + physical.guestDescriptor.reserve(entry.resources.size() * 8u); + for (std::size_t element = 0; element < entry.resources.size(); ++element) { + const auto& descriptor = snapshot.images.at(entry.resources[element]); + if (descriptor.dwordCount != 8u) fail("invalid runtime image descriptor width"); + const auto offset = physical.guestDescriptor.size(); + physical.guestDescriptor.insert(physical.guestDescriptor.end(), descriptor.dwords.begin(), descriptor.dwords.begin() + 8); + if (entry.identityImageSwizzle[element]) physical.guestDescriptor[offset + 3u] = (physical.guestDescriptor[offset + 3u] & ~0xfffu) | ShaderImageIdentitySwizzle; } break; case DescriptorRole::GuestSamplers: - physical.guestDescriptor = GuestSamplersDescriptor(logical.resources, snapshot); - for (std::size_t element = 0; element < logical.resources.size(); ++element) { - const auto& sampler = info.samplers.at(logical.resources[element]); - physical.samplerDepthCompare.push_back(sampler.depthCompare); - physical.samplerUnnormalized.push_back(unnormalized.samplers.at(logical.resources[element])); - if (sampler.forcePointFiltering) { - auto& filter = physical.guestDescriptor.at(element * 4u + 2u); - filter = PointFilteredSamplerWord(physical.guestDescriptor.at(element * 4u), filter); - } + physical.guestDescriptor = GuestSamplersDescriptor(entry.resources, snapshot); + for (const auto element : entry.samplerFilterElements) { + auto& filter = physical.guestDescriptor.at(element * 4u + 2u); + filter = PointFilteredSamplerWord(physical.guestDescriptor.at(element * 4u), filter); } break; case DescriptorRole::FlattenedSrt: @@ -363,10 +494,7 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con physical.guestDescriptor = snapshot.flattenedSrt; break; case DescriptorRole::ShaderData: - if (layout.UsesPushData()) { - fail("DescriptorBindingBuilder::Populate shader-data binding must not exist when push data is used"); - } - physical.guestDescriptor = shaderData; + physical.guestDescriptor = std::move(shaderData); break; case DescriptorRole::Gds: case DescriptorRole::BdaPagetable: diff --git a/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp b/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp index 249778f06..58981be3a 100644 --- a/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp +++ b/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp @@ -64,6 +64,29 @@ bool RequestMemoryView::ReadGuestMemory(void* userContext, std::uint64_t address return true; } +bool RequestMemoryView::Covered(void* userContext, std::uint64_t address, std::uint64_t bytes) { + const auto* self = static_cast(userContext); + if (bytes > std::numeric_limits::max() - address) { + return false; + } + const std::uint64_t end = address + bytes; + while (address < end) { + const auto it = std::upper_bound(self->regions.begin(), self->regions.end(), address, [](std::uint64_t addressValue, const MemoryRegion& region) { + return addressValue < region.guestAddress; + }); + if (it == self->regions.begin()) { + return false; + } + const MemoryRegion& region = *std::prev(it); + const std::uint64_t regionEnd = region.guestAddress + region.bytes.size(); + if (address >= regionEnd) { + return false; + } + address = regionEnd; + } + return true; +} + SrtRuntime RequestMemoryView::MakeRuntime(std::span userData, std::uint64_t shaderBase) { SrtRuntime runtime; runtime.userData = userData; @@ -71,6 +94,7 @@ SrtRuntime RequestMemoryView::MakeRuntime(std::span userDat runtime.userContext = this; runtime.readMemory = &ReadGuestMemory; runtime.readSpecializationMemory = &ReadGuestMemory; + runtime.accessible = &Covered; return runtime; } diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 08c9b1595..fe2061160 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -1,8 +1,9 @@ #include "Optimization/ResourceMaterializer.hpp" #include "Optimization/SrtWalker/SrtFlatSlotClasses.hpp" -#include "IntermediateRepresentation/IrBuilder.hpp" #include "Optimization/ShaderStageInputInfo.hpp" #include "RdnaDecoder/RdnaDescriptorFormat.hpp" +#include "RdnaDecoder/RdnaImageOpDecoder.hpp" +#include "SpirvBackend/SpirvBufferFormat.hpp" #include #include #include @@ -26,8 +27,6 @@ namespace ShaderRecompiler { namespace { -constexpr std::uint32_t NoRemap = std::numeric_limits::max(); - std::atomic specializationNanoseconds{0}; bool MaterializeProfiled() { @@ -160,6 +159,7 @@ DecodedImage decodeImageDescriptor(const DescriptorValue& descriptor, const Imag } const bool storage = base.resourceClass == ImageResourceClass::Storage; decoded.fmask = IsFmaskTextureFormat(format); + if (decoded.fmask && !base.fmaskCompatible) throw std::runtime_error("FMASK requires a direct 32-bit image load"); if (decoded.fmask && (storage || base.depthCompare || base.indirectRoot != ImageResource::NoIndirectImage)) { throw std::runtime_error("FMASK requires a direct sampled image load"); } @@ -169,8 +169,32 @@ DecodedImage decodeImageDescriptor(const DescriptorValue& descriptor, const Imag } decoded.packedFormat = format; } + if (base.byElements != 0u) { + const bool eightBit = format == IrBufferFormat::Format8UNorm || format == IrBufferFormat::Format8SNorm || format == IrBufferFormat::Format8UInt || format == IrBufferFormat::Format8SInt; + const bool sixteenBit = format == IrBufferFormat::Format16UNorm || format == IrBufferFormat::Format16SNorm || format == IrBufferFormat::Format16UInt || format == IrBufferFormat::Format16SInt || format == IrBufferFormat::Format16Float; + const bool eightBitPair = format == IrBufferFormat::Format8_8UNorm || format == IrBufferFormat::Format8_8SNorm || format == IrBufferFormat::Format8_8UInt || format == IrBufferFormat::Format8_8SInt; + const bool packedEight = format == IrBufferFormat::Format8UNorm || format == IrBufferFormat::Format8UInt || format == IrBufferFormat::Format8SInt; + const bool packedSixteen = format == IrBufferFormat::Format16UNorm || format == IrBufferFormat::Format16UInt || format == IrBufferFormat::Format16SInt || format == IrBufferFormat::Format8_8UNorm || format == IrBufferFormat::Format8_8UInt || format == IrBufferFormat::Format8_8SInt; + const bool measured = base.packed ? packedEight || (base.byElements == 2u && packedSixteen) + : base.byElements == 4u ? base.byComponents == 1u && eightBit : base.byElements == 2u && (base.byComponents == 1u ? eightBit || sixteenBit : base.byComponents == 2u && eightBitPair); + if (!measured || descriptorImageSwizzle(descriptor) != ShaderImageIdentitySwizzle || rawImageType(descriptor) != ImageType::Color2D || base.indirectRoot != ImageResource::NoIndirectImage) { + throw std::runtime_error(base.packed ? "MIMG PCK2/PCK4 requires a direct, identity-swizzled 2D UNORM, UINT or SINT image whose elements fill one dword: R8, R16 or RG8 for PCK2, R8 for PCK4" + : "MIMG BY2/BY4 requires a direct, identity-swizzled 2D image whose elements fill one dword: R8, R16 or RG8 for BY2, R8 for BY4"); + } + } decoded.conversionFormat = RemapTextureFormat(format) != format ? format : IrBufferFormat::Invalid; + if (format == IrBufferFormat::Format11_11_10UNorm || format == IrBufferFormat::Format10_11_11Float) { + const bool floating = format == IrBufferFormat::Format10_11_11Float; + if (!base.srgbDecodeCompatible) throw std::runtime_error(floating ? "samples or gathers a converted float image, or queries its LOD, which is not implemented" : "sampling, gathering or querying LOD of a converted unorm image is not implemented"); + if (!base.depthBitsCompatible) throw std::runtime_error(floating ? "reads or writes a converted float image with 16-bit data, which is not implemented" : "reads or writes a converted unorm image with 16-bit data, which is not implemented"); + for (std::uint32_t component = 0; component < 4u; ++component) { + if (((descriptorImageSwizzle(descriptor) >> (component * 3u)) & 7u) == 7u) throw std::runtime_error("selects a channel the converted image format does not have"); + } + } decoded.srgbDecode = !storage && (srgbDecodeFormats & SrgbDecodeBit(format)) != 0u; + if (decoded.srgbDecode && !base.srgbDecodeCompatible) { + throw std::runtime_error("samples or gathers an sRGB image the device cannot sample, which is not implemented"); + } if (storage || decoded.conversionFormat != IrBufferFormat::Invalid) { decoded.shaderSwizzle = descriptorImageSwizzle(descriptor); } @@ -180,6 +204,7 @@ DecodedImage decodeImageDescriptor(const DescriptorValue& descriptor, const Imag decoded.numericClass = base.atomic ? IrTextureNumericClass::Uint : SampledTextureNumericClass(format); if (!storage && !base.depthCompare && IsDepthBitsTexture(descriptor.dwords[1], descriptor.dwords[3])) { decoded.depthBits = true; + if (!base.depthBitsCompatible) throw std::runtime_error("runtime image reads depth bits as unsupported 16-bit results"); decoded.depthUnorm16 = DepthBitsTextureWidth(descriptor.dwords[1], descriptor.dwords[3]) == 16u; decoded.numericClass = IrTextureNumericClass::Float; decoded.shaderSwizzle = descriptorImageSwizzle(descriptor); @@ -192,6 +217,7 @@ DecodedImage decodeImageDescriptor(const DescriptorValue& descriptor, const Imag decoded.numericClass = IrTextureNumericClass::Uint; } if ((rawSintStorage || (base.atomic && format == IrBufferFormat::Format32SInt)) && !base.packed) { + if (!base.depthBitsCompatible) throw std::runtime_error("stores 16-bit data to an image of a SINT format"); decoded.conversionFormat = format; } } else if (decoded.numericClass == IrTextureNumericClass::Unsupported || (base.depthCompare && decoded.numericClass != IrTextureNumericClass::Float)) { @@ -200,14 +226,8 @@ DecodedImage decodeImageDescriptor(const DescriptorValue& descriptor, const Imag return decoded; } -bool requiresPointSampler(const ResourceSpecialization::Image& image) { - return image.numericClass == IrTextureNumericClass::Sint || image.conversionFormat != IrBufferFormat::Invalid || image.depthBits; -} - constexpr std::uint32_t TableEntryBytes = 32; -// Bindless image tables: the most material records mode M reads to enumerate a table's keys -// (APS5_BINDLESS_MATERIAL_SCAN, default 256); beyond it the whole table is bound instead. std::uint32_t MaterialScanLimit() { static const std::uint32_t limit = [] { const char* text = std::getenv("APS5_BINDLESS_MATERIAL_SCAN"); @@ -270,10 +290,6 @@ void reportBindless() { static_cast(rejected[0]), static_cast(rejected[1]), static_cast(rejected[2]), static_cast(rejected[3]), static_cast(rejected[4]), static_cast(rejected[5])); } -// A bindless image table's bound slots and its (key, slot) mapping, keys ascending. Slots the -// mapping does not name (past the keys, or an entry that is null, invalid or of another shape) -// hold a copy of the first usable entry: a key the mapping lacks samples zeros in the shader, as -// a null T# does on hardware. Empty when the image is no table root. struct TableResolution { std::vector slots; std::vector> mapping; @@ -308,14 +324,16 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con if (image.resourceClass != ImageResourceClass::Sampled) { rejectTable(BindlessRejection::Storage, "bindless storage image tables are unsupported"); } - if (image.packed) { - throw std::runtime_error("bindless packed image tables are unsupported"); + if (image.packed || image.byElements != 0u) { + throw std::runtime_error("bindless packed and BY2/BY4 image tables are unsupported"); } const auto slots = ResourceMaterializer::BindlessSlots(); DescriptorValue heapValue; walker.EvaluateDescriptorSource(plan, table.heapSource, runtime, heapValue); const ShaderBufferResource heap = decodeBufferDescriptor(heapValue); const std::uint64_t heapSize = heap.GetSize(); + if (heap.Type() != 0u || heap.Base48() > std::numeric_limits::max() - heapSize) throw std::runtime_error("bindless heap buffer is invalid"); + if (heapSize != 0u && (heapSize < table.entryOffset || (heapSize - table.entryOffset) % TableEntryBytes != 0u)) throw std::runtime_error("bindless heap contains a partial descriptor"); const auto entries = heapSize > table.entryOffset ? static_cast(std::min((heapSize - table.entryOffset) / TableEntryBytes, std::numeric_limits::max())) : 0u; const auto readWord = [&](std::uint64_t address, std::uint32_t& word) { if (!runtime.readMemory(runtime.userContext, address, &word)) { @@ -329,18 +347,21 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con std::uint32_t materialEntries = 0; std::uint64_t materialBase = 0; std::uint32_t outOfRange = 0; - if (table.hasMaterial && table.selectorStride != 0u) { + if (table.hasMaterial) { + if (table.selectorStride < sizeof(std::uint32_t) || table.selectorOffset > table.selectorStride - sizeof(std::uint32_t)) throw std::runtime_error("bindless material selector is outside its record"); DescriptorValue materialValue; walker.EvaluateDescriptorSource(plan, table.materialSource, runtime, materialValue); const ShaderBufferResource material = decodeBufferDescriptor(materialValue); materialBase = material.Base48(); const std::uint64_t materialSize = material.GetSize(); + if (material.Type() != 0u || materialSize % table.selectorStride != 0u) throw std::runtime_error("bindless material buffer has an invalid layout"); materialEntries = static_cast(std::min(materialSize / table.selectorStride, std::numeric_limits::max())); + if (materialSize / table.selectorStride > MaterialScanLimit()) rejectTable(BindlessRejection::MaterialScan, "bindless material scan capacity exceeded"); if (materialEntries <= MaterialScanLimit()) { materialMode = true; for (std::uint32_t record = 0; record < materialEntries; record++) { const std::uint64_t offset = static_cast(record) * table.selectorStride + table.selectorOffset; - if (offset + sizeof(std::uint32_t) > materialSize) break; + if (offset + sizeof(std::uint32_t) > materialSize) throw std::runtime_error("bindless material selector exceeds its buffer"); std::uint32_t key = 0; readWord(materialBase + offset, key); if (key >= entries) { @@ -364,69 +385,38 @@ void resolveTableImage(const IrResourcePlan& plan, std::uint32_t imageIndex, con for (std::uint32_t key = 0; key < entries; key++) keys[key] = key; } counters.outOfRange.fetch_add(outOfRange, std::memory_order_relaxed); - if (keys.empty()) { - rejectTable(BindlessRejection::NoEntry, "bindless image table selects no entry"); - } - const std::uint64_t heapBase = heap.Base48(); std::vector candidates(keys.size()); std::vector valid(keys.size(), 0u); - std::optional shape; - std::uint32_t paddedNull = 0; - std::uint32_t paddedShape = 0; - std::uint32_t paddedConversion = 0; - for (std::size_t i = 0; i < keys.size(); i++) { + std::uint32_t paddedNull = 0u; + for (std::size_t i = 0u; i < keys.size(); ++i) { auto& candidate = candidates[i]; candidate.dwordCount = 8u; const std::uint64_t address = heapBase + table.entryOffset + static_cast(keys[i]) * TableEntryBytes; - for (std::uint32_t dword = 0; dword < 8u; dword++) { - readWord(address + dword * sizeof(std::uint32_t), candidate.dwords[dword]); - } - DecodedImage decoded; - bool usable = !nullImageDescriptor(candidate) && validImageDescriptor(candidate, image.r128); - if (usable) { - try { - decoded = decodeImageDescriptor(candidate, image, plan.srgbDecodeFormats); - } catch (const std::exception&) { - usable = false; - } - } - if (!usable) { - paddedNull++; + for (std::uint32_t dword = 0u; dword < 8u; ++dword) readWord(address + dword * sizeof(std::uint32_t), candidate.dwords[dword]); + if (nullImageDescriptor(candidate)) { + ++paddedNull; continue; } - if (decoded.conversionFormat != IrBufferFormat::Invalid || decoded.fmask || decoded.srgbDecode) { - paddedConversion++; - continue; - } - if (shape.has_value() && (decoded.numericClass != shape->numericClass || decoded.dimension != shape->dimension || decoded.cube != shape->cube)) { - paddedShape++; - continue; - } - if (!shape.has_value()) shape = decoded; + if (!validImageDescriptor(candidate, image.r128)) throw std::runtime_error("bindless image table contains an invalid descriptor"); + const auto decoded = decodeImageDescriptor(candidate, image, plan.srgbDecodeFormats); + if (decoded.fmask) throw std::runtime_error("bindless FMASK images are unsupported"); + static_cast(ResourceMaterializer::RuntimeImageMode(image, candidate, plan.info.runtimeImageModes.at(imageIndex))); valid[i] = 1u; } - if (!shape.has_value()) { - rejectTable(BindlessRejection::NoEntry, "bindless image table has no valid entry"); - } - const auto pad = candidates[static_cast(std::find(valid.begin(), valid.end(), std::uint8_t{1}) - valid.begin())]; resolution.mapping.clear(); - for (std::size_t i = 0; i < keys.size(); i++) { - if (valid[i] == 0u) { - candidates[i] = pad; - continue; - } - resolution.mapping.emplace_back(keys[i], static_cast(i)); + for (std::size_t i = 0u; i < keys.size(); ++i) { + if (valid[i] != 0u) resolution.mapping.emplace_back(keys[i], static_cast(i)); } + DescriptorValue nullDescriptor; + nullDescriptor.dwordCount = 8u; resolution.slots = std::move(candidates); - resolution.slots.resize(slots, pad); + resolution.slots.resize(slots, nullDescriptor); resolved = resolution.slots[0]; (materialMode ? counters.tablesMaterial : counters.tablesWhole).fetch_add(1, std::memory_order_relaxed); counters.keys.fetch_add(resolution.mapping.size(), std::memory_order_relaxed); counters.paddedNull.fetch_add(paddedNull, std::memory_order_relaxed); - counters.paddedShape.fetch_add(paddedShape, std::memory_order_relaxed); - counters.paddedConversion.fetch_add(paddedConversion, std::memory_order_relaxed); if (BindlessTraced()) traceTable(plan, image, table, heapBase, materialBase, {materialMode, entries, materialEntries, static_cast(resolution.mapping.size())}); } @@ -454,7 +444,7 @@ void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime, std::vector values; std::vector activeSources; - walker.EvaluateRuntimeSources(plan, plan.materializationSources, runtime, values, snapshot.flattenedSrt, plan.cleanFlatSlots, activeSources); + walker.EvaluateRuntimeSources(plan, plan.materializationSources, runtime, values, snapshot.flattenedSrt, plan.cleanFlatSlots, activeSources, &snapshot.srtPoison); std::size_t cursor = 0; if (values.size() < plan.info.buffers.size()) { @@ -465,18 +455,17 @@ void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime, snapshot.images.resize(plan.info.images.size()); tables.assign(plan.info.images.size(), {}); - std::uint32_t activeTables = 0; + std::uint32_t tableCount = 0; for (std::uint32_t i = 0; i < plan.info.images.size(); i++) { const auto& image = plan.info.images[i]; if (image.source >= plan.descriptorSources.size()) { throw std::runtime_error("image resource references an unknown descriptor source"); } - const bool active = image.source >= activeSources.size() || activeSources[image.source] != 0u; - if (plan.descriptorSources[image.source].indirectImage.has_value() && active) activeTables++; + if (plan.descriptorSources[image.source].indirectImage.has_value()) tableCount++; } const auto tableSlots = ResourceMaterializer::BindlessSlots(); - if (activeTables != 0u && plan.info.images.size() + static_cast(tableSlots - 1u) * activeTables > ShaderInfo::MaxImages) { - rejectTable(BindlessRejection::ImageSlots, "bindless image tables need " + std::to_string(plan.info.images.size() + static_cast(tableSlots - 1u) * activeTables) + " image slots, limit " + std::to_string(ShaderInfo::MaxImages)); + if (tableCount != 0u && plan.info.images.size() + static_cast(tableSlots - 1u) * tableCount > ShaderInfo::MaxImages) { + rejectTable(BindlessRejection::ImageSlots, "bindless image tables need " + std::to_string(plan.info.images.size() + static_cast(tableSlots - 1u) * tableCount) + " image slots, limit " + std::to_string(ShaderInfo::MaxImages)); } for (std::uint32_t i = 0; i < plan.info.images.size(); i++) { const auto& image = plan.info.images[i]; @@ -484,6 +473,7 @@ void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime, if (source.indirectImage.has_value()) { if (image.source < activeSources.size() && activeSources[image.source] == 0u) { snapshot.images[i].dwordCount = 8u; + tables[i].slots.assign(tableSlots, snapshot.images[i]); continue; } resolveTableImage(plan, i, *source.indirectImage, runtime, walker, snapshot.images[i], tables[i]); @@ -498,7 +488,7 @@ void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime, throw std::runtime_error("image descriptor has an invalid width"); } if (!validImageDescriptor(descriptor, image.r128) && !nullImageDescriptor(descriptor)) { - descriptor.dwords.fill(0u); + throw std::runtime_error("runtime image descriptor is invalid"); } snapshot.images[i] = descriptor; } @@ -527,18 +517,20 @@ std::uint32_t colorCompareReference(IrBufferFormat format) { } } -std::uint32_t emulatedCompareState(const IrResourcePlan& plan, const ResourceSnapshot& snapshot, std::uint32_t index) { - const auto& image = plan.info.images[index]; +std::uint32_t emulatedCompareState(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint32_t index) { + const auto& image = info.images[index]; const auto& descriptor = snapshot.images[index]; if (!image.depthCompare || descriptor.dwordCount != 8u || nullImageDescriptor(descriptor)) return 0u; const auto format = rawImageFormat(descriptor); if (format == IrBufferFormat::Format32Float || format == IrBufferFormat::Format16UNorm || IsDepthBitsTexture(descriptor.dwords[1], descriptor.dwords[3])) return 0u; + if (image.indirectRoot != ImageResource::NoIndirectImage || (image.emulatedCompare & EmulatedCompare::Unsupported) != 0u) throw std::runtime_error("unsupported color comparison image instructions"); + if ((image.emulatedCompare & EmulatedCompare::RequiresSingleLevel) != 0u && ((descriptor.dwords[3] >> 12u) & 0xfu) != ((descriptor.dwords[3] >> 16u) & 0xfu)) throw std::runtime_error("color comparison requires a single mip level"); const auto reference = colorCompareReference(format); const auto type = rawImageType(descriptor); if (type != ImageType::Color2D && type != ImageType::Color2DArray) throw std::runtime_error("comparison sampling of a color texture is implemented only for 2D and 2D array views"); if ((descriptorImageSwizzle(descriptor) & 0x7u) != 4u) throw std::runtime_error("comparison sampling of a color texture is implemented only when the view's X channel is red"); std::optional samplerState; - for (const auto& pair : plan.info.sampledPairs) { + for (const auto& pair : info.sampledPairs) { if (pair.image != index) continue; if (pair.sampler >= snapshot.samplers.size() || snapshot.samplers[pair.sampler].dwordCount != 4u) throw std::runtime_error("comparison sampling of a color texture has no sampler descriptor"); const auto& words = snapshot.samplers[pair.sampler].dwords; @@ -572,317 +564,245 @@ std::uint32_t emulatedCompareState(const IrResourcePlan& plan, const ResourceSna return *samplerState | (reference << EmulatedCompare::ReferenceShift) | (singleLevel ? EmulatedCompare::SingleLevel : 0u); } -void buildResourceSpecialization(const IrResourcePlan& plan, ResourceSnapshot& snapshot, const std::vector& tables, ResourceSpecialization& specialization) { - ResourceSpecialization result; - result.buffers.reserve(plan.info.buffers.size()); - for (std::uint32_t i = 0; i < plan.info.buffers.size(); i++) { - const ShaderBufferResource decoded = decodeBufferDescriptor(snapshot.buffers[i]); - if (decoded.Type() != 0u) { - throw std::runtime_error("buffer descriptor uses an unsupported type"); - } - auto packedStride = decoded.PackedStride(); - const auto stride = packedStride & 0x3fffu; - const bool swizzleActive = stride != 0u && ((packedStride >> 14u) & 1u) != 0u; - if (stride == 0u) { - packedStride &= ~((1u << 14u) | (3u << 16u)); - } else if (!swizzleActive) { - packedStride &= ~(3u << 16u); - } - const auto& buffer = plan.info.buffers[i]; - ResourceSpecialization::Buffer entry; - entry.packedStride = packedStride; - entry.descriptorFormat = buffer.formatted ? decoded.Format() : IrBufferFormat::Invalid; - entry.descriptorSwizzle = buffer.formatted ? decoded.DstSelXYZW() : DstSel(4, 5, 6, 7); - entry.empty = decoded.GetSize() == 0u || decoded.Base48() == 0u; - result.buffers.push_back(entry); +void materializeTables(const IrResourcePlan& plan, ResourceSnapshot& snapshot, const std::vector& tables) { + for (std::uint32_t i = 0u; i < plan.info.buffers.size(); ++i) { + const auto decoded = decodeBufferDescriptor(snapshot.buffers.at(i)); + if (decoded.Type() != 0u) throw std::runtime_error("buffer descriptor uses an unsupported type"); + if (plan.stage != IrShaderStage::Compute && decoded.AddTid()) throw std::runtime_error("buffer ADD_TID is only valid for compute shaders"); } - - result.images.reserve(plan.info.images.size()); - for (std::uint32_t i = 0; i < plan.info.images.size(); i++) { + if (snapshot.flattenedSrt.size() != plan.srtReads.size()) throw std::runtime_error("runtime SRT size differs from the static interface"); + for (std::uint32_t i = 0u; i < plan.info.images.size(); ++i) { const auto& image = plan.info.images[i]; - const DecodedImage decoded = decodeImageDescriptor(snapshot.images[i], image, plan.srgbDecodeFormats); - if (decoded.fmask && std::any_of(plan.info.sampledPairs.begin(), plan.info.sampledPairs.end(), [i](const SampledResourcePair& pair) { return pair.image == i; })) { - throw std::runtime_error("FMASK requires a direct image load"); - } - ResourceSpecialization::Image entry; - entry.numericClass = decoded.numericClass; - entry.dimension = decoded.dimension; - entry.mipCount = decoded.mipCount; - entry.conversionFormat = decoded.conversionFormat; - entry.shaderSwizzle = decoded.shaderSwizzle; - entry.indirectRoot = ImageResource::NoIndirectImage; - entry.indirectMappingOffset = 0u; - entry.indirectSearchIterations = 0u; - entry.cube = decoded.cube; - entry.fmask = decoded.fmask; - entry.depthBits = decoded.depthBits; - entry.depthUnorm16 = decoded.depthUnorm16; - entry.packedFormat = decoded.packedFormat; - entry.emulatedCompare = emulatedCompareState(plan, snapshot, i); - entry.srgbDecode = decoded.srgbDecode; - result.images.push_back(entry); - } - - // A table root is followed by its slots 1..C-1 as extra images of the root's shape; the - // (key, slot) mapping the SPIR-V selector searches is appended to the flattened SRT in a - // block of fixed size, so the offsets (part of the specialization) never depend on the keys. - for (std::uint32_t i = 0; i < plan.info.images.size(); i++) { - const auto& table = tables[i]; - if (table.slots.empty()) { + static_cast(ResourceMaterializer::RuntimeImageMode(image, snapshot.images.at(i), plan.info.runtimeImageModes.at(i))); + const auto& table = tables.at(i); + if (!plan.descriptorSources.at(image.source).indirectImage.has_value()) { + if (!table.slots.empty()) throw std::runtime_error("direct image has runtime table slots"); continue; } - const auto mappingOffset = static_cast(snapshot.flattenedSrt.size()); + const auto slots = ResourceMaterializer::BindlessSlots(); + if (table.slots.size() != slots) throw std::runtime_error("runtime table size differs from the static interface"); + const auto mappingOffset = snapshot.flattenedSrt.size(); snapshot.flattenedSrt.push_back(static_cast(table.mapping.size())); for (const auto& [key, slot] : table.mapping) { + if (slot >= slots) throw std::runtime_error("runtime table slot exceeds its static capacity"); snapshot.flattenedSrt.push_back(key); - snapshot.flattenedSrt.push_back(slot); - } - snapshot.flattenedSrt.resize(mappingOffset + 1u + 2u * table.slots.size(), 0u); - auto root = result.images[i]; - root.indirectRoot = i; - root.indirectMappingOffset = mappingOffset; - root.indirectSearchIterations = static_cast(std::bit_width(table.slots.size())); - result.images[i] = root; - root.indirectMappingOffset = 0u; - root.indirectSearchIterations = 0u; - for (std::uint32_t slot = 1; slot < table.slots.size(); slot++) { - result.images.push_back(root); - snapshot.images.push_back(table.slots[slot]); + snapshot.flattenedSrt.push_back(slot == 0u ? i : static_cast(snapshot.images.size()) + slot - 1u); } + snapshot.flattenedSrt.resize(mappingOffset + 1u + 2u * slots, 0u); + snapshot.images.insert(snapshot.images.end(), table.slots.begin() + 1u, table.slots.end()); } +} - result.boundDescriptors.clear(); - result.boundDescriptors.reserve(result.buffers.size() + result.images.size()); - for (std::uint32_t index = 0; index < result.buffers.size(); index++) { - result.boundDescriptors.push_back(index); +void materializeSrtGuards(const IrResourcePlan& plan, ResourceSnapshot& snapshot) { + const auto& guarded = plan.guardedSrtSlots; + const auto flags = snapshot.flattenedSrt.size(); + snapshot.flattenedSrt.resize(flags + guarded.size(), 0u); + for (std::uint32_t record = 0u; record < snapshot.srtPoison.size(); ++record) { + const auto& poison = snapshot.srtPoison[record]; + const auto found = std::lower_bound(guarded.begin(), guarded.end(), poison.slot); + if (found == guarded.end() || *found != poison.slot) throw std::runtime_error("SRT read at flat offset " + std::to_string(poison.slot) + " reads inaccessible memory but has no guard"); + snapshot.flattenedSrt[flags + static_cast(found - guarded.begin())] = record + 1u; + snapshot.flattenedSrt.insert(snapshot.flattenedSrt.end(), {poison.pc, static_cast(poison.address), static_cast(poison.address >> 32u)}); } - for (std::uint32_t index = 0; index < result.images.size(); index++) { - result.boundDescriptors.push_back(index); - } - specialization = std::move(result); } } -void ResourceMaterializer::Apply(IrProgram& program, const ResourceSpecialization& specialization) const { - IrResourcePlan& resources = program.Resources(); - if (!resources.resourceTrackingComplete) { - throw std::runtime_error("ResourceMaterializer::Apply requires a completed resource plan"); +std::vector ResourceMaterializer::RuntimeImageModes(const ImageResource& image) { + std::vector modes; + const bool storage = image.resourceClass == ImageResourceClass::Storage; + const auto append = [&](IrTextureNumericClass numeric, IrBufferFormat conversion, IrBufferFormat packed, bool depth, bool unorm16) { + auto mode = image; + mode.numericClass = numeric; + mode.conversionFormat = conversion; + mode.packedFormat = packed; + mode.depthBits = depth; + mode.depthUnorm16 = unorm16; + mode.cube = false; + mode.mipCount = mode.mipMode == ImageMipMode::DynamicStorage ? RuntimeAbi::StorageHeapCapacity : 1u; + mode.shaderSwizzle = ShaderImageIdentitySwizzle; + if (conversion == IrBufferFormat::Format11_11_10UNorm || conversion == IrBufferFormat::Format10_11_11Float) mode.shaderSwizzle = 0x2acu; + modes.push_back(mode); + if (image.dimension == RdnaImageDimension::Dim2D && image.fmaskCompatible && !depth && packed == IrBufferFormat::Invalid && image.byElements == 0u) { + auto volume = mode; + volume.dimension = RdnaImageDimension::Dim3D; + modes.push_back(volume); + } + if (image.dimension == RdnaImageDimension::Dim1DArray || image.dimension == RdnaImageDimension::Dim2DArray || image.dimension == RdnaImageDimension::Dim2DMsaaArray) { + auto plain = mode; + plain.dimension = image.dimension == RdnaImageDimension::Dim1DArray ? RdnaImageDimension::Dim1D : image.dimension == RdnaImageDimension::Dim2DArray ? RdnaImageDimension::Dim2D : RdnaImageDimension::Dim2DMsaa; + modes.push_back(plain); + } + if (image.dimension == RdnaImageDimension::Dim2DArray) { + mode.cube = true; + modes.push_back(mode); + } + }; + if (image.atomic) { + append(IrTextureNumericClass::Uint, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false); + } else if (image.packed) { + for (std::uint32_t value = 1u; value <= 77u; ++value) { + const auto format = static_cast(value); + const auto info = GetFormatInfo(format); + if (info.packedBitfield || info.componentCount == 0u || (storage && info.byteSize == 12u)) continue; + if (image.byElements != 0u && info.byteSize * 8u * image.byElements > 32u) continue; + const auto numeric = storage && (format == IrBufferFormat::Format32SInt || format == IrBufferFormat::Format32_32SInt || format == IrBufferFormat::Format32_32_32_32SInt) ? IrTextureNumericClass::Uint : SampledTextureNumericClass(format); + bool supported = numeric != IrTextureNumericClass::Unsupported && (!storage || numeric != IrTextureNumericClass::Sint); + for (std::uint32_t component = 0u; component < info.componentCount; ++component) { + const auto bits = info.componentBits[component]; + const bool exactRead = info.type == SpirvFormatComponentType::Uint || info.type == SpirvFormatComponentType::Sint || (info.type == SpirvFormatComponentType::Unorm && bits <= 16u) || (info.type == SpirvFormatComponentType::Float && bits == 32u); + const bool exactWrite = info.type == SpirvFormatComponentType::Uint || (bits == 32u && (info.type == SpirvFormatComponentType::Sint || info.type == SpirvFormatComponentType::Float)); + supported &= storage ? exactWrite : exactRead; + } + if (supported) append(numeric, IrBufferFormat::Invalid, format, false, false); + } + } else { + if ((image.emulatedCompare & EmulatedCompare::NativeOffsetUnsupported) == 0u) append(IrTextureNumericClass::Float, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false); + if (!image.depthCompare) { + append(IrTextureNumericClass::Uint, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false); + if (!storage) append(IrTextureNumericClass::Sint, IrBufferFormat::Invalid, IrBufferFormat::Invalid, false, false); + append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UInt, IrBufferFormat::Invalid, false, false); + if (image.srgbDecodeCompatible && image.depthBitsCompatible) { + append(IrTextureNumericClass::Uint, IrBufferFormat::Format11_11_10UNorm, IrBufferFormat::Invalid, false, false); + append(IrTextureNumericClass::Uint, IrBufferFormat::Format10_11_11Float, IrBufferFormat::Invalid, false, false); + } + if (!storage) { + if (image.depthBitsCompatible) { + append(IrTextureNumericClass::Float, IrBufferFormat::Invalid, IrBufferFormat::Invalid, true, false); + append(IrTextureNumericClass::Float, IrBufferFormat::Invalid, IrBufferFormat::Invalid, true, true); + } + if (image.fmaskCompatible && image.indirectRoot == ImageResource::NoIndirectImage) append(IrTextureNumericClass::Float, IrBufferFormat::Invalid, IrBufferFormat::Fmask8_S2_F1, false, false); + } + } } - if (resources.info.buffers.size() != specialization.buffers.size()) { - throw std::runtime_error("ResourceMaterializer::Apply buffer count mismatch"); + if (storage && image.depthBitsCompatible && !image.packed && !image.atomic64 && ((image.written && !image.read) || image.atomic)) { + constexpr std::array formats{IrBufferFormat::Format32SInt, IrBufferFormat::Format32_32SInt, IrBufferFormat::Format32_32_32_32SInt, IrBufferFormat::Format16SInt, IrBufferFormat::Format8_8SInt, IrBufferFormat::Format16_16SInt, IrBufferFormat::Format8_8_8_8SInt, IrBufferFormat::Format16_16_16_16SInt}; + for (const auto format : formats) { + if (image.atomic && format != IrBufferFormat::Format32SInt) continue; + append(IrTextureNumericClass::Uint, format, IrBufferFormat::Invalid, false, false); + } } - if (resources.info.images.size() > specialization.images.size()) { - throw std::runtime_error("ResourceMaterializer::Apply image count mismatch"); + if (image.depthCompare && image.indirectRoot == ImageResource::NoIndirectImage && (image.emulatedCompare & EmulatedCompare::Unsupported) == 0u && (image.dimension == RdnaImageDimension::Dim2D || image.dimension == RdnaImageDimension::Dim2DArray)) { + auto mode = image; + mode.numericClass = IrTextureNumericClass::Float; + mode.depthCompare = false; + mode.cube = false; + mode.emulatedCompare |= EmulatedCompare::Enabled; + mode.conversionFormat = IrBufferFormat::Invalid; + mode.packedFormat = IrBufferFormat::Invalid; + mode.shaderSwizzle = ShaderImageIdentitySwizzle; + modes.push_back(mode); } + if (image.srgbDecodeFormats != 0u && image.srgbDecodeCompatible && !storage && !image.depthCompare && !image.packed) { + const auto count = modes.size(); + for (std::size_t index = 0; index < count; ++index) { + const auto& base = modes[index]; + if (base.numericClass != IrTextureNumericClass::Float || base.conversionFormat != IrBufferFormat::Invalid || base.packedFormat != IrBufferFormat::Invalid || base.depthBits) continue; + auto mode = base; + mode.srgbDecode = true; + modes.push_back(mode); + } + } + if (modes.empty()) throw std::runtime_error("image instruction has no supported runtime modes"); + return modes; +} - auto buffers = resources.info.buffers; - for (std::uint32_t i = 0; i < buffers.size(); i++) { - buffers[i].packedStride = specialization.buffers[i].packedStride; - buffers[i].descriptorFormat = specialization.buffers[i].descriptorFormat; - buffers[i].descriptorSwizzle = specialization.buffers[i].descriptorSwizzle; - buffers[i].empty = specialization.buffers[i].empty; +std::uint32_t ResourceMaterializer::RuntimeImageMode(const ImageResource& image, const DescriptorValue& descriptor, std::span modes) { + if (descriptor.dwordCount != 8u) throw std::runtime_error("runtime image descriptor must contain eight dwords"); + if (modes.empty()) throw std::runtime_error("prepared runtime image modes are missing"); + if (nullImageDescriptor(descriptor)) return 0u; + const auto decoded = decodeImageDescriptor(descriptor, image, image.srgbDecodeFormats); + const auto format = rawImageFormat(descriptor); + const bool emulated = image.depthCompare && format != IrBufferFormat::Format32Float && format != IrBufferFormat::Format16UNorm && !IsDepthBitsTexture(descriptor.dwords[1], descriptor.dwords[3]); + if (!emulated && (image.emulatedCompare & EmulatedCompare::NativeOffsetUnsupported) != 0u) throw std::runtime_error("native comparison with a nonconstant texel offset requires VK_KHR_maintenance8 and shaderImageGatherExtended"); + if (image.packed && decoded.packedFormat != IrBufferFormat::Invalid) { + const auto format = GetFormatInfo(decoded.packedFormat); + if (format.packedBitfield) throw std::runtime_error("runtime packed image accesses a bitfield format"); + const bool storage = image.resourceClass == ImageResourceClass::Storage; + if (storage && format.byteSize == 12u) throw std::runtime_error("runtime packed image uses a format the hardware does not write"); + for (std::uint32_t component = 0u; component < format.componentCount; ++component) { + const auto bits = format.componentBits[component]; + const bool exact = storage ? format.type == SpirvFormatComponentType::Uint || (bits == 32u && (format.type == SpirvFormatComponentType::Sint || format.type == SpirvFormatComponentType::Float)) : format.type == SpirvFormatComponentType::Uint || format.type == SpirvFormatComponentType::Sint || (format.type == SpirvFormatComponentType::Unorm && bits <= 16u) || (format.type == SpirvFormatComponentType::Float && bits == 32u); + if (!exact) throw std::runtime_error(storage ? "runtime packed image bits are not reproducible through the view" : "runtime packed image bits are not recoverable from the view"); + } } + if (decoded.mipCount > (image.mipMode == ImageMipMode::DynamicStorage ? RuntimeAbi::StorageHeapCapacity : 1u)) throw std::runtime_error("runtime storage image mip capacity exceeded"); + for (std::uint32_t index = 0u; index < modes.size(); ++index) { + const auto& mode = modes[index]; + if (((mode.emulatedCompare & EmulatedCompare::Enabled) != 0u) != emulated) continue; + if (decoded.fmask) { + if (mode.packedFormat == IrBufferFormat::Fmask8_S2_F1) return index; + continue; + } + if (mode.numericClass == decoded.numericClass && mode.dimension == decoded.dimension && mode.conversionFormat == decoded.conversionFormat && mode.packedFormat == decoded.packedFormat && mode.cube == decoded.cube && mode.depthBits == decoded.depthBits && mode.depthUnorm16 == decoded.depthUnorm16 && mode.srgbDecode == decoded.srgbDecode) return index; + } + if (image.dimension == RdnaImageDimension::Dim1D && decoded.dimension != RdnaImageDimension::Dim1D) throw std::runtime_error("image address has too few coordinate components"); + throw std::runtime_error("image descriptor is incompatible with the static runtime image interface"); +} +std::uint32_t ResourceMaterializer::EmulatedCompareState(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint32_t index) { + return emulatedCompareState(info, snapshot, index); +} + +void ResourceMaterializer::ApplyStaticInterface(IrProgram& program, bool nativeSampleOffsets) const { + auto& resources = program.Resources(); + if (!resources.resourceTrackingComplete || !resources.srtPlanComplete) throw std::runtime_error("static resource interface requires a completed resource plan"); auto images = resources.info.images; - images.reserve(specialization.images.size()); - for (std::uint32_t index = 0; index < specialization.images.size(); index++) { - const auto& source = specialization.images[index]; - if (index >= images.size()) { - if (source.indirectRoot >= resources.info.images.size()) { - throw std::runtime_error("ResourceMaterializer::Apply indirect image root is out of range"); + if (!nativeSampleOffsets) { + for (const auto& block : program.Blocks()) { + for (const auto* inst : block->Instructions()) { + if (inst->Opcode() != IrOpcode::ImageSampleRaw) continue; + const auto& memory = resources.memoryInfo.at(inst->Flags().index); + if ((memory.imageSampleFlags & (RdnaImageSampleFlagCompare | RdnaImageSampleFlagOffset)) != (RdnaImageSampleFlagCompare | RdnaImageSampleFlagOffset)) continue; + const auto* address = inst->Argument(2)->Resolve(); + const auto component = GetRdnaImageAddressComponentLayout(memory.imageSampleFlags, 0u); + const auto argument = component.bitOffset / 32u; + if (component.bitWidth == 32u && argument < address->ArgumentCount() && address->Argument(argument)->Resolve()->HasImmediate()) continue; + images.at(memory.resource).emulatedCompare |= EmulatedCompare::NativeOffsetUnsupported; } - images.push_back(resources.info.images[source.indirectRoot]); - } - auto& image = images[index]; - image.numericClass = source.numericClass; - image.dimension = source.dimension; - image.mipCount = source.mipCount; - image.conversionFormat = source.conversionFormat; - image.shaderSwizzle = source.shaderSwizzle; - image.indirectRoot = source.indirectRoot; - image.indirectMappingOffset = source.indirectMappingOffset; - image.indirectSearchIterations = source.indirectSearchIterations; - image.cube = source.cube; - image.depthBits = source.depthBits; - image.depthUnorm16 = source.depthUnorm16; - image.packedFormat = source.packedFormat; - image.emulatedCompare = source.emulatedCompare; - image.srgbDecode = source.srgbDecode; - if ((source.emulatedCompare & EmulatedCompare::Enabled) != 0u) image.depthCompare = false; - image.indirectResources.clear(); - } - for (std::uint32_t index = 0; index < images.size(); index++) { - const auto root = images[index].indirectRoot; - if (root != ImageResource::NoIndirectImage) { - if (root >= images.size()) { - throw std::runtime_error("ResourceMaterializer::Apply indirect image root is out of range"); - } - images[root].indirectResources.push_back(index); } } - - std::vector pointSampler(resources.info.samplers.size(), NoRemap); - std::uint32_t samplerCount = static_cast(resources.info.samplers.size()); - std::vector samplerUsage(resources.info.samplers.size(), 0u); + const auto directCount = static_cast(images.size()); + auto mappingOffset = static_cast(resources.srtReads.size()); + const auto slots = BindlessSlots(); + for (std::uint32_t index = 0u; index < directCount; ++index) { + auto image = images[index]; + image.srgbDecodeFormats = resources.srgbDecodeFormats; + if (image.indirectRoot != ImageResource::NoIndirectImage) throw std::runtime_error("static image interface was already expanded"); + image.numericClass = image.atomic ? IrTextureNumericClass::Uint : IrTextureNumericClass::Float; + image.mipCount = image.mipMode == ImageMipMode::DynamicStorage ? RuntimeAbi::StorageHeapCapacity : 1u; + if (resources.descriptorSources.at(image.source).indirectImage.has_value()) { + if (images.size() + slots - 1u > ShaderInfo::MaxImages) throw std::runtime_error("static bindless image capacity exceeded"); + image.indirectRoot = index; + image.indirectResources.push_back(index); + for (std::uint32_t slot = 1u; slot < slots; ++slot) { + auto entry = image; + entry.indirectResources.clear(); + image.indirectResources.push_back(static_cast(images.size())); + images.push_back(std::move(entry)); + } + image.indirectMappingOffset = mappingOffset; + image.indirectSearchIterations = static_cast(std::bit_width(slots)); + mappingOffset += 1u + 2u * slots; + } + images[index] = std::move(image); + } for (const auto& pair : resources.info.sampledPairs) { - if (pair.image >= images.size() || pair.sampler >= resources.info.samplers.size()) { - throw std::runtime_error("ResourceMaterializer::Apply sampled pair is out of range"); - } - samplerUsage[pair.sampler] |= requiresPointSampler(specialization.images[pair.image]) ? 2u : 1u; + if (pair.image >= images.size() || pair.sampler >= resources.info.samplers.size()) throw std::runtime_error("static sampled pair is out of range"); + auto& sampler = resources.info.samplers[pair.sampler]; + sampler.depthCompare = sampler.depthCompare || images[pair.image].depthCompare; } - for (std::uint32_t index = 0; index < resources.info.samplers.size(); index++) { - if ((samplerUsage[index] & 2u) == 0u) { - continue; - } - if ((samplerUsage[index] & 1u) == 0u) { - pointSampler[index] = index; - } else { - if (samplerCount >= ShaderInfo::MaxSamplers) { - throw std::runtime_error("ResourceMaterializer::Apply exceeds the sampler resource limit"); - } - pointSampler[index] = samplerCount; - samplerCount++; - } - } - auto samplers = resources.info.samplers; - auto sampledPairs = resources.info.sampledPairs; - samplers.reserve(samplerCount); - for (std::uint32_t index = 0; index < resources.info.samplers.size(); index++) { - const auto target = pointSampler[index]; - if (target == NoRemap) { - continue; - } - if (target == index) { - samplers[index].forcePointFiltering = true; - } else { - if (target != samplers.size()) { - throw std::runtime_error("ResourceMaterializer::Apply sampler plan is inconsistent"); - } - auto sampler = samplers[index]; - sampler.forcePointFiltering = true; - samplers.push_back(sampler); - } - } - for (auto& pair : sampledPairs) { - if (requiresPointSampler(specialization.images[pair.image])) { - if (pointSampler[pair.sampler] == NoRemap) { - throw std::runtime_error("ResourceMaterializer::Apply missing point sampler for pair"); - } - pair.sampler = pointSampler[pair.sampler]; - } - samplers[pair.sampler].depthCompare = samplers[pair.sampler].depthCompare || images[pair.image].depthCompare; - } - - auto memoryInfo = resources.memoryInfo; - std::vector imageRemap(specialization.images.size()); - std::uint32_t remapCount = 0; - for (std::uint32_t i = 0; i < specialization.images.size(); i++) { - imageRemap[i] = specialization.images[i].fmask ? NoRemap : remapCount; - if (!specialization.images[i].fmask) { - remapCount++; - } - } - - IrBuilder builder(program); - for (auto& block : program.Blocks()) { - builder.SetInsertionPoint(*block); - for (auto* value : block->Instructions()) { - const auto imageOpcode = ImageOpcodeInfoOf(value->Opcode()); - if (imageOpcode.access == ImageAccess::None) { - continue; - } - const auto flags = value->Flags(); - if (flags.index >= memoryInfo.size()) { - throw std::runtime_error("ResourceMaterializer::Apply memory info index is out of range"); - } - auto& memory = memoryInfo[flags.index]; - if (memory.resource >= images.size()) { - throw std::runtime_error("ResourceMaterializer::Apply memory resource index is out of range"); - } - if (specialization.images[memory.resource].fmask) { - if (value->Opcode() != IrOpcode::ImageRead || memory.dataBits != 32u) { - throw std::runtime_error("ResourceMaterializer::Apply found an unsupported FMASK access"); - } - constexpr std::array fragmentIndices{0x76543210u, 0xfedcba98u}; - std::array fragments{}; - for (std::uint32_t component = 0; component < fragments.size(); component++) { - IrValue& selected = program.CreateValue(IrOpcode::SelectU32, IrType::U32); - selected.AddArgument(value->Argument(2)); - selected.AddArgument(&builder.Constant(fragmentIndices[component])); - selected.AddArgument(&builder.Constant(0u)); - block->InsertInstructionBefore(value, &selected); - fragments[component] = &selected; - } - IrValue& result = program.CreateValue(IrOpcode::CompositeConstructU32x4, IrType::U32x4); - result.AddArgument(fragments[0]); - result.AddArgument(fragments[1]); - result.AddArgument(&builder.Constant(0u)); - result.AddArgument(&builder.Constant(0u)); - block->InsertInstructionBefore(value, &result); - value->ReplaceAllUsesWith(&result); - continue; - } - if (imageOpcode.needsSampler && requiresPointSampler(specialization.images[memory.resource]) && memory.sampler < resources.info.samplers.size()) { - if (pointSampler[memory.sampler] == NoRemap) { - throw std::runtime_error("ResourceMaterializer::Apply missing point sampler for image access"); - } - memory.sampler = pointSampler[memory.sampler]; - } - } - } - - for (auto& memory : memoryInfo) { - if (memory.kind == ResourceKind::Image && !memory.planningOnly) { - if (memory.resource >= imageRemap.size() || imageRemap[memory.resource] == NoRemap) { - throw std::runtime_error("ResourceMaterializer::Apply cannot remap an image memory reference"); - } - memory.resource = imageRemap[memory.resource]; - } - } - for (auto& buffer : buffers) { - if (buffer.imageAlias != BufferResource::NoImageAlias) { - if (buffer.imageAlias >= imageRemap.size() || imageRemap[buffer.imageAlias] == NoRemap) { - throw std::runtime_error("ResourceMaterializer::Apply cannot remap a buffer image alias"); - } - buffer.imageAlias = imageRemap[buffer.imageAlias]; - } - } - for (auto& pair : sampledPairs) { - if (pair.image >= imageRemap.size() || imageRemap[pair.image] == NoRemap) { - throw std::runtime_error("ResourceMaterializer::Apply cannot remap a sampled pair image"); - } - pair.image = imageRemap[pair.image]; - } - for (auto& image : images) { - if (image.indirectRoot != ImageResource::NoIndirectImage) { - if (image.indirectRoot >= imageRemap.size() || imageRemap[image.indirectRoot] == NoRemap) { - throw std::runtime_error("ResourceMaterializer::Apply cannot remap an indirect image root"); - } - image.indirectRoot = imageRemap[image.indirectRoot]; - } - for (auto& resource : image.indirectResources) { - if (resource >= imageRemap.size() || imageRemap[resource] == NoRemap) { - throw std::runtime_error("ResourceMaterializer::Apply cannot remap an indirect image resource"); - } - resource = imageRemap[resource]; - } - } - if (images.size() != imageRemap.size()) { - throw std::runtime_error("ResourceMaterializer::Apply image remap size mismatch"); - } - for (std::uint32_t index = 0; index < images.size(); index++) { - if (imageRemap[index] != NoRemap && imageRemap[index] != index) { - images[imageRemap[index]] = std::move(images[index]); - } - } - images.resize(remapCount); - - resources.info.buffers = std::move(buffers); resources.info.images = std::move(images); - resources.info.samplers = std::move(samplers); - resources.info.sampledPairs = std::move(sampledPairs); - resources.memoryInfo = std::move(memoryInfo); + resources.guardedSrtSlots = Detail::ComputeGuardedFlatSlots(resources); + resources.srtGuardOffset = mappingOffset; + if (!resources.guardedSrtSlots.empty()) resources.info.usesFaultBuffer = true; + PrepareImageModes(resources.info); +} + +void ResourceMaterializer::PrepareImageModes(ShaderInfo& info) { + info.runtimeImageModes.clear(); + info.runtimeImageModes.reserve(info.images.size()); + for (const auto& image : info.images) info.runtimeImageModes.push_back(RuntimeImageModes(image)); } namespace { @@ -967,6 +887,7 @@ IrResourcePlan ResourceMaterializer::ExtractPlan(const IrProgram& program) const plan.srtPlanComplete = source.srtPlanComplete; plan.resourceTrackingComplete = source.resourceTrackingComplete; plan.info = source.info; + PrepareImageModes(plan.info); plan.uniformFill = source.uniformFill; ownPlanValues(plan); const auto addSource = [&plan](std::uint32_t index) { @@ -988,10 +909,11 @@ IrResourcePlan ResourceMaterializer::ExtractPlan(const IrProgram& program) const } for (const auto& sampler : plan.info.samplers) addSource(sampler.source); plan.pureFlatSlots = Detail::ComputePureFlatSlots(plan); + plan.guardedSrtSlots = Detail::ComputeGuardedFlatSlots(plan); return plan; } -void ResourceMaterializer::Materialize(const IrResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot, ResourceSpecialization& specialization) const { +void ResourceMaterializer::Materialize(const IrResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot) const { const IrResourcePlan& plan = program; if (!plan.resourceTrackingComplete) { throw std::runtime_error("ResourceMaterializer::Materialize requires a completed resource plan"); @@ -1008,12 +930,11 @@ void ResourceMaterializer::Materialize(const IrResourcePlan& program, const SrtR reportBindless(); throw; } - ResourceSpecialization nextSpecialization; const auto started = MaterializeProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; - buildResourceSpecialization(plan, nextSnapshot, tables, nextSpecialization); + materializeTables(plan, nextSnapshot, tables); + materializeSrtGuards(plan, nextSnapshot); if (MaterializeProfiled()) specializationNanoseconds.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed); snapshot = std::move(nextSnapshot); - specialization = std::move(nextSpecialization); reportBindless(); } @@ -1022,28 +943,11 @@ std::uint64_t ResourceMaterializer::SpecializationNanoseconds() { } std::uint32_t ResourceMaterializer::BindlessSlots() { - static const std::uint32_t slots = [] { - const char* text = std::getenv("APS5_BINDLESS_SLOTS"); - const auto value = text != nullptr ? std::strtoul(text, nullptr, 0) : 16ul; - return static_cast(std::clamp(value, 1ul, 48ul)); - }(); - return slots; + return RuntimeAbi::SampledHeapCapacity; } void ResourceMaterializer::CountBindlessRejection(BindlessRejection reason) { if (reason < BindlessRejection::Count) bindlessCounters().rejected[static_cast(reason)].fetch_add(1, std::memory_order_relaxed); } -bool ResourceSpecialization::Buffer::operator==(const Buffer& other) const { - return packedStride == other.packedStride && descriptorFormat == other.descriptorFormat && descriptorSwizzle == other.descriptorSwizzle && empty == other.empty; -} - -bool ResourceSpecialization::Image::operator==(const Image& other) const { - return numericClass == other.numericClass && dimension == other.dimension && mipCount == other.mipCount && conversionFormat == other.conversionFormat && shaderSwizzle == other.shaderSwizzle && indirectRoot == other.indirectRoot && indirectMappingOffset == other.indirectMappingOffset && indirectSearchIterations == other.indirectSearchIterations && cube == other.cube && fmask == other.fmask && depthBits == other.depthBits && depthUnorm16 == other.depthUnorm16 && packedFormat == other.packedFormat && emulatedCompare == other.emulatedCompare && srgbDecode == other.srgbDecode; -} - -bool ResourceSpecialization::operator==(const ResourceSpecialization& other) const { - return buffers == other.buffers && images == other.images; -} - } diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index c6839c69f..ddeeb2cb0 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -1,6 +1,7 @@ #include "Optimization/ResourceTracker.hpp" #include "Optimization/SrtWalker.hpp" #include "Optimization/ResourceMaterializer.hpp" +#include "SpirvBackend/SpirvBufferFormat.hpp" #include "IntermediateRepresentation/IrBuilder.hpp" #include "Optimization/SrtWalker/SrtInstructionPredicates.hpp" @@ -798,19 +799,20 @@ private: DescriptorSource descriptor; MakeSource(*handle, 4u, false, false, descriptor); std::uint32_t badDword = 0; - if (ValidateSource(descriptor, badDword)) { - return false; - } - const auto op = inst.Opcode(); - const bool load = op == IrOpcode::LoadBufferU32 || op == IrOpcode::LoadBufferU32x2 || op == IrOpcode::LoadBufferU32x3 || op == IrOpcode::LoadBufferU32x4 || op == IrOpcode::ReadConstBuffer; - const bool store = op == IrOpcode::StoreBufferU32 || op == IrOpcode::StoreBufferU32x2 || op == IrOpcode::StoreBufferU32x3 || op == IrOpcode::StoreBufferU32x4; auto& memory = m_program.Resources().memoryInfo[memoryIndex]; - if ((!load && !store) || memory.formatted || memory.typed || memory.dataBits != 32u) { - return false; + const auto access = BufferAccessOf(inst.Opcode()); + if (ValidateSource(descriptor, badDword)) { + const auto source = InternSource(descriptor); + const auto resource = AddBuffer(source, memory, inst.Opcode(), inst.Flags().pc); + if (resource == std::numeric_limits::max()) fail("buffer resource limit exceeded"); + AddMemoryPatch(memoryIndex, resource, 0u, false); + memory.gpuDescriptor = false; + m_info.usesDma = m_info.usesDma || access == BufferAccess::Atomic; + return true; } memory.gpuDescriptor = true; m_info.usesDma = true; - m_info.bdaWrites = m_info.bdaWrites || store; + m_info.bdaWrites = m_info.bdaWrites || access == BufferAccess::Write || access == BufferAccess::Atomic; return true; } @@ -849,6 +851,13 @@ private: resource.written = resource.written || write; resource.atomic = resource.atomic || atomic; resource.formatted = resource.formatted || memory.formatted; + resource.descriptorFormatted = resource.descriptorFormatted || (memory.formatted && !memory.typed); + if (memory.formatted && memory.typed) { + const auto format = GetFormatInfo(DecodeTBufferFormat(memory.dataFormat, memory.numberFormat)); + if (format.byteSize == 0u) fail("typed buffer instruction has an invalid format"); + resource.typedAlignment = std::max(resource.typedAlignment, static_cast(std::min(format.byteSize, 4u))); + } + if (memory.formatted && !memory.typed && !write) resource.formattedReadMask |= (1u << std::min(memory.dataDwords, 4u)) - 1u; resource.scalar = resource.scalar || op == IrOpcode::ReadConstBuffer || memory.kind == ResourceKind::ScalarBuffer; } @@ -858,8 +867,8 @@ private: const bool depth = (memory.imageSampleFlags & RdnaImageSampleFlagCompare) != 0; for (std::uint32_t i = 0; i < m_info.images.size(); i++) { auto& image = m_info.images[i]; - if (image.source == source && image.resourceClass == resourceClass && image.dimension == memory.imageDimension && image.mipMode == mip && image.depthCompare == depth && image.r128 == memory.imageR128 && image.packed == memory.imagePacked) { - Merge(image, op, pc); + if (image.source == source && image.resourceClass == resourceClass && image.dimension == memory.imageDimension && image.mipMode == mip && image.depthCompare == depth && image.r128 == memory.imageR128 && image.packed == memory.imagePacked && image.byElements == memory.imageByElements && image.byComponents == (memory.imageByElements != 0u ? memory.dataDwords / memory.imageByElements : 0u)) { + Merge(image, memory, op, pc); return i; } } @@ -875,12 +884,14 @@ private: image.depthCompare = depth; image.r128 = memory.imageR128; image.packed = memory.imagePacked; - Merge(image, op, pc); + image.byElements = memory.imageByElements; + image.byComponents = memory.imageByElements != 0u ? memory.dataDwords / memory.imageByElements : 0u; + Merge(image, memory, op, pc); m_info.images.push_back(image); return static_cast(m_info.images.size() - 1); } - static void Merge(ImageResource& image, IrOpcode op, std::uint32_t pc) { + static void Merge(ImageResource& image, const MemoryInfo& memory, IrOpcode op, std::uint32_t pc) { const auto access = ImageOpcodeInfoOf(op).access; const bool atomic = access == ImageAccess::Atomic; const bool write = access == ImageAccess::Write || atomic; @@ -893,6 +904,14 @@ private: image.read = image.read || !write || atomic; image.written = image.written || write; image.atomic = image.atomic || atomic; + image.srgbDecodeCompatible = image.srgbDecodeCompatible && !ImageOpcodeInfoOf(op).needsSampler; + image.fmaskCompatible = image.fmaskCompatible && op == IrOpcode::ImageRead && memory.dataBits == 32u; + image.depthBitsCompatible = image.depthBitsCompatible && memory.dataBits == 32u; + if ((memory.imageSampleFlags & RdnaImageSampleFlagCompare) != 0u) { + constexpr auto unsupported = RdnaImageSampleFlagLod | RdnaImageSampleFlagDerivative; + if (op == IrOpcode::ImageGatherRaw || (memory.imageSampleFlags & unsupported) != 0u) image.emulatedCompare |= EmulatedCompare::Unsupported; + if ((memory.imageSampleFlags & RdnaImageSampleFlagLevelZero) == 0u) image.emulatedCompare |= EmulatedCompare::RequiresSingleLevel; + } } std::uint32_t AddSampler(std::uint32_t source, std::uint32_t pc) { @@ -979,13 +998,7 @@ private: std::uint32_t resource = 0; if (buffer != BufferAccess::None) { - if (TakeGpuDescriptor(inst, flags.index)) { - return; - } - GetHandle(inst.Argument(0), IrOpcode::GetBufferResource, 4, handle, source); - resource = AddBuffer(source, memory, op, flags.pc); - AddHandlePatch(handle, resource); - AddMemoryPatch(flags.index, resource, 0, false); + if (!TakeGpuDescriptor(inst, flags.index)) fail("buffer operation requires a four-dword runtime V#"); return; } if (addressInfo.access != AddressAccess::None) { diff --git a/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp b/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp index e8a2c6ec0..93f66f1c3 100644 --- a/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp +++ b/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp @@ -106,6 +106,20 @@ void ValidateValueReferences(const IrProgram& program, ShaderStageInputInfo inpu } break; } + case IrOpcode::GetInterpolationParameterF16: { + const IrValue* input = inst->Argument(0)->Resolve(); + const IrValue* component = inst->Argument(1)->Resolve(); + const IrValue* mode = inst->Argument(2)->Resolve(); + const IrValue* high = inst->Argument(3)->Resolve(); + if (program.Resources().stage != IrShaderStage::Pixel || !input->HasImmediate() || input->Type() != IrType::U32 || !component->HasImmediate() || component->Type() != IrType::U32 || + !mode->HasImmediate() || mode->Type() != IrType::U32 || !high->HasImmediate() || high->Type() != IrType::U32) { + return Fail("16-bit interpolation parameter reference is invalid"); + } + if (input->ImmediateU32() >= inputInfo.pixel->inputNum || component->ImmediateU32() >= 4u || mode->ImmediateU32() >= 3u || high->ImmediateU32() >= 2u) { + return Fail("16-bit interpolation parameter reference is out of range"); + } + break; + } case IrOpcode::GetBuiltin: { const IrValue* kindValue = inst->Argument(0)->Resolve(); const IrValue* componentValue = inst->Argument(1)->Resolve(); @@ -189,7 +203,7 @@ void CollectVertexInputs(const IrProgram& program, const ShaderVertexInputInfo* if (inst->Opcode() == IrOpcode::GetAttribute) { const auto attribute = inst->Argument(0)->Resolve()->ImmediateU32(); const auto channel = inst->Argument(1)->Resolve()->ImmediateU32(); - usedComponents[attribute] = std::max(usedComponents[attribute], channel + 1u); + usedComponents[attribute] = std::max(usedComponents[attribute], inst->Flags() == 1u ? 4u : channel + 1u); } } } @@ -219,6 +233,7 @@ void CollectPixelInputs(const IrProgram& program, const ShaderPixelInputInfo* pi std::array read {}; std::array perVertex {}; std::array interpolated {}; + std::array halfRead {}; for (const auto& block : program.Blocks()) { for (const IrValue* inst : block->Instructions()) { if (inst->Opcode() == IrOpcode::GetAttribute) { @@ -233,6 +248,14 @@ void CollectPixelInputs(const IrProgram& program, const ShaderPixelInputInfo* pi const auto mode = inst->Argument(2)->Resolve()->ImmediateU32(); read[input] = true; perVertex[input] = perVertex[input] || mode < 2u || !IsPixelParameterFlat(*pixel, input); + } else if (inst->Opcode() == IrOpcode::GetInterpolationParameterF16) { + const auto input = inst->Argument(0)->Resolve()->ImmediateU32(); + const bool high = inst->Argument(3)->Resolve()->ImmediateU32() != 0u; + if (!pixel->InputHalfIsDefault(input, high)) { + read[input] = true; + perVertex[input] = true; + halfRead[input] = true; + } } } } @@ -248,7 +271,7 @@ void CollectPixelInputs(const IrProgram& program, const ShaderPixelInputInfo* pi slotInterpolation.fill(unassigned); std::array slotPerVertex {}; for (std::uint32_t input = 0; input < pixel->inputNum; input++) { - if (!read[input] || pixel->InputIsDefault(input)) { + if (!read[input] || (pixel->InputIsDefault(input) && !halfRead[input])) { continue; } const auto slot = pixel->InputSlot(input); @@ -262,7 +285,7 @@ void CollectPixelInputs(const IrProgram& program, const ShaderPixelInputInfo* pi bool smooth = false; bool noPerspective = false; for (std::uint32_t input = 0; input < pixel->inputNum; input++) { - if (!read[input] || pixel->InputIsDefault(input)) { + if (!read[input] || (pixel->InputIsDefault(input) && !halfRead[input])) { continue; } const bool vertexInput = slotPerVertex[pixel->InputSlot(input)]; @@ -491,13 +514,16 @@ void ShaderInfoCollector::Collect(IrProgram& program, const ShaderStageInputInfo break; case IrShaderStage::TessellationControl: case IrShaderStage::TessellationEvaluation: + break; case IrShaderStage::Mesh: + AddInput(next, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex"); break; case IrShaderStage::Pixel: CollectPixelInputs(program, inputInfo.pixel, next); break; case IrShaderStage::Compute: CollectComputeInputs(inputInfo.compute, next); + if (!next.buffers.empty()) AddInput(next, StageInputKind::LocalInvocationIndex, 0, 1, "gl_LocalInvocationIndex"); break; default: return Fail("unsupported shader stage for info collection"); diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp index 7ea4c4a78..2ef278ff6 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp @@ -44,7 +44,7 @@ const DescriptorSource* Source(const IrResourcePlan& program, std::uint32_t sour } -bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources) { +bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison) { failureReason().clear(); static const bool debug = std::getenv("APS5_SRT_DEBUG") != nullptr; if (debug) { @@ -117,8 +117,14 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span flattened; + std::vector poisoned; if (evaluateFlat) { flattened.resize(program.srtReads.size()); + InaccessibleRead inaccessible; + if (poison != nullptr) { + evaluator.ReportInaccessibleReads(&inaccessible); + cleanEvaluator.ReportInaccessibleReads(&inaccessible); + } for (const auto& read : program.srtReads) { const bool clean = read.flatOffset < cleanFlatSlots.size() && cleanFlatSlots[read.flatOffset] != 0u; auto& selected = clean ? cleanEvaluator : evaluator; @@ -131,8 +137,15 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::spanleaf = read.value->Resolve(); trace->leafSlot = read.flatOffset; } + inaccessible = {}; const bool evaluated = read.flatOffset < flattened.size() && selected.Evaluate(read.value, flattened[read.flatOffset]); if (pure) trace->leaf = nullptr; + if (!evaluated && inaccessible.read != nullptr) { + if (debug) std::fprintf(stderr, "[srt] flat offset %u reads inaccessible 0x%llx at pc 0x%x: poisoned\n", read.flatOffset, static_cast(inaccessible.address), inaccessible.read->Flags().pc); + flattened[read.flatOffset] = 0u; + poisoned.push_back({read.flatOffset, inaccessible.read->Flags().pc, inaccessible.address}); + continue; + } if (!evaluated) { return Fail(std::string(clean ? "clean " : "") + "SRT read at flat offset " + std::to_string(read.flatOffset) + ": " + DescribeValue(read.value, 4)); } @@ -143,6 +156,9 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::spanleaf) trace->leaves.emplace_back(trace->leafSlot, address); else trace->otherReads.push_back(address); @@ -187,6 +191,7 @@ bool Evaluator::EvaluateInst(IrValue& inst, std::uint64_t& result) { case IrOpcode::Phi: return EvaluatePhi(inst, result); case IrOpcode::ReadFirstLane: { Evaluator active(_program, _runtime, _cleanFlatSlots, _cleanEvaluator, inst.Argument(1)); + active.ReportInaccessibleReads(_inaccessible); return active.EvaluateWide(inst.Argument(0), result); } case IrOpcode::BitCastU32F32: diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp index d0b787b7f..cca8ad626 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp @@ -1,4 +1,5 @@ #include "Optimization/SrtWalker/SrtFlatSlotClasses.hpp" +#include "Optimization/SrtWalker/SrtInstructionPredicates.hpp" #include #include @@ -61,4 +62,47 @@ std::vector ComputePureFlatSlots(const IrResourcePlan& plan) { return pure; } +std::vector ComputeGuardedFlatSlots(const IrResourcePlan& plan) { + std::unordered_map> users; + std::unordered_set seen; + std::vector pending; + std::vector poisonable; + for (const auto& read : plan.srtReads) { + if (read.value != nullptr) pending.push_back(read.value->Resolve()); + } + while (!pending.empty()) { + const IrValue* value = pending.back(); + pending.pop_back(); + if (!seen.insert(value).second) continue; + const IrValue* handle = value->ArgumentCount() != 0u && value->Argument(0) != nullptr ? value->Argument(0)->Resolve() : nullptr; + if (IsRawRead(plan, *value) && handle != nullptr && LoadedFromMemory(*handle)) poisonable.push_back(value); + const auto follow = [&](const IrValue* argument) { + if (argument == nullptr) return; + argument = argument->Resolve(); + users[argument].push_back(value); + pending.push_back(argument); + }; + for (std::size_t i = 0; i < value->ArgumentCount(); ++i) follow(value->Argument(i)); + if (value->Opcode() == IrOpcode::ReadConst && value->ArgumentCount() >= 2 && value->Argument(1) != nullptr) { + const IrValue* slot = value->Argument(1)->Resolve(); + if (slot->HasImmediate() && slot->Type() == IrType::U32 && slot->ImmediateU32() < plan.srtReads.size()) follow(plan.srtReads[slot->ImmediateU32()].value); + } + } + std::unordered_set reaching(poisonable.begin(), poisonable.end()); + while (!poisonable.empty()) { + const IrValue* value = poisonable.back(); + poisonable.pop_back(); + const auto found = users.find(value); + if (found == users.end()) continue; + for (const IrValue* user : found->second) { + if (reaching.insert(user).second) poisonable.push_back(user); + } + } + std::vector guarded; + for (std::uint32_t slot = 0; slot < plan.srtReads.size(); ++slot) { + if (plan.srtReads[slot].value != nullptr && reaching.contains(plan.srtReads[slot].value->Resolve())) guarded.push_back(slot); + } + return guarded; +} + } diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp index 46735d2cc..274524be7 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp @@ -1,5 +1,8 @@ #include "Optimization/SrtWalker/SrtInstructionPredicates.hpp" +#include +#include + namespace ShaderRecompiler::Detail { bool IsRawRead(const IrResourcePlan& program, const IrValue& inst) { @@ -15,6 +18,24 @@ bool IsRawRead(const IrResourcePlan& program, const IrValue& inst) { return (op == IrOpcode::LoadAddressU32 && kind == ResourceKind::ScalarAddress) || (op == IrOpcode::ReadConstBuffer && kind == ResourceKind::ScalarBuffer); } +bool LoadedFromMemory(const IrValue& handle) { + std::vector pending; + std::vector visited; + for (std::size_t index = 0; index < handle.ArgumentCount(); ++index) pending.push_back(handle.Argument(index)); + while (!pending.empty()) { + const IrValue* value = pending.back(); + pending.pop_back(); + if (value == nullptr) continue; + value = value->Resolve(); + if (value->HasImmediate() || std::find(visited.begin(), visited.end(), value) != visited.end()) continue; + visited.push_back(value); + const auto opcode = value->Opcode(); + if (opcode == IrOpcode::ReadConst || opcode == IrOpcode::LoadAddressU32 || opcode == IrOpcode::ReadConstBuffer) return true; + for (std::size_t index = 0; index < value->ArgumentCount(); ++index) pending.push_back(value->Argument(index)); + } + return false; +} + bool IsDescriptorHandle(IrOpcode opcode) { switch (opcode) { case IrOpcode::GetBufferResource: diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp index 2000b1c90..4e8a85d89 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp @@ -49,8 +49,8 @@ void SrtWalker::EvaluateDescriptorSources(const IrResourcePlan& program, std::sp } } -void SrtWalker::EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources) const { - if (!Detail::EvaluateRuntimeSourcesImpl(program, sources, runtime, results, flat, true, cleanFlatSlots, activeSources)) { +void SrtWalker::EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison) const { + if (!Detail::EvaluateRuntimeSourcesImpl(program, sources, runtime, results, flat, true, cleanFlatSlots, activeSources, poison)) { throw std::runtime_error("SrtWalker::EvaluateRuntimeSources failed to evaluate runtime sources: " + Detail::RuntimeSourceFailureReason()); } } diff --git a/core/shader/recompiler/PipelineSpecialization.hpp b/core/shader/recompiler/PipelineSpecialization.hpp new file mode 100644 index 000000000..4b7fea929 --- /dev/null +++ b/core/shader/recompiler/PipelineSpecialization.hpp @@ -0,0 +1,42 @@ +#ifndef CORE_SHADER_RECOMPILER_PIPELINESPECIALIZATION_HPP +#define CORE_SHADER_RECOMPILER_PIPELINESPECIALIZATION_HPP + +#include + +namespace ShaderRecompiler { + +struct PipelineSpecializationConstant { + std::uint32_t id; + std::uint32_t value; + + bool operator==(const PipelineSpecializationConstant&) const = default; +}; + +namespace PipelineSpecialization { + +inline constexpr std::uint32_t CompareBase = 49152u; +inline constexpr std::uint32_t CompareWords = 6u; +inline constexpr std::uint32_t BufferBase = 0u; +inline constexpr std::uint32_t BufferWords = 4u; +inline constexpr std::uint32_t ImageBase = 1024u; +inline constexpr std::uint32_t ImageWords = 5u; +inline constexpr std::uint32_t VertexBase = 4096u; +inline constexpr std::uint32_t VertexWords = 6u; +inline constexpr std::uint32_t PushDataOffset = 8188u; +inline constexpr std::uint32_t ExportBase = 8192u; +inline constexpr std::uint32_t HeapCountBase = 8256u; +inline constexpr std::uint32_t MipCountBase = 8320u; +inline constexpr std::uint32_t DescriptorIndexBase = 16384u; +inline constexpr std::uint32_t DescriptorIndexStride = 128u; +inline constexpr std::uint32_t ImageModeBase = 32768u; +inline constexpr std::uint32_t ImageModeStride = 256u; + +inline constexpr std::uint32_t DescriptorIndex(std::uint32_t binding, std::uint32_t element) { + return DescriptorIndexBase + binding * DescriptorIndexStride + element; +} + +} + +} + +#endif diff --git a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaDescriptorFormat.hpp b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaDescriptorFormat.hpp index 32796f010..5eb41904a 100644 --- a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaDescriptorFormat.hpp +++ b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaDescriptorFormat.hpp @@ -17,6 +17,7 @@ enum class ImageType : std::uint32_t { Color2DMsaaArray = 15 }; +[[nodiscard]] IrTextureNumericClass VertexInputNumericClass(IrBufferFormat format); [[nodiscard]] bool IsFmaskTextureFormat(IrBufferFormat format); [[nodiscard]] IrTextureNumericClass SampledTextureNumericClass(IrBufferFormat format); [[nodiscard]] IrBufferFormat RemapTextureFormat(IrBufferFormat format); diff --git a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp index 9cc00be2f..bdffac7f0 100644 --- a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp +++ b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp @@ -203,6 +203,7 @@ enum class RdnaOpcode : std::uint16_t { SMovrelsd2B32, SQuadmaskB64, SGetpcB64, + SCallB64, SSubvectorLoopBegin, SSubvectorLoopEnd, SAndSaveexecB32, @@ -932,10 +933,27 @@ enum class RdnaOpcode : std::uint16_t { DsXorRtnB64, DsMskorRtnB64, DsWrxchgRtnB64, + DsCondxchg32RtnB64, DsCmpstRtnB64, DsCmpstRtnF64, DsMinRtnF64, DsMaxRtnF64, + DsAddSrc2U32, + DsSubSrc2U32, + DsRsubSrc2U32, + DsIncSrc2U32, + DsDecSrc2U32, + DsMinSrc2I32, + DsMaxSrc2I32, + DsMinSrc2U32, + DsMaxSrc2U32, + DsAndSrc2B32, + DsOrSrc2B32, + DsXorSrc2B32, + DsWriteSrc2B32, + DsMinSrc2F32, + DsMaxSrc2F32, + DsAddSrc2F32, DsNop, DsReadU8D16, DsReadU8D16Hi, @@ -960,6 +978,22 @@ enum class RdnaOpcode : std::uint16_t { ImageStoreMip, ImageStorePck, ImageStoreMipPck, + ImageLoadBy2, + ImageLoadBy4, + ImageLoadMipBy2, + ImageLoadMipBy4, + ImageStoreBy2, + ImageStoreBy4, + ImageStoreMipBy2, + ImageStoreMipBy4, + ImageLoadPck2, + ImageLoadPck4, + ImageLoadMipPck2, + ImageLoadMipPck4, + ImageStorePck2, + ImageStorePck4, + ImageStoreMipPck2, + ImageStoreMipPck4, ImageMsaaLoad, ImageAtomicSwap, ImageAtomicAdd, @@ -984,6 +1018,8 @@ enum class RdnaOpcode : std::uint16_t { ImageGather4CO, ImageGather4CLzO, ImageGather4h, + ImageGather4hPck, + ImageGather8hPck, ImageGather4, ImageGather4B, ImageGather4BCl, @@ -1007,6 +1043,9 @@ enum class RdnaOpcode : std::uint16_t { VInterpP1F32, VInterpP2F32, VInterpMovF32, + VInterpP1llF16, + VInterpP1lvF16, + VInterpP2F16, SWaitcntDepctr, SSendmsg, SGetregB32, @@ -1041,6 +1080,18 @@ enum class RdnaOpcode : std::uint16_t { Exp, VMulHiI32I24, VMulHiU32U24, + SSwappcB64, + SSethalt, + SSendmsghalt, + SCodeEnd, + SRfeB64, + DsGwsInit, + DsGwsSemaV, + DsGwsSemaBr, + DsGwsSemaP, + DsGwsSemaReleaseAll, + DsGwsBarrier, + DsOrderedCount, Count }; diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaDescriptorFormat.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaDescriptorFormat.cpp index 21faa359a..7cefea341 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaDescriptorFormat.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaDescriptorFormat.cpp @@ -13,7 +13,7 @@ struct FormatInfo { bool sint32; }; -constexpr std::array kFormatInfoTable {{ +constexpr std::array kFormatInfoTable {{ {IrBufferFormat::Format8UNorm, true, false, false}, {IrBufferFormat::Format8SNorm, false, false, false}, {IrBufferFormat::Format8UInt, true, true, false}, @@ -38,6 +38,7 @@ constexpr std::array kFormatInfoTable {{ {IrBufferFormat::Format11_11_10UNorm, true, true, false}, {IrBufferFormat::Format11_11_10UInt, true, true, false}, {IrBufferFormat::Format11_11_10Float, true, false, false}, + {IrBufferFormat::Format10_11_11Float, true, true, false}, {IrBufferFormat::Format10_10_10_2UNorm, true, false, false}, {IrBufferFormat::Format10_10_10_2UInt, true, true, false}, {IrBufferFormat::Format8_8_8_8UNorm, true, false, false}, @@ -120,7 +121,7 @@ IrTextureNumericClass SampledTextureNumericClass(IrBufferFormat format) { } IrBufferFormat RemapTextureFormat(IrBufferFormat format) { - return format == IrBufferFormat::Format11_11_10UNorm || format == IrBufferFormat::Format11_11_10UInt ? IrBufferFormat::Format32UInt : format; + return format == IrBufferFormat::Format11_11_10UNorm || format == IrBufferFormat::Format11_11_10UInt || format == IrBufferFormat::Format10_11_11Float ? IrBufferFormat::Format32UInt : format; } std::uint32_t DepthBitsTextureWidth(std::uint32_t word1, std::uint32_t word3) { @@ -136,4 +137,30 @@ bool IsDepthBitsTexture(std::uint32_t word1, std::uint32_t word3) { return DepthBitsTextureWidth(word1, word3) != 0u; } +IrTextureNumericClass VertexInputNumericClass(IrBufferFormat format) { + switch (format) { + case IrBufferFormat::Format8UInt: + case IrBufferFormat::Format16UInt: + case IrBufferFormat::Format8_8UInt: + case IrBufferFormat::Format32UInt: + case IrBufferFormat::Format16_16UInt: + case IrBufferFormat::Format8_8_8_8UInt: + case IrBufferFormat::Format32_32UInt: + case IrBufferFormat::Format16_16_16_16UInt: + case IrBufferFormat::Format32_32_32UInt: + case IrBufferFormat::Format32_32_32_32UInt: return IrTextureNumericClass::Uint; + case IrBufferFormat::Format8SInt: + case IrBufferFormat::Format16SInt: + case IrBufferFormat::Format8_8SInt: + case IrBufferFormat::Format32SInt: + case IrBufferFormat::Format16_16SInt: + case IrBufferFormat::Format8_8_8_8SInt: + case IrBufferFormat::Format32_32SInt: + case IrBufferFormat::Format16_16_16_16SInt: + case IrBufferFormat::Format32_32_32SInt: + case IrBufferFormat::Format32_32_32_32SInt: return IrTextureNumericClass::Sint; + default: return IrTextureNumericClass::Float; + } +} + } diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp index 4d0869dc2..640be3b59 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp @@ -87,6 +87,8 @@ constexpr ImageOpcodeInfo imageOpcodes[] = { {0x58u, RdnaOpcode::ImageGather4CO, nullptr, RdnaImageSampleFlagCompare | RdnaImageSampleFlagOffset, false, true, false}, {0x5fu, RdnaOpcode::ImageGather4CLzO, nullptr, RdnaImageSampleFlagCompare | RdnaImageSampleFlagLevelZero | RdnaImageSampleFlagOffset, false, true, false}, {0x61u, RdnaOpcode::ImageGather4h, nullptr, RdnaImageSampleFlagGatherHorizontal, false, true, false}, + {0x62u, RdnaOpcode::ImageGather4hPck, nullptr, RdnaImageSampleFlagGatherHorizontal, false, true, false}, + {0x63u, RdnaOpcode::ImageGather8hPck, nullptr, RdnaImageSampleFlagGatherHorizontal, false, true, false}, {0x50u, RdnaOpcode::ImageGather4O, nullptr, RdnaImageSampleFlagOffset, false, true, false}, {0x54u, RdnaOpcode::ImageGather4LO, nullptr, RdnaImageSampleFlagLod | RdnaImageSampleFlagOffset, false, true, false}, {0x44u, RdnaOpcode::ImageGather4L, nullptr, RdnaImageSampleFlagLod, false, true, false}, @@ -132,6 +134,22 @@ constexpr ImageOpcodeInfo imageOpcodes[] = { {0x0au, RdnaOpcode::ImageStorePck, nullptr, 0, false, false, false}, {0x0bu, RdnaOpcode::ImageStoreMipPck, nullptr, 0, false, false, false}, {0x0eu, RdnaOpcode::ImageGetResinfo, nullptr, 0, false, false, false}, + {0x42u, RdnaOpcode::ImageLoadBy2, nullptr, 0, false, false, false}, + {0x43u, RdnaOpcode::ImageLoadBy4, nullptr, 0, false, false, false}, + {0x4au, RdnaOpcode::ImageLoadMipBy2, nullptr, 0, false, false, false}, + {0x4bu, RdnaOpcode::ImageLoadMipBy4, nullptr, 0, false, false, false}, + {0x52u, RdnaOpcode::ImageStoreBy2, nullptr, 0, false, false, false}, + {0x53u, RdnaOpcode::ImageStoreBy4, nullptr, 0, false, false, false}, + {0x5au, RdnaOpcode::ImageStoreMipBy2, nullptr, 0, false, false, false}, + {0x5bu, RdnaOpcode::ImageStoreMipBy4, nullptr, 0, false, false, false}, + {0x70u, RdnaOpcode::ImageLoadPck2, nullptr, 0, false, false, false}, + {0x71u, RdnaOpcode::ImageLoadPck4, nullptr, 0, false, false, false}, + {0x73u, RdnaOpcode::ImageLoadMipPck2, nullptr, 0, false, false, false}, + {0x74u, RdnaOpcode::ImageLoadMipPck4, nullptr, 0, false, false, false}, + {0x76u, RdnaOpcode::ImageStorePck2, nullptr, 0, false, false, false}, + {0x77u, RdnaOpcode::ImageStorePck4, nullptr, 0, false, false, false}, + {0x79u, RdnaOpcode::ImageStoreMipPck2, nullptr, 0, false, false, false}, + {0x7au, RdnaOpcode::ImageStoreMipPck4, nullptr, 0, false, false, false}, {0x80u, RdnaOpcode::ImageMsaaLoad, nullptr, 0, false, false, false}, {0x60u, RdnaOpcode::ImageGetLod, nullptr, 0, false, false, false}, {0xe6u, RdnaOpcode::ImageBvhIntersectRay, "image_bvh_intersect_ray", 0, false, false, false}, @@ -315,7 +333,7 @@ RdnaInstruction DecodeRdnaMimg(std::uint32_t programCounter, std::span> 18u) & 0x7Fu) | ((word0 & 1u) << 7u); const auto& info = lookupOpcode(opcode); const bool samplerOp = info.sample || info.gather || opcode == 0x60u; - const auto texelStatusBits = samplerOp ? 0x00010000u : opcode <= 0x0eu ? 0x00030000u : 0u; + const auto texelStatusBits = info.sample || info.gather || opcode <= 0x0eu ? 0x00030000u : samplerOp ? 0x00010000u : 0u; const auto reservedWord0 = (info.sample || info.gather ? 0x00035040u : info.atomic ? 0x000340C0u : 0x00034040u) & ~texelStatusBits; if ((word0 & reservedWord0) != 0u || (word1 & 0x3C000000u) != 0u) { char message[96]; @@ -347,10 +365,20 @@ RdnaInstruction DecodeRdnaMimg(std::uint32_t programCounter, std::span> 8u) & 15u; + const bool pckGather = opcode == 0x62u || opcode == 0x63u; + const bool by = opcode == 0x42u || opcode == 0x43u || opcode == 0x4au || opcode == 0x4bu || opcode == 0x52u || opcode == 0x53u || opcode == 0x5au || opcode == 0x5bu; + if (by && ((dmask != 15u && (dmask != 3u || (opcode & 1u) != 0u)) || a16 || d16 || dimension != RdnaImageDimension::Dim2D || (word0 & 0x8000u) != 0u)) { + throw std::runtime_error("MIMG BY2/BY4 requires a data mask that covers every element (BY2 0x3 or 0xf, BY4 0xf), 32-bit addresses and data, a 2D image and a full descriptor"); + } + const bool pckN = opcode >= 0x70u && opcode <= 0x7au && opcode != 0x72u && opcode != 0x75u && opcode != 0x78u; + const bool pckNMip = opcode == 0x73u || opcode == 0x74u || opcode == 0x79u || opcode == 0x7au; + if (pckN && (dmask != 1u || a16 || d16 || dimension != RdnaImageDimension::Dim2D || (word0 & 0x8000u) != 0u)) { + throw std::runtime_error("MIMG PCK2/PCK4 requires data mask 0x1, 32-bit addresses and data, a 2D image and a full descriptor"); + } const bool compareSwap = info.opcode == RdnaOpcode::ImageAtomicCmpswap || info.opcode == RdnaOpcode::ImageAtomicFcmpswap; const bool floatAtomic = info.opcode == RdnaOpcode::ImageAtomicFcmpswap || info.opcode == RdnaOpcode::ImageAtomicFmin || info.opcode == RdnaOpcode::ImageAtomicFmax; const bool atomic64 = info.atomic && !floatAtomic && !d16 && dmask == (compareSwap ? 15u : 3u); - if (dmask == 0u || (!atomic64 && (compareSwap ? dmask != 3u : (info.gather || info.atomic || msaaLoad) && !std::has_single_bit(dmask)))) { + if (dmask == 0u || (!atomic64 && (compareSwap ? dmask != 3u : ((info.gather && !pckGather) || info.atomic || msaaLoad) && !std::has_single_bit(dmask)))) { throw std::runtime_error("invalid MIMG data mask"); } if (d16 && !(info.sample || info.gather || opcode == 0u || opcode == 1u || opcode == 8u || opcode == 9u)) { @@ -359,7 +387,7 @@ RdnaInstruction DecodeRdnaMimg(std::uint32_t programCounter, std::span 256u - vaddr) { throw std::runtime_error("MIMG address register range overflow"); } - const auto dataComponents = info.gather || msaaLoad ? 4u : static_cast(std::popcount(dmask)); + const auto dataComponents = pckGather ? static_cast(std::popcount(dmask)) : info.gather || msaaLoad ? 4u : static_cast(std::popcount(dmask)); const auto dataDwords = d16 ? (dataComponents + 1u) / 2u : dataComponents; const auto statusDwords = (word0 & 0x00010000u) != 0u ? 1u : 0u; if (dataDwords + statusDwords > 256u - vdata) { diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaInstructionDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaInstructionDecoder.cpp index f1d527ac9..f03944673 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaInstructionDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaInstructionDecoder.cpp @@ -160,7 +160,7 @@ void DecodeRdnaProgram(std::span code, RdnaProgram& program const RdnaInstruction& instruction = program.instructions.back(); wordIndex += instruction.wordCount; - if (IsDirectBranchOpcode(instruction.op)) { + if (IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64) { const std::uint32_t targetIndex = instruction.branchTarget / static_cast(sizeof(std::uint32_t)); if (targetIndex >= code.size()) { throw std::out_of_range("branch target is out of the code span bounds"); @@ -168,7 +168,7 @@ void DecodeRdnaProgram(std::span code, RdnaProgram& program furthestBranchTarget = std::max(furthestBranchTarget, targetIndex); } - if (instruction.op == RdnaOpcode::SEndpgm && furthestBranchTarget < wordIndex) { + if ((instruction.op == RdnaOpcode::SEndpgm || instruction.op == RdnaOpcode::SCodeEnd) && furthestBranchTarget < wordIndex) { return; } } diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp index 3f155040a..8f8c5c142 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp @@ -234,6 +234,12 @@ constexpr MemoryOpcodeInfo dsOpcodes[] = { {0x11u, RdnaOpcode::DsCmpstF32, 1, 32, false, false, false}, {0x14u, RdnaOpcode::DsNop, 1, 32, false, false, false}, {0x15u, RdnaOpcode::DsAddF32, 1, 32, false, false, false}, + {0x18u, RdnaOpcode::DsGwsSemaReleaseAll, 1, 32, false, false, false}, + {0x19u, RdnaOpcode::DsGwsInit, 1, 32, false, false, false}, + {0x1au, RdnaOpcode::DsGwsSemaV, 1, 32, false, false, false}, + {0x1bu, RdnaOpcode::DsGwsSemaBr, 1, 32, false, false, false}, + {0x1cu, RdnaOpcode::DsGwsSemaP, 1, 32, false, false, false}, + {0x1du, RdnaOpcode::DsGwsBarrier, 1, 32, false, false, false}, {0x22u, RdnaOpcode::DsRsubRtnU32, 1, 32, false, false, false}, {0x2cu, RdnaOpcode::DsMskorRtnB32, 1, 32, false, false, false}, {0x30u, RdnaOpcode::DsCmpstRtnB32, 1, 32, false, false, false}, @@ -277,6 +283,22 @@ constexpr MemoryOpcodeInfo dsOpcodes[] = { {0x71u, RdnaOpcode::DsCmpstRtnF64, 2, 32, false, false, false}, {0x72u, RdnaOpcode::DsMinRtnF64, 2, 32, false, false, false}, {0x73u, RdnaOpcode::DsMaxRtnF64, 2, 32, false, false, false}, + {0x80u, RdnaOpcode::DsAddSrc2U32, 1, 32, false, false, false}, + {0x81u, RdnaOpcode::DsSubSrc2U32, 1, 32, false, false, false}, + {0x82u, RdnaOpcode::DsRsubSrc2U32, 1, 32, false, false, false}, + {0x83u, RdnaOpcode::DsIncSrc2U32, 1, 32, false, false, false}, + {0x84u, RdnaOpcode::DsDecSrc2U32, 1, 32, false, false, false}, + {0x85u, RdnaOpcode::DsMinSrc2I32, 1, 32, false, false, false}, + {0x86u, RdnaOpcode::DsMaxSrc2I32, 1, 32, false, false, false}, + {0x87u, RdnaOpcode::DsMinSrc2U32, 1, 32, false, false, false}, + {0x88u, RdnaOpcode::DsMaxSrc2U32, 1, 32, false, false, false}, + {0x89u, RdnaOpcode::DsAndSrc2B32, 1, 32, false, false, false}, + {0x8au, RdnaOpcode::DsOrSrc2B32, 1, 32, false, false, false}, + {0x8bu, RdnaOpcode::DsXorSrc2B32, 1, 32, false, false, false}, + {0x8du, RdnaOpcode::DsWriteSrc2B32, 1, 32, false, false, false}, + {0x92u, RdnaOpcode::DsMinSrc2F32, 1, 32, false, false, false}, + {0x93u, RdnaOpcode::DsMaxSrc2F32, 1, 32, false, false, false}, + {0x95u, RdnaOpcode::DsAddSrc2F32, 1, 32, false, false, false}, {0x01u, RdnaOpcode::DsSubU32, 1, 32, false, false, false}, {0x05u, RdnaOpcode::DsMinI32, 1, 32, false, false, false}, {0x06u, RdnaOpcode::DsMaxI32, 1, 32, false, false, false}, @@ -316,6 +338,7 @@ constexpr MemoryOpcodeInfo dsOpcodes[] = { {0x3cu, RdnaOpcode::DsReadU16, 1, 16, false, false, false}, {0x3du, RdnaOpcode::DsConsume, 1, 32, false, false, false}, {0x3eu, RdnaOpcode::DsAppend, 1, 32, false, false, false}, + {0x3fu, RdnaOpcode::DsOrderedCount, 1, 32, false, false, false}, {0x4du, RdnaOpcode::DsWriteB64, 2, 32, false, false, false}, {0x4eu, RdnaOpcode::DsWrite2B64, 4, 32, false, false, false}, {0x4fu, RdnaOpcode::DsWrite2st64B64, 4, 32, false, false, false}, @@ -324,6 +347,7 @@ constexpr MemoryOpcodeInfo dsOpcodes[] = { {0x76u, RdnaOpcode::DsReadB64, 2, 32, false, false, false}, {0x77u, RdnaOpcode::DsRead2B64, 4, 32, false, false, false}, {0x78u, RdnaOpcode::DsRead2st64B64, 4, 32, false, false, false}, + {0x7eu, RdnaOpcode::DsCondxchg32RtnB64, 2, 32, false, false, false}, {0xa0u, RdnaOpcode::DsWriteB8D16Hi, 1, 8, false, false, false}, {0xa1u, RdnaOpcode::DsWriteB16D16Hi, 1, 16, false, false, false}, {0xa2u, RdnaOpcode::DsReadU8D16, 1, 8, false, false, false}, @@ -468,6 +492,18 @@ bool isDsWriteOpcode(RdnaOpcode opcode) { } } +bool isDsGwsOpcode(RdnaOpcode opcode) { + switch (opcode) { + case RdnaOpcode::DsGwsInit: + case RdnaOpcode::DsGwsSemaV: + case RdnaOpcode::DsGwsSemaBr: + case RdnaOpcode::DsGwsSemaP: + case RdnaOpcode::DsGwsSemaReleaseAll: + case RdnaOpcode::DsGwsBarrier: return true; + default: return false; + } +} + bool isDsAtomicOpcode(RdnaOpcode opcode) { switch (opcode) { case RdnaOpcode::DsAddU32: @@ -526,6 +562,7 @@ bool isDsAtomicOpcode(RdnaOpcode opcode) { case RdnaOpcode::DsOrRtnB64: case RdnaOpcode::DsXorRtnB64: case RdnaOpcode::DsWrxchgRtnB64: + case RdnaOpcode::DsCondxchg32RtnB64: case RdnaOpcode::DsMinRtnF64: case RdnaOpcode::DsMaxRtnF64: return true; @@ -533,6 +570,29 @@ bool isDsAtomicOpcode(RdnaOpcode opcode) { } } +bool isDsSrc2Opcode(RdnaOpcode opcode) { + switch (opcode) { + case RdnaOpcode::DsAddSrc2U32: + case RdnaOpcode::DsSubSrc2U32: + case RdnaOpcode::DsRsubSrc2U32: + case RdnaOpcode::DsIncSrc2U32: + case RdnaOpcode::DsDecSrc2U32: + case RdnaOpcode::DsMinSrc2I32: + case RdnaOpcode::DsMaxSrc2I32: + case RdnaOpcode::DsMinSrc2U32: + case RdnaOpcode::DsMaxSrc2U32: + case RdnaOpcode::DsAndSrc2B32: + case RdnaOpcode::DsOrSrc2B32: + case RdnaOpcode::DsXorSrc2B32: + case RdnaOpcode::DsWriteSrc2B32: + case RdnaOpcode::DsMinSrc2F32: + case RdnaOpcode::DsMaxSrc2F32: + case RdnaOpcode::DsAddSrc2F32: + return true; + default: return false; + } +} + std::uint32_t dsSourceCount(RdnaOpcode opcode) { switch (opcode) { case RdnaOpcode::DsWrxchg2RtnB32: @@ -563,7 +623,10 @@ std::uint32_t dsSourceCount(RdnaOpcode opcode) { case RdnaOpcode::DsBpermuteB32: return 2u; case RdnaOpcode::DsReadAddtidB32: case RdnaOpcode::DsConsume: - case RdnaOpcode::DsAppend: return 0u; + case RdnaOpcode::DsAppend: + case RdnaOpcode::DsGwsSemaV: + case RdnaOpcode::DsGwsSemaP: + case RdnaOpcode::DsGwsSemaReleaseAll: return 0u; default: return isDsWriteOpcode(opcode) || isDsAtomicOpcode(opcode) ? 2u : 1u; } } @@ -918,6 +981,14 @@ RdnaInstruction DecodeRdnaDs(std::uint32_t programCounter, std::span 256u) { + throw std::runtime_error("DS conditional exchange source register range overflow"); + } + if (vdst + 2u > 256u) { + throw std::runtime_error("DS conditional exchange destination register range overflow"); + } + } if (info.opcode == RdnaOpcode::DsReadAddtidB32 && (data0 != 0u || data1 != 0u)) { throw std::runtime_error("DS read addtid data operands are not supported"); } @@ -943,6 +1014,12 @@ RdnaInstruction DecodeRdnaDs(std::uint32_t programCounter, std::span(immediate * 4); return instruction; } diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp index 880f08383..326343f03 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp @@ -493,6 +493,7 @@ constexpr VectorOpcodeInfo vop3Opcodes[] = { {0x357u, RdnaOpcode::VMed3F16}, {0x358u, RdnaOpcode::VMed3I16}, {0x359u, RdnaOpcode::VMed3U16}, + {0x35au, RdnaOpcode::VInterpP2F16}, {0x35fu, RdnaOpcode::VDivFixupF16}, {0x15au, RdnaOpcode::VSadU8}, {0x15bu, RdnaOpcode::VSadHiU8}, @@ -534,6 +535,8 @@ constexpr VectorOpcodeInfo vop3Opcodes[] = { {0x311u, RdnaOpcode::VPackB32F16}, {0x314u, RdnaOpcode::VLshlrevB16}, {0x319u, RdnaOpcode::VSubrevI32}, + {0x342u, RdnaOpcode::VInterpP1llF16}, + {0x343u, RdnaOpcode::VInterpP1lvF16}, {0x344u, RdnaOpcode::VPermB32}, {0x14du, RdnaOpcode::VLerpU8}, {0x345u, RdnaOpcode::VXadU32}, @@ -2073,6 +2076,43 @@ void checkNativeVop3Modifiers(RdnaOpcode opcode, bool permlane, bool carryInOut, } } +bool isVop3F16InterpolationOpcode(RdnaOpcode opcode) { + return opcode == RdnaOpcode::VInterpP1llF16 || opcode == RdnaOpcode::VInterpP1lvF16 || opcode == RdnaOpcode::VInterpP2F16; +} + +void decodeVop3F16Interpolation(RdnaInstruction& instruction, std::uint32_t vdst, std::uint32_t src0, std::uint32_t src1, std::uint32_t src2, std::uint32_t abs, std::uint32_t opSel, std::uint32_t clamp, std::uint32_t omod, std::uint32_t neg) { + const bool hasSource2 = instruction.op != RdnaOpcode::VInterpP1llF16; + const bool f16Result = instruction.op == RdnaOpcode::VInterpP2F16; + if ((abs & 0x1u) != 0u || (neg & 0x1u) != 0u || opSel != 0u || (f16Result && omod != 0u) || (!hasSource2 && ((abs | neg) & 0x4u) != 0u)) { + throw std::invalid_argument("VOP3 interpolation modifiers are not implemented"); + } + if (src1 < 256u || (hasSource2 && src2 < 256u)) { + throw std::invalid_argument("VOP3 interpolation source is not a vector register"); + } + const bool high = ((src0 >> 8u) & 0x1u) != 0u; + instruction.destination = DecodeRdnaVectorGpr(vdst); + instruction.destination.clamp = clamp != 0u; + instruction.destination.omod = omod; + instruction.source0 = DecodeRdnaVectorGpr(src1 - 256u); + instruction.source0.absolute = (abs & 0x2u) != 0u; + instruction.source0.negate = (neg & 0x2u) != 0u; + instruction.source1.kind = RdnaOperandKind::IntegerInlineConstant; + instruction.source1.value = src0 & 0x3fu; + instruction.source1.signedVal = static_cast(instruction.source1.value); + instruction.source1.opSel = high; + instruction.source2.kind = RdnaOperandKind::IntegerInlineConstant; + instruction.source2.value = (src0 >> 6u) & 0x3u; + instruction.source2.signedVal = static_cast(instruction.source2.value); + instruction.sourceCount = 3; + if (hasSource2) { + instruction.source3 = DecodeRdnaVectorGpr(src2 - 256u); + instruction.source3.absolute = (abs & 0x4u) != 0u; + instruction.source3.negate = (neg & 0x4u) != 0u; + instruction.source3.opSel = high && !f16Result; + instruction.sourceCount = 4; + } +} + void applyNativeVop3SourceModifiers(RdnaInstruction& instruction, std::uint32_t abs, std::uint32_t neg) { RdnaOperand* sources[] = {&instruction.source0, &instruction.source1, &instruction.source2}; for (std::uint32_t i = 0; i < instruction.sourceCount && i < 3u; ++i) { @@ -2233,6 +2273,11 @@ RdnaInstruction DecodeRdnaVop3(std::uint32_t programCounter, std::span #include "CacheKey.hpp" #include "CompiledVariant.hpp" +#include "VertexInputSpecialization.hpp" +#include "SpirvBackend/SpirvSpecialization.hpp" +#include "RdnaDecoder/RdnaDescriptorFormat.hpp" #include "ShaderDiskCache.hpp" #include +#include +#include #include #include #include @@ -86,7 +91,8 @@ std::uint32_t HostSubgroupSize(const RecompileRequest& request) { ShaderStageInputInfo RequestInputInfo(const RecompileRequest& request) { const auto* mesh = request.graphics && request.graphics->mesh ? &*request.graphics->mesh : nullptr; - return BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request), mesh); + const auto* tessellation = request.graphics && request.graphics->tessellation ? &*request.graphics->tessellation : nullptr; + return BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request), mesh, tessellation); } } @@ -99,7 +105,11 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request) { const auto decoded = decoder.Decode(request.shader.code); constexpr GraphBuilder graphBuilder; - auto cfg = graphBuilder.Build(decoded); + SwappcInfo swappcInfo; + swappcInfo.fetchCallAllowed = inputInfo.vertex != nullptr; + swappcInfo.userDataBaseRegister = request.context.userDataBaseRegister; + swappcInfo.userDataCount = static_cast(request.context.userData.size()); + auto cfg = graphBuilder.Build(decoded, &swappcInfo); constexpr Structurizer structurizer; structurizer.Structurize(cfg); @@ -110,19 +120,18 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request) { translateOptions.waveSize = request.context.waveSize; translateOptions.userDataBaseRegister = request.context.userDataBaseRegister; translateOptions.userDataCount = static_cast(request.context.userData.size()); - translateOptions.embeddedFetch = nullptr; + translateOptions.scratchDwords = request.context.compute.has_value() ? request.context.compute->scratchDwords : 0u; translateOptions.fragmentShaderBarycentricEnabled = request.target.fragmentShaderBarycentricEnabled; + translateOptions.floatMode = request.context.floatMode; translateOptions.inputInfo = inputInfo; constexpr InstructionTranslator translator; EmbeddedFetchPlan embeddedFetch; if ((stageKind == ShaderStageKind::Vertex || stageKind == ShaderStageKind::Local) && inputInfo.vertex != nullptr && inputInfo.vertex->fetchEmbedded) { - constexpr EmbeddedVertexFetchAnalyzer embeddedFetchAnalyzer; - embeddedFetch = embeddedFetchAnalyzer.Analyze(decoded, inputInfo.vertex->fetchAttribReg, inputInfo.vertex->fetchBufferReg, request.context.userDataBaseRegister, static_cast(request.context.userData.size()), request.context.waveSize); + embeddedFetch = EmbeddedVertexFetchAnalyzer{}.Analyze(decoded, inputInfo.vertex->fetchAttribReg, inputInfo.vertex->fetchBufferReg, request.context.userDataBaseRegister, static_cast(request.context.userData.size()), request.context.waveSize); + translateOptions.embeddedFetch = &embeddedFetch; } - translateOptions.embeddedFetch = embeddedFetch.loads.empty() ? nullptr : &embeddedFetch; - auto program = translator.Translate(decoded, cfg, translateOptions); // Debug aid: APS5_DUMP_IR= (or "all") prints the program after each front-end pass. const auto dumpIr = [&](const char* pass) { @@ -188,7 +197,6 @@ struct ResultMemoEntry { struct EmissionFailure { std::uint64_t codeAddress; BindingLayout layout; - ResourceSpecialization specialization; std::exception_ptr failure; }; @@ -247,6 +255,7 @@ std::shared_ptr getSource(const RecompileRequest& request) { struct SourceKeyStorage {}; auto& key = HostThreadLocal, SourceKeyStorage>(); RecompileCacheKey::Build(request, key); + key.push_back(HostSubgroupSize(request)); const auto find = [&]() -> std::shared_ptr { const auto found = sources.find(key); if (found == sources.end()) return nullptr; @@ -304,11 +313,12 @@ std::uint64_t nextVariantId() { return variants.fetch_add(1, std::memory_order_relaxed) + 1; } -CompiledVariant compileVariant(const RecompileRequest& request, IrProgram program, const ResourceSnapshot& resourceSnapshot, const ResourceSpecialization& resourceSpecialization, std::exception_ptr* emissionFailure = nullptr) { +CompiledVariant compileVariant(const RecompileRequest& request, IrProgram program, std::exception_ptr* emissionFailure = nullptr) try { const auto inputInfo = RequestInputInfo(request); constexpr DeadCodeEliminator deadCodeEliminator; constexpr ResourceMaterializer resourceMaterializer; - resourceMaterializer.Apply(program, resourceSpecialization); + const bool nativeSampleOffsets = request.target.nonConstantImageOffsets && std::find(request.target.supportedCapabilities.begin(), request.target.supportedCapabilities.end(), spv::CapabilityImageGatherExtended) != request.target.supportedCapabilities.end(); + resourceMaterializer.ApplyStaticInterface(program, nativeSampleOffsets); deadCodeEliminator.RemoveIdentities(program); deadCodeEliminator.Eliminate(program); @@ -319,9 +329,6 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra constexpr BindingAllocator bindingAllocator; auto bindings = bindingAllocator.Allocate(program, request.layout); - constexpr DescriptorBindingBuilder descriptorBindingBuilder; - descriptorBindingBuilder.Populate(bindings, program, resourceSnapshot, partialThreads(request)); - SpirvTargetOptions targetOptions {}; targetOptions.vulkanVersion = request.target.vulkanVersion; targetOptions.spirvVersion = request.target.spirvVersion; @@ -330,24 +337,21 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra targetOptions.supportedCapabilities = request.target.supportedCapabilities; targetOptions.supportedExtensions = request.target.supportedExtensions; targetOptions.nonConstantImageOffsets = request.target.nonConstantImageOffsets; + targetOptions.narrowSubgroupClock = request.target.narrowSubgroupClock; constexpr SpirvEmitter spirvEmitter; - RecompileResult result; + CompiledShaderArtifact result; result.variantId = nextVariantId(); - try { - result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions); -#if ANYPS5_ENABLE_SPIRV_TOOLS - result.spirv = ValidateAndOptimizeSpirv(result.spirv, request.target.vulkanVersion, request.target.spirvVersion, request.target.nonConstantImageOffsets); -#endif - } catch (const std::bad_alloc&) { - throw; - } catch (...) { - if (emissionFailure != nullptr) *emissionFailure = std::current_exception(); - throw; - } + result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions); - result.bdaAbiVersion = program.Info().usesDma ? request.target.bdaAbiVersion : 0u; + result.bdaAbiVersion = program.Info().usesDma || program.Info().usesFaultBuffer ? request.target.bdaAbiVersion : 0u; result.memoryOffsetDword = bindings.layout.memoryOffsetDword; + result.shaderDataDwords = bindings.layout.ShaderDataDwords(); + result.imageMetadataDword = bindings.layout.ImageMetadataDword(); + result.runtimeImageCount = bindings.layout.runtimeImageCount; + for (std::uint32_t index = 0; index < program.Info().images.size(); ++index) { + if (program.Info().images[index].indirectRoot != ImageResource::NoIndirectImage) result.runtimeImageResources.push_back(index); + } result.hostSubgroupSize = HostSubgroupSize(request); result.vertexOffsetSgpr = program.Info().vertexOffsetSgpr; result.instanceOffsetSgpr = program.Info().instanceOffsetSgpr; @@ -364,31 +368,319 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra for (const auto& input : program.Info().inputs) { if (input.kind != StageInputKind::Parameter) continue; if (input.location >= static_cast(inputInfo.vertex->resourcesNum)) throw std::runtime_error("vertex attribute location exceeds resource count"); - result.vertexAttributes.push_back({input.location, input.componentCount, {inputInfo.vertex->resources[input.location].fields}, inputInfo.vertex->resourcesDst[input.location].fetchIndex}); + result.vertexInputs.push_back({input.location, input.componentCount, inputInfo.vertex->resourcesDst[input.location].fetchIndex}); + if (inputInfo.vertex->fetchEmbedded) { + for (const auto& block : program.Blocks()) { + for (const auto* instruction : block->Instructions()) { + if (instruction->Opcode() == IrOpcode::GetAttribute && instruction->Argument(0)->Resolve()->ImmediateU32() == input.location) result.vertexInputs.back().outputMask |= 1u << instruction->Argument(1)->Resolve()->ImmediateU32(); + } + } + } } } - result.bindings.clear(); - result.pushConstants.clear(); - for (auto& attribute : result.vertexAttributes) attribute.resource = {}; - bindings.bindings.clear(); - bindings.pushConstants.clear(); - return {resourceSpecialization, request.layout, std::move(program).TakeCompiledInfo(), std::move(bindings), std::move(result)}; + const bool embedded = !result.vertexInputs.empty() && inputInfo.vertex != nullptr && inputInfo.vertex->fetchEmbedded; + if (embedded) PrepareVertexInputSpecialization(result); +#if ANYPS5_ENABLE_SPIRV_TOOLS + result.spirv = ValidateAndOptimizeSpirv(result.spirv, request.target.vulkanVersion, request.target.spirvVersion, request.target.nonConstantImageOffsets, !embedded); + if (embedded) { + std::array classes{}; + for (std::uint32_t kind = 1u; kind < 3u; ++kind) { + classes.fill(kind); + const auto specialized = SpecializeVertexInputTypes(result, classes); + static_cast(ValidateAndOptimizeSpirv(specialized, request.target.vulkanVersion, request.target.spirvVersion, request.target.nonConstantImageOffsets, false)); + } + } +#endif + + return {request.layout, std::move(program).TakeCompiledInfo(), std::move(static_cast(bindings)), std::move(result)}; +} catch (const std::bad_alloc&) { + throw; +} catch (...) { + if (emissionFailure != nullptr) *emissionFailure = std::current_exception(); + throw; +} + +struct SpecializedModule { + std::uint64_t specializationId = 0; + SharedSpirv spirv; + std::vector bindings; + bool pushData = false; +}; + +std::shared_ptr buildSpecializedModule(const CompiledShaderArtifact& artifact, std::span classes, std::span constants, const SpirvTarget& target) { + auto source = SpecializeVertexInputTypes(artifact, classes); + std::map supplied; + for (const auto& constant : constants) { + if (!supplied.emplace(constant.id, constant.value).second) throw std::runtime_error("duplicate prepared specialization ID"); + } + std::map values; + const auto& words = source.Words(); + if (words.size() < 5u || words[0] != spv::MagicNumber) throw std::runtime_error("invalid prepared specialization module"); + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + const auto op = static_cast(words[cursor] & 0xffffu); + if (count == 0u || count > words.size() - cursor) throw std::runtime_error("truncated prepared specialization instruction"); + if (op == spv::OpDecorate && count == 4u && words[cursor + 2u] == spv::DecorationSpecId) { + const auto found = supplied.find(words[cursor + 3u]); + if (found == supplied.end() || !values.emplace(words[cursor + 1u], found->second).second) throw std::runtime_error("missing or duplicate prepared specialization value"); + } + cursor += count; + } + std::vector materialized(words.begin(), words.begin() + 5); + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + const auto op = static_cast(words[cursor] & 0xffffu); + if (op == spv::OpSpecConstant) { + if (count != 4u || !values.contains(words[cursor + 2u])) throw std::runtime_error("invalid prepared specialization constant"); + materialized.insert(materialized.end(), {(4u << 16u) | spv::OpConstant, words[cursor + 1u], words[cursor + 2u], values.at(words[cursor + 2u])}); + } else if (!(op == spv::OpDecorate && count == 4u && words[cursor + 2u] == spv::DecorationSpecId)) { + materialized.insert(materialized.end(), words.begin() + cursor, words.begin() + cursor + count); + } + cursor += count; + } + materialized = SpecializeSpirv(materialized); +#if ANYPS5_ENABLE_SPIRV_TOOLS + materialized = ValidateAndOptimizeSpirv(materialized, target.vulkanVersion, target.spirvVersion, target.nonConstantImageOffsets, true, true); +#endif + std::map descriptorVariables; + std::set usedDescriptors; + for (std::size_t cursor = 5; cursor < materialized.size();) { + const auto count = materialized[cursor] >> 16u; + const auto op = static_cast(materialized[cursor] & 0xffffu); + if (op == spv::OpDecorate && count == 4u && materialized[cursor + 2u] == spv::DecorationBinding) descriptorVariables.emplace(materialized[cursor + 1u], materialized[cursor + 3u]); + if ((op == spv::OpAccessChain || op == spv::OpInBoundsAccessChain || op == spv::OpPtrAccessChain || op == spv::OpInBoundsPtrAccessChain || op == spv::OpLoad || op == spv::OpCopyObject) && count >= 4u) usedDescriptors.insert(materialized[cursor + 3u]); + cursor += count; + } + const auto unused = [&](std::uint32_t id) { return descriptorVariables.contains(id) && !usedDescriptors.contains(id); }; + std::vector compact(materialized.begin(), materialized.begin() + 5); + for (std::size_t cursor = 5; cursor < materialized.size();) { + const auto count = materialized[cursor] >> 16u; + const auto op = static_cast(materialized[cursor] & 0xffffu); + if (op == spv::OpEntryPoint) { + const auto start = compact.size(); + std::size_t interfaceIndex = 3u; + for (; interfaceIndex < count; ++interfaceIndex) { + const auto word = materialized[cursor + interfaceIndex]; + if ((word & 0xffu) == 0u || (word & 0xff00u) == 0u || (word & 0xff0000u) == 0u || (word & 0xff000000u) == 0u) { ++interfaceIndex; break; } + } + compact.insert(compact.end(), materialized.begin() + cursor, materialized.begin() + cursor + interfaceIndex); + for (auto index = interfaceIndex; index < count; ++index) if (!unused(materialized[cursor + index])) compact.push_back(materialized[cursor + index]); + compact[start] = (static_cast(compact.size() - start) << 16u) | spv::OpEntryPoint; + } else if (!((op == spv::OpVariable && unused(materialized[cursor + 2u])) || ((op == spv::OpDecorate || op == spv::OpName) && unused(materialized[cursor + 1u])))) { + compact.insert(compact.end(), materialized.begin() + cursor, materialized.begin() + cursor + count); + } + cursor += count; + } + materialized = std::move(compact); +#if ANYPS5_ENABLE_SPIRV_TOOLS + materialized = ValidateAndOptimizeSpirv(materialized, target.vulkanVersion, target.spirvVersion, target.nonConstantImageOffsets, false); +#endif + auto module = std::make_shared(); + std::map bindingNumbers; + for (std::size_t cursor = 5; cursor < materialized.size();) { + const auto count = materialized[cursor] >> 16u; + const auto op = static_cast(materialized[cursor] & 0xffffu); + if (op == spv::OpDecorate && count == 4u && materialized[cursor + 2u] == spv::DecorationBinding) bindingNumbers.emplace(materialized[cursor + 1u], materialized[cursor + 3u]); + if (op == spv::OpVariable && count >= 4u) { + if (const auto found = bindingNumbers.find(materialized[cursor + 2u]); found != bindingNumbers.end()) module->bindings.push_back(found->second); + module->pushData |= materialized[cursor + 3u] == spv::StorageClassPushConstant; + } + cursor += count; + } + module->spirv = std::move(materialized); + module->specializationId = constants.empty() ? 0u : nextVariantId(); + return module; +} + +struct SpecializedModuleEntry { + std::once_flag ready; + std::shared_ptr module; +}; + +struct PreparedModuleEntry { + std::once_flag ready; + std::shared_ptr module; + DescriptorBindingPlan bindings; +}; + +struct PreparedBindingPlan { + std::once_flag ready; + std::exception_ptr failure; + DescriptorBindingPlan bindings; + std::shared_mutex mutex; + std::map, std::shared_ptr> modules; +}; + +std::shared_ptr preparedBindingPlan(const CompiledVariant& variant, const ResourceSnapshot& snapshot, std::span exports) { + struct BindingPlanKeyStorage {}; + auto& key = HostThreadLocal, BindingPlanKeyStorage>(); + key.clear(); + key.push_back(variant.artifact.variantId); + for (std::size_t index = 0; index < variant.info.info.buffers.size(); ++index) { + const auto& descriptor = snapshot.buffers.at(index); + key.push_back(descriptor.dwordCount); + key.push_back(descriptor.dwords[1] & 0xffff0000u); + key.push_back(descriptor.dwords[3]); + const bool present = descriptor.dwords[2] != 0u && (descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffffu) != 0u); + key.push_back(present ? 4u | (descriptor.dwords[0] & 3u) : 0u); + } + for (std::size_t index = 0; index < variant.info.info.images.size(); ++index) { + const auto& descriptor = snapshot.images.at(index); + key.push_back(descriptor.dwordCount); + key.push_back(descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffu) != 0u); + key.push_back(descriptor.dwords[1] & ~0xffu); + for (std::size_t word = 2; word < 8; ++word) key.push_back(descriptor.dwords[word]); + } + for (std::size_t index = 0; index < variant.info.info.samplers.size(); ++index) { + const auto& descriptor = snapshot.samplers.at(index); + key.push_back(descriptor.dwordCount); + for (std::size_t word = 0; word < 4; ++word) key.push_back(descriptor.dwords[word]); + } + key.push_back(exports.size()); + for (const auto mapping : exports) key.push_back(mapping); + static std::shared_mutex mutex; + static std::map, std::shared_ptr> plans; + std::shared_ptr plan; + { + std::shared_lock lock(mutex); + if (const auto found = plans.find(key); found != plans.end()) plan = found->second; + } + if (plan == nullptr) { + std::unique_lock lock(mutex); + const auto found = plans.find(key); + plan = found != plans.end() ? found->second : plans.emplace(key, std::make_shared()).first->second; + } + std::call_once(plan->ready, [&] { + try { + plan->bindings = DescriptorBindingBuilder{}.Prepare(variant.bindings.layout, variant.info.info, variant.info.stage, snapshot, exports); + } catch (...) { + plan->failure = std::current_exception(); + } + }); + if (plan->failure) std::rethrow_exception(plan->failure); + return plan; +} + +std::shared_ptr specializeModule(const CompiledShaderArtifact& artifact, std::span classes, std::span constants, const SpirvTarget& target) { + struct ModuleKeyStorage {}; + auto& key = HostThreadLocal, ModuleKeyStorage>(); + key.clear(); + key.push_back(artifact.variantId); + for (const auto& constant : constants) key.push_back((static_cast(constant.id) << 32u) | constant.value); + static std::shared_mutex mutex; + static std::map, std::shared_ptr> modules; + std::shared_ptr entry; + { + std::shared_lock lock(mutex); + if (const auto found = modules.find(key); found != modules.end()) entry = found->second; + } + if (entry == nullptr) { + std::unique_lock lock(mutex); + const auto found = modules.find(key); + entry = found != modules.end() ? found->second : modules.emplace(key, std::make_shared()).first->second; + } + std::call_once(entry->ready, [&] { entry->module = buildSpecializedModule(artifact, classes, constants, target); }); + return entry->module; } RecompileResult materializeResult(const CompiledVariant& variant, const RecompileRequest& request, const ResourceSnapshot& snapshot) { - auto result = variant.result; + RecompileResult result; + const auto& artifact = variant.artifact; + static_cast(result) = { + .memoryOffsetDword = artifact.memoryOffsetDword, + .shaderDataDwords = artifact.shaderDataDwords, + .imageMetadataDword = artifact.imageMetadataDword, + .runtimeImageCount = artifact.runtimeImageCount, + .runtimeImageResources = artifact.runtimeImageResources, + .bdaAbiVersion = artifact.bdaAbiVersion, + .runtimeAbiVersion = artifact.runtimeAbiVersion, + .vertexInputs = artifact.vertexInputs, + .vertexOffsetSgpr = artifact.vertexOffsetSgpr, + .instanceOffsetSgpr = artifact.instanceOffsetSgpr, + .vertexOffsetShared = artifact.vertexOffsetShared, + .instanceOffsetShared = artifact.instanceOffsetShared, + .vertexOffsetConflict = artifact.vertexOffsetConflict, + .instanceOffsetConflict = artifact.instanceOffsetConflict, + .hostSubgroupSize = artifact.hostSubgroupSize, + .parameterExports = artifact.parameterExports, + .fragmentParameters = artifact.fragmentParameters, + .variantId = artifact.variantId + }; + const auto plan = preparedBindingPlan(variant, snapshot, request.context.pixel ? std::span(request.context.pixel->targetExportMapping) : std::span{}); + const DescriptorBindingPlan* bindingPlan = &plan->bindings; + std::shared_ptr entry; + struct LocalModuleKeyStorage {}; + auto& moduleKey = HostThreadLocal, LocalModuleKeyStorage>(); + moduleKey.clear(); + if (variant.bindings.layout.UsesPushData()) moduleKey.push_back(request.layout.pushConstantOffsetBytes / 4u); + result.vertexAttributes.reserve(result.vertexInputs.size()); + std::array vertexClasses{}; + for (const auto& input : result.vertexInputs) { + if (!request.context.vertex || input.location >= request.context.vertex->resourcesNum || input.location >= request.context.vertex->resources.size()) throw std::runtime_error("Shader cache: invalid vertex attribute metadata"); + const auto& vertex = *request.context.vertex; + const auto& resource = vertex.resources[input.location]; + const auto fetchIndex = vertex.fetchEmbedded ? vertex.resourcesDst[input.location].fetchIndex : input.fetchIndex; + result.vertexAttributes.push_back({input.location, input.components, resource, fetchIndex}); + if (!artifact.vertexInputPatches.empty()) { + const auto numeric = VertexInputNumericClass(static_cast((resource.fields[3] >> 12u) & 0x7fu)); + if (numeric == IrTextureNumericClass::Unsupported) throw std::runtime_error("unsupported prepared vertex format"); + const auto kind = numeric == IrTextureNumericClass::Float ? 0u : numeric == IrTextureNumericClass::Sint ? 1u : 2u; + vertexClasses.at(input.location) = kind; + std::uint32_t selectors = kind << 12u; + auto& formatComponents = result.vertexAttributes.back().formatComponents; + formatComponents = 1u; + for (std::uint32_t component = 0; component < 4u; ++component) { + const auto selector = (input.outputMask & (1u << component)) != 0u ? (resource.fields[3] >> (component * 3u)) & 7u : 0u; + if (selector == 2u || selector == 3u) throw std::runtime_error("reserved prepared vertex component selector"); + if (selector >= 4u) formatComponents = std::max(formatComponents, selector - 3u); + selectors |= selector << (component * 3u); + } + moduleKey.push_back(selectors); + } + } + if (!plan->bindings.specialization.empty() || variant.bindings.layout.UsesPushData() || !artifact.vertexInputPatches.empty()) { + { + std::shared_lock lock(plan->mutex); + if (const auto found = plan->modules.find(moduleKey); found != plan->modules.end()) entry = found->second; + } + if (entry == nullptr) { + std::unique_lock lock(plan->mutex); + const auto found = plan->modules.find(moduleKey); + entry = found != plan->modules.end() ? found->second : plan->modules.emplace(moduleKey, std::make_shared()).first->second; + } + std::call_once(entry->ready, [&] { + auto constants = plan->bindings.specialization; + std::size_t index = 0; + if (variant.bindings.layout.UsesPushData()) constants.push_back({PipelineSpecialization::PushDataOffset, moduleKey[index++]}); + if (!artifact.vertexInputPatches.empty()) { + for (const auto& input : result.vertexInputs) { + const auto selectors = moduleKey[index++]; + const auto first = PipelineSpecialization::VertexBase + input.location * PipelineSpecialization::VertexWords; + for (std::uint32_t component = 0; component < 4u; ++component) constants.push_back({first + component, (selectors >> (component * 3u)) & 7u}); + const auto kind = selectors >> 12u; + constants.push_back({first + 4u, kind == 0u ? 0x3f800000u : 1u}); + constants.push_back({first + 5u, kind}); + } + } + const auto module = specializeModule(variant.artifact, vertexClasses, constants, request.target); + if (variant.bindings.layout.UsesPushData() && !module->pushData) throw std::runtime_error("specialization removed the prepared push constant interface"); + auto selected = DescriptorBindingBuilder{}.Select(plan->bindings, module->bindings); + entry->bindings = std::move(selected); + entry->module = module; + }); + const auto& module = entry->module; + result.specializationId = module->specializationId; + result.spirv = module->spirv; + bindingPlan = &entry->bindings; + } else { + result.spirv = artifact.spirv; + } BindingAllocationResult bindings; - bindings.layout = variant.bindings.layout; - bindings.pushConstantOffsetBytes = variant.bindings.pushConstantOffsetBytes; - bindings.pushConstantSizeBytes = variant.bindings.pushConstantSizeBytes; - DescriptorBindingBuilder{}.Populate(bindings, variant.info.info, variant.info.stage, variant.info.userDataBase, snapshot, partialThreads(request)); + DescriptorBindingBuilder{}.Populate(bindings, variant.bindings, *bindingPlan, variant.info.userDataBase, snapshot, partialThreads(request)); result.bindings = std::move(bindings.bindings); result.pushConstants = std::move(bindings.pushConstants); - for (auto& attribute : result.vertexAttributes) { - if (!request.context.vertex || attribute.location >= request.context.vertex->resourcesNum) throw std::runtime_error("Shader cache: invalid vertex attribute metadata"); - attribute.resource = request.context.vertex->resources[attribute.location]; - } + result.poisonedSrtReads = static_cast(snapshot.srtPoison.size()); return result; } @@ -396,27 +688,26 @@ bool sameLayout(const BindingLayout& left, const BindingLayout& right) { return left.descriptorSet == right.descriptorSet && left.firstBinding == right.firstBinding && left.pushConstantOffsetBytes == right.pushConstantOffsetBytes && left.pushConstantSizeBytes == right.pushConstantSizeBytes; } -std::shared_ptr findOrCompileVariant(SourceEntry& source, const RecompileRequest& request, const ResourceSnapshot& snapshot, const ResourceSpecialization& specialization, bool& cacheHit) { +std::shared_ptr findOrCompileVariant(SourceEntry& source, const RecompileRequest& request, bool& cacheHit) { for (const auto& candidate : source.variants) { - if (sameLayout(candidate->layout, request.layout) && candidate->specialization == specialization) { + if (sameLayout(candidate->layout, request.layout)) { cacheHit = true; return candidate; } } for (const auto& failure : source.emissionFailures) { - if (failure.codeAddress == request.shader.codeAddress && sameLayout(failure.layout, request.layout) && failure.specialization == specialization) std::rethrow_exception(failure.failure); + if (failure.codeAddress == request.shader.codeAddress && sameLayout(failure.layout, request.layout)) std::rethrow_exception(failure.failure); } cacheHit = false; const bool disk = ShaderDiskCache::Enabled() && !DebugProbeActive(); std::vector diskKey; std::shared_ptr variant; if (disk) { - ShaderDiskCache::BuildKey(request, HostSubgroupSize(request), specialization, diskKey); + ShaderDiskCache::BuildKey(request, HostSubgroupSize(request), diskKey); CompiledVariant loaded; if (ShaderDiskCache::Load(diskKey, loaded)) { - loaded.specialization = specialization; loaded.layout = request.layout; - loaded.result.variantId = nextVariantId(); + loaded.artifact.variantId = nextVariantId(); variant = std::make_shared(std::move(loaded)); } } @@ -425,9 +716,9 @@ std::shared_ptr findOrCompileVariant(SourceEntry& source, source.program.reset(); std::exception_ptr emissionFailure; try { - variant = std::make_shared(compileVariant(request, std::move(program), snapshot, specialization, &emissionFailure)); + variant = std::make_shared(compileVariant(request, std::move(program), &emissionFailure)); } catch (...) { - if (emissionFailure != nullptr && FailureMemo()) source.emissionFailures.push_back({request.shader.codeAddress, request.layout, specialization, emissionFailure}); + if (emissionFailure != nullptr && FailureMemo()) source.emissionFailures.push_back({request.shader.codeAddress, request.layout, emissionFailure}); throw; } if (disk) ShaderDiskCache::Store(std::move(diskKey), variant); @@ -437,13 +728,12 @@ std::shared_ptr findOrCompileVariant(SourceEntry& source, return variant; } -// The cached variant of `source` for the specialization, compiled on first use. -RecompileResult materializeVariant(SourceEntry& source, const RecompileRequest& request, const ResourceSnapshot& snapshot, const ResourceSpecialization& specialization) { +RecompileResult materializeVariant(SourceEntry& source, const RecompileRequest& request, const ResourceSnapshot& snapshot) { std::shared_ptr variant; bool cacheHit = false; { std::lock_guard lock(source.mutex); - variant = findOrCompileVariant(source, request, snapshot, specialization, cacheHit); + variant = findOrCompileVariant(source, request, cacheHit); } auto result = materializeResult(*variant, request, snapshot); result.cacheHit = cacheHit; @@ -455,18 +745,17 @@ RecompileResult RecompileImpl(const RecompileRequest& request) { RequestMemoryView memory(request.context.memory); const auto runtime = memory.MakeRuntime(request.context.userData, request.shader.codeAddress); ResourceSnapshot snapshot; - ResourceSpecialization specialization; constexpr ResourceMaterializer materializer; if (!request.useCache) { auto program = PrepareResourceProgram(request); const auto plan = materializer.ExtractPlan(program); - materializer.Materialize(plan, runtime, snapshot, specialization); - const auto variant = compileVariant(request, std::move(program), snapshot, specialization); + materializer.Materialize(plan, runtime, snapshot); + const auto variant = compileVariant(request, std::move(program)); return materializeResult(variant, request, snapshot); } const auto source = getSource(request); - materializer.Materialize(*source->plan, runtime, snapshot, specialization); - return materializeVariant(*source, request, snapshot, specialization); + materializer.Materialize(*source->plan, runtime, snapshot); + return materializeVariant(*source, request, snapshot); } // APS5_NO_RESULT_MEMO=1: every Recompile(request, capture) materializes its own result as before. @@ -536,12 +825,17 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh mix(snapshot.uniformFill.words); mix(snapshot.uniformFill.value); for (const auto threads : partialThreads(request)) mix(threads); + mix(request.layout.pushConstantOffsetBytes); + if (request.context.pixel) { + for (const auto mapping : request.context.pixel->targetExportMapping) mix(mapping); + } if (request.context.vertex) { const auto& vertex = *request.context.vertex; const auto count = std::min(vertex.resourcesNum, ShaderVertexStageInfo::MaxResources); mix(count); for (std::uint32_t i = 0; i < count; ++i) { for (const auto field : vertex.resources[i].fields) mix(field); + mix(vertex.resourcesDst[i].fetchIndex); } } else { mix(1ull << 32u); @@ -552,19 +846,16 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh // The memo'd result of `source`'s variant for the snapshot (design13 R5): a hit returns the shared // object, a miss materializes outside the source mutex and inserts (a concurrent miss's object is // as good). `memoHit` reports the hit. -std::shared_ptr materializeMemoized(SourceEntry& source, const RecompileRequest& request, const ResourceSnapshot& snapshot, const ResourceSpecialization& specialization, bool* memoHit) { +std::shared_ptr materializePreparedMemoized(SourceEntry& source, const std::shared_ptr& variant, const RecompileRequest& request, const ResourceSnapshot& snapshot, bool cacheHit, bool* memoHit) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - std::shared_ptr variant; - bool cacheHit = false; const auto hash = snapshotHash(request, snapshot); std::uint64_t index = 0; auto& counters = resultMemoCounters(); { std::lock_guard lock(source.mutex); - variant = findOrCompileVariant(source, request, snapshot, specialization, cacheHit); - index = (variant->result.variantId * 0x9e3779b97f4a7c15ull) ^ hash; + index = (variant->artifact.variantId * 0x9e3779b97f4a7c15ull) ^ hash; const auto found = source.memoIndex.find(index); - if (found != source.memoIndex.end() && found->second->variantId == variant->result.variantId && found->second->hash == hash) { + if (found != source.memoIndex.end() && found->second->variantId == variant->artifact.variantId && found->second->hash == hash) { source.memo.splice(source.memo.begin(), source.memo, found->second); counters.hits.fetch_add(1, std::memory_order_relaxed); if (memoHit != nullptr) *memoHit = true; @@ -582,7 +873,7 @@ std::shared_ptr materializeMemoized(SourceEntry& source, std::lock_guard lock(source.mutex); const auto found = source.memoIndex.find(index); if (found != source.memoIndex.end()) { - if (found->second->variantId == variant->result.variantId && found->second->hash == hash) { + if (found->second->variantId == variant->artifact.variantId && found->second->hash == hash) { source.memo.splice(source.memo.begin(), source.memo, found->second); shared = found->second->result; } else { @@ -591,7 +882,7 @@ std::shared_ptr materializeMemoized(SourceEntry& source, } } if (source.memoIndex.find(index) == source.memoIndex.end()) { - source.memo.push_front({variant->result.variantId, hash, shared}); + source.memo.push_front({variant->artifact.variantId, hash, shared}); source.memoIndex.emplace(index, source.memo.begin()); while (source.memo.size() > ResultMemoEntries) { const auto& last = source.memo.back(); @@ -605,16 +896,26 @@ std::shared_ptr materializeMemoized(SourceEntry& source, return shared; } +std::shared_ptr materializeMemoized(SourceEntry& source, const RecompileRequest& request, const ResourceSnapshot& snapshot, bool* memoHit) { + std::shared_ptr variant; + bool cacheHit = false; + { + std::lock_guard lock(source.mutex); + variant = findOrCompileVariant(source, request, cacheHit); + } + return materializePreparedMemoized(source, variant, request, snapshot, cacheHit, memoHit); +} + // The capture already resolved the source entry (stage input validation included) and materialized // the request over exactly the words the driver captured, so neither is repeated here. std::shared_ptr RecompileImpl(const RecompileRequest& request, const ResourceCapture& capture, bool* memoHit) { if (!request.useCache || capture.source == nullptr) { auto program = PrepareResourceProgram(request); - const auto variant = compileVariant(request, std::move(program), capture.snapshot, capture.specialization); + const auto variant = compileVariant(request, std::move(program)); return std::make_shared(materializeResult(variant, request, capture.snapshot)); } - if (!ResultMemo()) return std::make_shared(materializeVariant(*capture.source, request, capture.snapshot, capture.specialization)); - return materializeMemoized(*capture.source, request, capture.snapshot, capture.specialization, memoHit); + if (!ResultMemo()) return std::make_shared(materializeVariant(*capture.source, request, capture.snapshot)); + return materializeMemoized(*capture.source, request, capture.snapshot, memoHit); } template @@ -645,12 +946,12 @@ namespace { void materializeCapture(ResourceCapture& capture, const SrtRuntime& runtime) { const auto& plan = *capture.plan; if (std::none_of(plan.pureFlatSlots.begin(), plan.pureFlatSlots.end(), [](std::uint8_t pure) { return pure != 0u; })) { - ResourceMaterializer{}.Materialize(plan, runtime, capture.snapshot, capture.specialization); + ResourceMaterializer{}.Materialize(plan, runtime, capture.snapshot); return; } SrtRuntime traced = runtime; traced.readTrace = &capture.readTrace; - ResourceMaterializer{}.Materialize(plan, traced, capture.snapshot, capture.specialization); + ResourceMaterializer{}.Materialize(plan, traced, capture.snapshot); auto& other = capture.readTrace.otherReads; std::sort(other.begin(), other.end()); other.erase(std::unique(other.begin(), other.end()), other.end()); @@ -681,7 +982,98 @@ std::shared_ptr ResolveSource(const RecompileRequest& reques return std::make_shared(SourceHandle{getSource(request)}); } +std::shared_ptr PrepareShader(const RecompileRequest& request) { + return recompileReporting(request, [&]() -> std::shared_ptr { + static_cast(RequestInputInfo(request)); + auto handle = std::make_shared(); + handle->source = getSource(request); + RecompileCacheKey::BuildInterface(request, handle->staticKey); + handle->staticKey.push_back(HostSubgroupSize(request)); + bool cacheHit = false; + std::lock_guard lock(handle->source->mutex); + handle->artifact = findOrCompileVariant(*handle->source, request, cacheHit); + return handle; + }); +} + +bool MatchesPreparedShader(const RecompileRequest& request, const SourceHandle& handle) { + struct PreparedKeyStorage {}; + auto& key = HostThreadLocal, PreparedKeyStorage>(); + BuildPreparedShaderKey(request, key); + return MatchesPreparedShader(request, handle, key); +} + +void BuildPreparedShaderKey(const RecompileRequest& request, std::vector& key) { + RecompileCacheKey::BuildInterface(request, key); + key.push_back(HostSubgroupSize(request)); +} + +bool MatchesPreparedShader(const RecompileRequest& request, const SourceHandle& handle, std::span key) { + if (handle.source == nullptr || handle.artifact == nullptr) return false; + const auto& layout = request.layout; + const auto& prepared = handle.artifact->bindings.layout; + if (layout.descriptorSet != handle.artifact->layout.descriptorSet || layout.firstBinding != handle.artifact->layout.firstBinding) return false; + if (layout.pushConstantOffsetBytes % 4u != 0u || layout.pushConstantSizeBytes % 4u != 0u || layout.pushConstantOffsetBytes > NativePushConstantSize || layout.pushConstantSizeBytes > NativePushConstantSize - layout.pushConstantOffsetBytes) return false; + const auto words = prepared.ShaderDataDwords(); + const bool usesPush = prepared.runtimeImageCount == 0u && words != 0u && words <= layout.pushConstantSizeBytes / 4u; + if (prepared.UsesPushData() != usesPush) return false; + if (!std::ranges::equal(key, handle.staticKey) || handle.source->code.size() != request.shader.code.size()) return false; + return handle.source->code.data() == request.shader.code.data() || std::ranges::equal(handle.source->code, request.shader.code); +} + +std::span GetPreparedCode(const SourceHandle& handle) { + if (handle.source == nullptr || handle.artifact == nullptr) throw std::runtime_error("ShaderRecompiler: prepared artifact is missing"); + return handle.source->code; +} + +PreparedShaderInvocation::PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle) : request(request), handle(handle) {} + +std::optional PreparedShaderInvocation::TryCreate(const RecompileRequest& request, const std::shared_ptr& handle) { + struct PreparedKeyStorage {}; + auto& key = HostThreadLocal, PreparedKeyStorage>(); + BuildPreparedShaderKey(request, key); + return TryCreate(request, handle, key); +} + +std::optional PreparedShaderInvocation::TryCreate(const RecompileRequest& request, const std::shared_ptr& handle, std::span key) { + if (handle == nullptr || !MatchesPreparedShader(request, *handle, key)) return std::nullopt; + if (request.shader.stage != ShaderStage::Compute && request.shader.stage != ShaderStage::Fragment) static_cast(RequestInputInfo(request)); + return PreparedShaderInvocation(request, handle); +} + +std::shared_ptr PreparedShaderInvocation::Capture(const SrtRuntime& runtime) const { + auto capture = std::make_shared(); + capture->source = handle->source; + capture->plan = handle->source->plan; + materializeCapture(*capture, runtime); + return capture; +} + +std::shared_ptr PreparedShaderInvocation::Materialize(const ResourceCapture& capture) const { + if (capture.source != handle->source || capture.plan != handle->source->plan) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); + if (request.useCache && ResultMemo()) return materializePreparedMemoized(*handle->source, handle->artifact, request, capture.snapshot, true, nullptr); + auto result = std::make_shared(materializeResult(*handle->artifact, request, capture.snapshot)); + result->cacheHit = true; + return result; +} + +const CompiledShaderArtifact& GetPreparedArtifact(const SourceHandle& handle) { + if (handle.artifact == nullptr) throw std::runtime_error("ShaderRecompiler: prepared artifact is missing"); + return handle.artifact->artifact; +} + +std::shared_ptr MaterializeShader(const RecompileRequest& request, const ResourceCapture& capture, const SourceHandle& handle) { + if (!MatchesPreparedShader(request, handle)) throw std::runtime_error("ShaderRecompiler: prepared artifact does not match the static ABI"); + if (capture.source != handle.source || capture.plan != handle.source->plan) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); + if (request.useCache && ResultMemo()) return materializePreparedMemoized(*handle.source, handle.artifact, request, capture.snapshot, true, nullptr); + auto result = std::make_shared(materializeResult(*handle.artifact, request, capture.snapshot)); + result->cacheHit = true; + return result; +} + std::shared_ptr CaptureResources(const RecompileRequest& request, const SrtRuntime& runtime, const SourceHandle& handle) { + if (handle.source == nullptr) throw std::runtime_error("ShaderRecompiler: source handle is missing"); + if (handle.artifact != nullptr && !MatchesPreparedShader(request, handle)) throw std::runtime_error("ShaderRecompiler: prepared artifact does not match the static ABI"); static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; const auto started = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; // The whole vertex family (Vertex, Local, TC, TE, Mesh) validates V# fields the memo key does not cover. diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 7eddc2813..603d7b937 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -1,6 +1,8 @@ #ifndef CORE_SHADER_RECOMPILIER_INCLUDE_SHADER_RECOMPILIER_RECOMPILER_HPP #define CORE_SHADER_RECOMPILIER_INCLUDE_SHADER_RECOMPILIER_RECOMPILER_HPP +#include "RuntimeAbi.hpp" +#include "PipelineSpecialization.hpp" #include #include #include @@ -44,6 +46,7 @@ struct ShaderComputeStageInfo { bool tgSizeEnable; std::uint32_t threadIdComponentCount; std::array partialThreads; + std::uint32_t scratchDwords = 0; [[nodiscard]] bool PartialGroups() const { return partialThreads != std::array{}; @@ -126,6 +129,7 @@ struct ShaderPixelStageInfo { bool earlyZ; bool executeOnNoop; ConservativeZExport conservativeZExport; + bool orderedPixelShader; std::array targetOutputMode; std::array targetExportMapping; }; @@ -151,6 +155,19 @@ struct ShaderVertexStageInfo { bool fetchEmbedded; }; +struct ShaderFloatMode { + std::uint32_t floatMode = 0; + bool dx10Clamp = false; + bool ieeeMode = false; + bool fp16Overflow = false; + + bool operator==(const ShaderFloatMode&) const = default; +}; + +inline constexpr std::uint32_t InterpolationQuiet = 1u; +inline constexpr std::uint32_t InterpolationFlush32 = 2u; +inline constexpr std::uint32_t InterpolationFlush16 = 4u; + struct GuestContext { std::uint32_t waveSize; std::uint32_t userDataBaseRegister; @@ -159,6 +176,7 @@ struct GuestContext { std::optional pixel; std::optional vertex; std::span memory; + std::optional floatMode; }; struct MeshTargetLimits { @@ -198,6 +216,7 @@ struct SpirvTarget { std::optional tessellation; bool nonConstantImageOffsets = false; std::uint32_t srgbDecodeFormats = 0; + bool narrowSubgroupClock = false; }; struct BindingLayout { @@ -341,6 +360,7 @@ struct VertexAttribute { std::uint32_t components; ShaderVertexBufferResource resource; std::uint32_t fetchIndex; + std::uint32_t formatComponents = 0; }; struct FragmentParameter { @@ -396,17 +416,36 @@ private: std::shared_ptr> words; }; -struct RecompileResult { +struct VertexInput { + std::uint32_t location; + std::uint32_t components; + std::uint32_t fetchIndex; + std::uint32_t outputMask = 0; + + bool operator==(const VertexInput& other) const = default; +}; + +struct VertexInputPatch { + std::uint32_t location; + std::uint32_t word; + std::array values; + + bool operator==(const VertexInputPatch& other) const = default; +}; + +struct CompiledShaderArtifact { + std::vector vertexInputPatches; SharedSpirv spirv; - std::vector bindings; - std::vector pushConstants; std::uint32_t memoryOffsetDword = 0; + std::uint32_t shaderDataDwords = 0; + std::uint32_t imageMetadataDword = 0; + std::uint32_t runtimeImageCount = 0; + std::vector runtimeImageResources; std::uint32_t bdaAbiVersion = 0; - std::vector vertexAttributes; + std::uint32_t runtimeAbiVersion = RuntimeAbi::Version; + std::vector vertexInputs; std::int32_t vertexOffsetSgpr = -1; std::int32_t instanceOffsetSgpr = -1; - // The offset SGPR is also read elsewhere in the program (so a value folded into the draw's - // first vertex / instance cannot stand in for it), or two SGPRs were added (the SGPR is -1). bool vertexOffsetShared = false; bool instanceOffsetShared = false; bool vertexOffsetConflict = false; @@ -414,20 +453,25 @@ struct RecompileResult { std::uint32_t hostSubgroupSize = 0; std::vector parameterExports; std::vector fragmentParameters; - bool cacheHit = false; - // Identifies the compiled variant the result came from: equal ids mean identical SPIR-V and - // bindings, so drivers can reuse pipeline objects. Zero when unknown. std::uint64_t variantId = 0; }; +struct ShaderInvocation { + std::vector specialization; + std::uint64_t specializationId = 0; + std::vector bindings; + std::vector pushConstants; + std::vector vertexAttributes; + std::uint32_t poisonedSrtReads = 0; +}; + +struct RecompileResult : CompiledShaderArtifact, ShaderInvocation { + bool cacheHit = false; + [[nodiscard]] std::uint64_t PipelineVariantId() const { return specializationId != 0 ? specializationId : variantId; } +}; + [[nodiscard]] RecompileResult Recompile(const RecompileRequest& request); -// The resource plan, snapshot and specialization a driver captured for the request (see -// CaptureResources in Optimization/ResourceProgram.hpp): this overload reuses them instead of -// materializing the request's memory regions again, and is otherwise Recompile(request). The -// result is immutable and shared: a capture that reproduces a snapshot the source's variant was -// materialized over before receives the same object (`memoHit`), so the descriptor population runs -// once per distinct snapshot. APS5_NO_RESULT_MEMO=1 materializes every call. struct ResourceCapture; [[nodiscard]] std::shared_ptr Recompile(const RecompileRequest& request, const ResourceCapture& capture, bool* memoHit = nullptr); diff --git a/core/shader/recompiler/RuntimeAbi.hpp b/core/shader/recompiler/RuntimeAbi.hpp new file mode 100644 index 000000000..66f7f8a2f --- /dev/null +++ b/core/shader/recompiler/RuntimeAbi.hpp @@ -0,0 +1,98 @@ +#ifndef CORE_SHADER_RECOMPILER_RUNTIMEABI_HPP +#define CORE_SHADER_RECOMPILER_RUNTIMEABI_HPP + +#include +#include +#include +#include +#include + +namespace ShaderRecompiler::RuntimeAbi { + +inline constexpr std::uint32_t Version = 10u; +inline constexpr std::uint32_t DescriptorSet = 0u; +inline constexpr std::uint32_t StageCount = 4u; +inline constexpr std::uint32_t PushConstantDwords = 32u; +inline constexpr std::uint32_t FirstImageBinding = 1u; +inline constexpr std::uint32_t FirstComparisonImageBinding = 22u; +inline constexpr std::uint32_t FirstStorageImageBinding = 29u; +inline constexpr std::uint32_t ImageBindingCount = 56u; + +enum class Binding : std::uint32_t { + Buffers = 0u, + Samplers = 57u, + Gds = 58u, + BdaPagetable = 59u, + FaultBuffer = 60u, + FlattenedSrt = 61u, + ShaderData = 62u, + Count = 63u +}; + +enum class Stage : std::uint32_t { Main, Fragment, TessellationControl, TessellationEvaluation }; + +inline constexpr std::uint32_t UserDataCapacity = 128u; +inline constexpr std::uint32_t BufferCapacity = 128u; +inline constexpr std::uint32_t ImageCapacity = 256u; +inline constexpr std::uint32_t SampledHeapCapacity = 16u; +inline constexpr std::uint32_t StorageHeapCapacity = 4u; +inline constexpr std::uint32_t SamplerHeapCapacity = 16u; + +struct ResourceMetadata { + std::uint32_t binding; + std::uint32_t firstElement; + std::uint32_t elementCount; + std::uint32_t flags; + std::array descriptor; +}; + +struct ShaderData { + std::uint32_t version; + std::uint32_t imageCount; + std::uint32_t samplerCount; + std::uint32_t reserved; + std::array userData; + std::array bufferOffsets; + std::array dispatchThreadLimit; + std::array images; + std::array samplers; + std::array exportMappings; +}; + +inline constexpr std::uint32_t UserDataDword = offsetof(ShaderData, userData) / sizeof(std::uint32_t); +inline constexpr std::uint32_t BufferOffsetsDword = offsetof(ShaderData, bufferOffsets) / sizeof(std::uint32_t); +inline constexpr std::uint32_t DispatchThreadLimitDword = offsetof(ShaderData, dispatchThreadLimit) / sizeof(std::uint32_t); +inline constexpr std::uint32_t ExportMappingsDword = offsetof(ShaderData, exportMappings) / sizeof(std::uint32_t); +inline constexpr std::uint32_t ShaderDataDwords = sizeof(ShaderData) / sizeof(std::uint32_t); + +inline std::uint32_t HeapCapacity(Binding binding) { + const auto value = static_cast(binding); + if (value >= FirstImageBinding && value < FirstStorageImageBinding) return SampledHeapCapacity; + if (value >= FirstStorageImageBinding && value < static_cast(Binding::Samplers)) return StorageHeapCapacity; + if (binding == Binding::Samplers) return SamplerHeapCapacity; + throw std::runtime_error("Shader runtime ABI: binding is not a typed heap"); +} + +static_assert(std::is_standard_layout_v && std::is_trivially_copyable_v && sizeof(ResourceMetadata) == 48u); +static_assert(std::is_standard_layout_v && std::is_trivially_copyable_v && sizeof(ShaderData) == 13760u); +static_assert(UserDataDword == 4u && BufferOffsetsDword == 132u && DispatchThreadLimitDword == 164u); +static_assert(ExportMappingsDword == 3432u); +static_assert(offsetof(ResourceMetadata, descriptor) == 16u && offsetof(ShaderData, images) == 672u && offsetof(ShaderData, samplers) == 12960u); + +inline void RequireVersion(std::uint32_t version) { + if (version != Version) throw std::runtime_error("Shader runtime ABI: incompatible version"); +} + +inline std::uint32_t BindingNumber(Stage stage, Binding binding) { + const auto group = static_cast(stage); + const auto index = static_cast(binding); + const auto count = static_cast(Binding::Count); + if (group >= StageCount || index >= count) throw std::runtime_error("Shader runtime ABI: invalid stage or binding"); + return group * count + index; +} + +static_assert(FirstImageBinding + ImageBindingCount == static_cast(Binding::Samplers)); + +} + +#endif diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index ed87d113e..40f10ab42 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -2,6 +2,7 @@ #include "CacheKey.hpp" #include "ShaderCacheDirectory.hpp" #include "ThreadOwned.hpp" +#include "Optimization/ResourceMaterializer.hpp" #ifdef _WIN32 #ifndef NOMINMAX #define NOMINMAX @@ -63,24 +64,25 @@ std::filesystem::path ShaderCacheDirectory() { namespace ShaderRecompiler::ShaderDiskCache { #if defined(__linux__) && defined(__x86_64__) && defined(__GLIBCXX__) -static_assert(sizeof(RecompileResult) == 176, "RecompileResult changed: update EncodeResult and DecodeResult"); +static_assert(sizeof(CompiledShaderArtifact) == 184, "CompiledShaderArtifact changed: update the artifact encoder"); +static_assert(sizeof(ShaderInvocation) == 112, "ShaderInvocation changed: update the invocation encoder"); +static_assert(sizeof(RecompileResult) == 296, "RecompileResult changed: update EncodeResult and DecodeResult"); static_assert(sizeof(DescriptorBinding) == 448, "DescriptorBinding changed: update the binding encoder"); -static_assert(sizeof(VertexAttribute) == 28, "VertexAttribute changed: update the attribute encoder"); +static_assert(sizeof(VertexAttribute) == 32, "VertexAttribute changed: update the attribute encoder"); +static_assert(sizeof(VertexInput) == 16, "VertexInput changed: update the vertex input encoder"); static_assert(sizeof(FragmentParameter) == 12, "FragmentParameter changed: update the parameter encoder"); -static_assert(sizeof(CompiledShaderInfo) == 304, "CompiledShaderInfo changed: update the info encoder"); -static_assert(sizeof(ShaderInfo) == 200, "ShaderInfo changed: update the info encoder"); -static_assert(sizeof(BufferResource) == 36, "BufferResource changed: update the info encoder"); -static_assert(sizeof(ImageResource) == 96, "ImageResource changed: update the info encoder"); -static_assert(sizeof(SamplerResource) == 12, "SamplerResource changed: update the info encoder"); +static_assert(sizeof(CompiledShaderInfo) == 344, "CompiledShaderInfo changed: update the info encoder"); +static_assert(sizeof(ShaderInfo) == 232, "ShaderInfo changed: update the info encoder"); +static_assert(sizeof(BufferResource) == 32, "BufferResource changed: update the info encoder"); +static_assert(sizeof(ImageResource) == 112, "ImageResource changed: update the info encoder"); +static_assert(sizeof(SamplerResource) == 16, "SamplerResource changed: update the info encoder"); static_assert(sizeof(SampledResourcePair) == 12, "SampledResourcePair changed: update the info encoder"); static_assert(sizeof(StageInput) == 56, "StageInput changed: update the info encoder"); static_assert(sizeof(StageOutput) == 48, "StageOutput changed: update the info encoder"); -static_assert(sizeof(IrBindingLayout) == 64, "IrBindingLayout changed: update the layout encoder"); +static_assert(sizeof(IrBindingLayout) == 72, "IrBindingLayout changed: update the layout encoder"); static_assert(sizeof(IrDescriptorBinding) == 32, "IrDescriptorBinding changed: update the layout encoder"); -static_assert(sizeof(BindingAllocationResult) == 120, "BindingAllocationResult changed: update the allocation encoder"); -static_assert(sizeof(ResourceSpecialization) == 72, "ResourceSpecialization changed: update BuildKey"); -static_assert(sizeof(ResourceSpecialization::Buffer) == 16, "ResourceSpecialization::Buffer changed: update BuildKey"); -static_assert(sizeof(ResourceSpecialization::Image) == 48, "ResourceSpecialization::Image changed: update BuildKey"); +static_assert(sizeof(BindingAllocationResult) == 152, "BindingAllocationResult changed: update the allocation encoder"); +static_assert(sizeof(CompiledBindingLayout) == 80, "CompiledBindingLayout changed: update the allocation encoder"); static_assert(sizeof(BindingLayout) == 16, "BindingLayout changed: update BuildKey"); #endif @@ -280,18 +282,25 @@ void decodeBinding(Reader& reader, DescriptorBinding& binding) { reader.Values(binding.imageSamplers); } -void encodeResult(Writer& writer, const RecompileResult& result) { +void encodeArtifact(Writer& writer, const CompiledShaderArtifact& result) { + writer.Value(result.runtimeAbiVersion); writer.Values(std::span(result.spirv.Words())); - writer.List(result.bindings, encodeBinding); - writer.Value(result.pushConstants.size()); - for (const auto byte : result.pushConstants) writer.Value(static_cast(byte)); writer.Value(result.bdaAbiVersion); writer.Value(result.memoryOffsetDword); - writer.List(result.vertexAttributes, [](Writer& out, const VertexAttribute& attribute) { + writer.Value(result.shaderDataDwords); + writer.Value(result.imageMetadataDword); + writer.Value(result.runtimeImageCount); + writer.Values(std::span(result.runtimeImageResources)); + writer.List(result.vertexInputPatches, [](Writer& out, const VertexInputPatch& patch) { + out.Value(patch.location); + out.Value(patch.word); + for (const auto value : patch.values) out.Value(value); + }); + writer.List(result.vertexInputs, [](Writer& out, const VertexInput& attribute) { out.Value(attribute.location); out.Value(attribute.components); - for (const auto field : attribute.resource.fields) out.Value(field); out.Value(attribute.fetchIndex); + out.Value(attribute.outputMask); }); writer.Value(result.vertexOffsetSgpr); writer.Value(result.instanceOffsetSgpr); @@ -310,22 +319,27 @@ void encodeResult(Writer& writer, const RecompileResult& result) { }); } -void decodeResult(Reader& reader, RecompileResult& result) { +void decodeArtifact(Reader& reader, CompiledShaderArtifact& result) { + reader.Value(result.runtimeAbiVersion); std::vector words; reader.Values(words); result.spirv = std::move(words); - reader.List(result.bindings, 8, decodeBinding); - std::vector pushConstants; - reader.Values(pushConstants); - result.pushConstants.resize(pushConstants.size()); - if (!pushConstants.empty()) std::memcpy(result.pushConstants.data(), pushConstants.data(), pushConstants.size()); reader.Value(result.bdaAbiVersion); reader.Value(result.memoryOffsetDword); - reader.List(result.vertexAttributes, 28, [](Reader& in, VertexAttribute& attribute) { + reader.Value(result.shaderDataDwords); + reader.Value(result.imageMetadataDword); + reader.Value(result.runtimeImageCount); + reader.Values(result.runtimeImageResources); + reader.List(result.vertexInputPatches, 20, [](Reader& in, VertexInputPatch& patch) { + in.Value(patch.location); + in.Value(patch.word); + for (auto& value : patch.values) in.Value(value); + }); + reader.List(result.vertexInputs, 16, [](Reader& in, VertexInput& attribute) { in.Value(attribute.location); in.Value(attribute.components); - for (auto& field : attribute.resource.fields) in.Value(field); in.Value(attribute.fetchIndex); + in.Value(attribute.outputMask); }); reader.Value(result.vertexOffsetSgpr); reader.Value(result.instanceOffsetSgpr); @@ -342,15 +356,54 @@ void decodeResult(Reader& reader, RecompileResult& result) { in.Value(parameter.perVertex); in.Value(parameter.custom); }); - result.cacheHit = false; result.variantId = 0; } +void encodeInvocation(Writer& writer, const ShaderInvocation& invocation) { + writer.List(invocation.specialization, [](Writer& out, const PipelineSpecializationConstant& constant) { + out.Value(constant.id); + out.Value(constant.value); + }); + writer.List(invocation.bindings, encodeBinding); + writer.Value(invocation.pushConstants.size()); + for (const auto byte : invocation.pushConstants) writer.Value(static_cast(byte)); + writer.List(invocation.vertexAttributes, [](Writer& out, const VertexAttribute& attribute) { + out.Value(attribute.location); + out.Value(attribute.components); + for (const auto field : attribute.resource.fields) out.Value(field); + out.Value(attribute.fetchIndex); + out.Value(attribute.formatComponents); + }); + writer.Value(invocation.poisonedSrtReads); +} + +void decodeInvocation(Reader& reader, ShaderInvocation& invocation) { + invocation.specializationId = 0; + reader.List(invocation.specialization, 8, [](Reader& in, PipelineSpecializationConstant& constant) { + in.Value(constant.id); + in.Value(constant.value); + }); + reader.List(invocation.bindings, 8, decodeBinding); + std::vector pushConstants; + reader.Values(pushConstants); + invocation.pushConstants.resize(pushConstants.size()); + if (!pushConstants.empty()) std::memcpy(invocation.pushConstants.data(), pushConstants.data(), pushConstants.size()); + reader.List(invocation.vertexAttributes, 32, [](Reader& in, VertexAttribute& attribute) { + in.Value(attribute.location); + in.Value(attribute.components); + for (auto& field : attribute.resource.fields) in.Value(field); + in.Value(attribute.fetchIndex); + in.Value(attribute.formatComponents); + }); + reader.Value(invocation.poisonedSrtReads); +} + void encodeLayout(Writer& writer, const IrBindingLayout& layout) { writer.Value(layout.pushDataStartDword); writer.Value(layout.memoryOffsetDword); writer.Value(layout.memoryOffsetCount); writer.Value(layout.dispatchThreadLimit); + writer.Value(layout.runtimeImageCount); writer.Values(std::span(layout.userDataRegisters)); writer.List(layout.descriptors, [](Writer& out, const IrDescriptorBinding& descriptor) { out.Value(descriptor.kind); @@ -363,6 +416,7 @@ void decodeLayout(Reader& reader, IrBindingLayout& layout) { reader.Value(layout.memoryOffsetDword); reader.Value(layout.memoryOffsetCount); reader.Value(layout.dispatchThreadLimit); + reader.Value(layout.runtimeImageCount); reader.Values(layout.userDataRegisters); reader.List(layout.descriptors, 12, [](Reader& in, IrDescriptorBinding& descriptor) { in.Value(descriptor.kind); @@ -385,16 +439,15 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { out.Value(buffer.source); out.Value(buffer.firstUsePc); out.Value(buffer.maxByteExtent); - out.Value(buffer.packedStride); - out.Value(buffer.descriptorFormat); - out.Value(buffer.descriptorSwizzle); out.Value(buffer.imageAlias); out.Value(buffer.read); out.Value(buffer.written); out.Value(buffer.atomic); out.Value(buffer.formatted); + out.Value(buffer.descriptorFormatted); + out.Value(buffer.formattedReadMask); out.Value(buffer.scalar); - out.Value(buffer.empty); + out.Value(buffer.typedAlignment); }); writer.List(info.images, [](Writer& out, const ImageResource& image) { out.Value(image.source); @@ -414,9 +467,15 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { out.Value(image.cube); out.Value(image.r128); out.Value(image.srgbDecode); + out.Value(image.srgbDecodeCompatible); + out.Value(image.srgbDecodeFormats); out.Value(image.depthBits); out.Value(image.depthUnorm16); out.Value(image.packed); + out.Value(image.fmaskCompatible); + out.Value(image.depthBitsCompatible); + out.Value(image.byElements); + out.Value(image.byComponents); out.Value(image.packedFormat); out.Value(image.emulatedCompare); out.Value(image.indirectRoot); @@ -424,9 +483,11 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { out.Value(image.indirectSearchIterations); out.Values(std::span(image.indirectResources)); }); + writer.Value(!info.runtimeImageModes.empty()); writer.List(info.samplers, [](Writer& out, const SamplerResource& sampler) { out.Value(sampler.source); out.Value(sampler.firstUsePc); + out.Value(sampler.copyOf); out.Value(sampler.forcePointFiltering); out.Value(sampler.depthCompare); out.Value(sampler.uses); @@ -459,6 +520,7 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { writer.Value(info.hasBitwiseXor); writer.Value(info.usesDma); writer.Value(info.bdaWrites); + writer.Value(info.usesFaultBuffer); writer.Value(info.dispatchThreadLimit); encodeLayout(writer, compiled.bindings); } @@ -474,22 +536,21 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { auto& info = compiled.info; reader.Value(info.scratchDwords); reader.Value(info.sharedMemoryBytes); - reader.List(info.buffers, 34, [](Reader& in, BufferResource& buffer) { + reader.List(info.buffers, 27, [](Reader& in, BufferResource& buffer) { in.Value(buffer.source); in.Value(buffer.firstUsePc); in.Value(buffer.maxByteExtent); - in.Value(buffer.packedStride); - in.Value(buffer.descriptorFormat); - in.Value(buffer.descriptorSwizzle); in.Value(buffer.imageAlias); in.Value(buffer.read); in.Value(buffer.written); in.Value(buffer.atomic); in.Value(buffer.formatted); + in.Value(buffer.descriptorFormatted); + in.Value(buffer.formattedReadMask); in.Value(buffer.scalar); - in.Value(buffer.empty); + in.Value(buffer.typedAlignment); }); - reader.List(info.images, 63, [](Reader& in, ImageResource& image) { + reader.List(info.images, 72, [](Reader& in, ImageResource& image) { in.Value(image.source); in.Value(image.firstUsePc); in.Value(image.resourceClass); @@ -507,9 +568,15 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { in.Value(image.cube); in.Value(image.r128); in.Value(image.srgbDecode); + in.Value(image.srgbDecodeCompatible); + in.Value(image.srgbDecodeFormats); in.Value(image.depthBits); in.Value(image.depthUnorm16); in.Value(image.packed); + in.Value(image.fmaskCompatible); + in.Value(image.depthBitsCompatible); + in.Value(image.byElements); + in.Value(image.byComponents); in.Value(image.packedFormat); in.Value(image.emulatedCompare); in.Value(image.indirectRoot); @@ -517,9 +584,13 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { in.Value(image.indirectSearchIterations); in.Values(image.indirectResources); }); - reader.List(info.samplers, 11, [](Reader& in, SamplerResource& sampler) { + bool preparedImageModes = false; + reader.Value(preparedImageModes); + if (preparedImageModes) ResourceMaterializer::PrepareImageModes(info); + reader.List(info.samplers, 15, [](Reader& in, SamplerResource& sampler) { in.Value(sampler.source); in.Value(sampler.firstUsePc); + in.Value(sampler.copyOf); in.Value(sampler.forcePointFiltering); in.Value(sampler.depthCompare); in.Value(sampler.uses); @@ -552,28 +623,21 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { reader.Value(info.hasBitwiseXor); reader.Value(info.usesDma); reader.Value(info.bdaWrites); + reader.Value(info.usesFaultBuffer); reader.Value(info.dispatchThreadLimit); decodeLayout(reader, compiled.bindings); } -void encodeAllocation(Writer& writer, const BindingAllocationResult& allocation) { - writer.List(allocation.bindings, encodeBinding); +void encodeAllocation(Writer& writer, const CompiledBindingLayout& allocation) { encodeLayout(writer, allocation.layout); writer.Value(allocation.pushConstantOffsetBytes); writer.Value(allocation.pushConstantSizeBytes); - writer.Value(allocation.pushConstants.size()); - for (const auto byte : allocation.pushConstants) writer.Value(static_cast(byte)); } -void decodeAllocation(Reader& reader, BindingAllocationResult& allocation) { - reader.List(allocation.bindings, 8, decodeBinding); +void decodeAllocation(Reader& reader, CompiledBindingLayout& allocation) { decodeLayout(reader, allocation.layout); reader.Value(allocation.pushConstantOffsetBytes); reader.Value(allocation.pushConstantSizeBytes); - std::vector pushConstants; - reader.Values(pushConstants); - allocation.pushConstants.resize(pushConstants.size()); - if (!pushConstants.empty()) std::memcpy(allocation.pushConstants.data(), pushConstants.data(), pushConstants.size()); } constexpr std::string_view NeutralSwitches[] = { @@ -792,7 +856,7 @@ std::uint64_t SourceVersion() { return Generated::SourceVersion; } -void BuildKey(const RecompileRequest& request, std::uint32_t hostSubgroupSize, const ResourceSpecialization& specialization, std::vector& key) { +void BuildKey(const RecompileRequest& request, std::uint32_t hostSubgroupSize, std::vector& key) { key.clear(); Writer writer(key); writer.Value(FileMagic); @@ -808,30 +872,6 @@ void BuildKey(const RecompileRequest& request, std::uint32_t hostSubgroupSize, c writer.Value(request.layout.firstBinding); writer.Value(request.layout.pushConstantOffsetBytes); writer.Value(request.layout.pushConstantSizeBytes); - writer.List(specialization.buffers, [](Writer& out, const ResourceSpecialization::Buffer& buffer) { - out.Value(buffer.packedStride); - out.Value(buffer.descriptorFormat); - out.Value(buffer.descriptorSwizzle); - out.Value(buffer.empty); - }); - writer.List(specialization.images, [](Writer& out, const ResourceSpecialization::Image& image) { - out.Value(image.numericClass); - out.Value(image.dimension); - out.Value(image.mipCount); - out.Value(image.conversionFormat); - out.Value(image.shaderSwizzle); - out.Value(image.indirectRoot); - out.Value(image.indirectMappingOffset); - out.Value(image.indirectSearchIterations); - out.Value(image.cube); - out.Value(image.fmask); - out.Value(image.depthBits); - out.Value(image.depthUnorm16); - out.Value(image.packedFormat); - out.Value(image.emulatedCompare); - out.Value(image.srgbDecode); - }); - writer.Values(std::span(specialization.boundDescriptors)); const auto& switches = switchKey(); key.insert(key.end(), switches.begin(), switches.end()); } @@ -842,20 +882,23 @@ std::string EntryName(std::span key) { void EncodeResult(const RecompileResult& result, std::vector& out) { Writer writer(out); - encodeResult(writer, result); + encodeArtifact(writer, result); + encodeInvocation(writer, result); } bool DecodeResult(std::span bytes, RecompileResult& result) { Reader reader(bytes); - decodeResult(reader, result); - return reader.Done(); + decodeArtifact(reader, result); + decodeInvocation(reader, result); + result.cacheHit = false; + return reader.Done() && result.runtimeAbiVersion == RuntimeAbi::Version; } std::vector EncodeEntry(std::span key, const CompiledVariant& variant) { std::vector payload; - payload.reserve(variant.result.spirv.size() * sizeof(std::uint32_t) + 4096); + payload.reserve(variant.artifact.spirv.size() * sizeof(std::uint32_t) + 4096); Writer writer(payload); - encodeResult(writer, variant.result); + encodeArtifact(writer, variant.artifact); encodeInfo(writer, variant.info); encodeAllocation(writer, variant.bindings); const FileHeader header{FileMagic, FormatVersion, SourceVersion(), key.size(), payload.size(), HashBytes(key), HashBytes(payload)}; @@ -880,13 +923,13 @@ LoadStatus DecodeEntry(std::span file, std::span& key); +void BuildKey(const RecompileRequest& request, std::uint32_t hostSubgroupSize, std::vector& key); [[nodiscard]] std::string EntryName(std::span key); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp index a1323899b..501ef58fc 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp @@ -16,6 +16,7 @@ std::uint32_t BdaLoadAddress(SpirvEmitterState& state, std::uint32_t index); void DefineBdaFaultFunction(SpirvEmitterState& state); void DefineBdaDwordReadFunctions(SpirvEmitterState& state); void RecordBdaFault(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); +void RecordBdaFaultWords(SpirvEmitterState& state, std::uint32_t addressLow, std::uint32_t addressHigh, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ReturnBdaFailureIf(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& target); // Whether a faulting BDA access may end its invocation. Programs with workgroup barriers must keep @@ -23,7 +24,7 @@ void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& targe bool BdaInvocationsMayStop(const IrProgram& program); void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition); std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits); -void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value); +void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u); void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits); std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bytes, const std::function& operation); // Reads the dwords of a 1-4 dword load at address + offset that the program extracts, with one diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp index 729fe3de2..d221ffe2e 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp @@ -19,12 +19,13 @@ struct SpirvTargetOptions { std::span supportedCapabilities; std::span supportedExtensions; bool nonConstantImageOffsets = false; + bool narrowSubgroupClock = false; }; class SpirvEmitter { public: - [[nodiscard]] std::vector Emit(const IrProgram& program, const BindingAllocationResult& bindings, const SpirvTargetOptions& target) const; - [[nodiscard]] std::vector Emit(const IrProgram& program, const ShaderStageInputInfo& inputInfo, const BindingAllocationResult& bindings, const SpirvTargetOptions& target) const; + [[nodiscard]] std::vector Emit(const IrProgram& program, const CompiledBindingLayout& bindings, const SpirvTargetOptions& target) const; + [[nodiscard]] std::vector Emit(const IrProgram& program, const ShaderStageInputInfo& inputInfo, const CompiledBindingLayout& bindings, const SpirvTargetOptions& target) const; }; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index 7d8f0874c..07af37ad0 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -61,7 +61,7 @@ std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t std::uint32_t LoadSamplerDescriptor(SpirvEmitterState& state, std::uint32_t sampler); std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t sampler, std::uint32_t slotId = 0); std::uint32_t StorageImageDescriptorPointer(SpirvEmitterState& state, std::uint32_t resource); -void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t mipLod, std::uint32_t coord, std::uint32_t texel); +void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t mipLod, std::uint32_t coord, std::uint32_t texel, std::uint32_t sample); std::uint32_t ExecutionModelForStage(IrShaderStage stage); std::uint32_t ConstantI32(SpirvEmitterState& state, std::int32_t value); std::uint32_t ConstantF32(SpirvEmitterState& state, std::uint32_t bits); @@ -71,7 +71,7 @@ std::uint32_t ConstantBool(SpirvEmitterState& state, bool value); std::uint32_t ConstantU64(SpirvEmitterState& state, std::uint64_t value); std::uint32_t ConstantU32CompositeZero(SpirvEmitterState& state, std::uint32_t components); std::uint32_t DefineInterfaceVariable(SpirvEmitterState& state, std::uint32_t type, std::uint32_t storage, const char* name); -void CheckBindings(const IrProgram& program, const BindingAllocationResult& bindings); +void CheckBindings(const IrProgram& program, const CompiledBindingLayout& bindings); void EmitBaseHeader(SpirvModule& module, const IrProgram& program); void DefineInputs(SpirvEmitterState& state); void DefineOutputs(SpirvEmitterState& state); @@ -90,6 +90,7 @@ DppTargetLane EmitDppRowRotateRightTargetLane(SpirvEmitterState& state, std::uin DppTargetLane EmitDppMirrorTargetLane(SpirvEmitterState& state, std::uint32_t subid, bool halfRow); DppTargetLane EmitDppTargetLane(SpirvEmitterState& state, std::uint32_t control); std::uint32_t InputVariableForKind(const SpirvEmitterState& state, StageInputKind kind); +bool OrderedPixelShader(const SpirvEmitterState& state); const SpirvInputBinding* SpirvInputBindingForParameter(const SpirvEmitterState& state, std::uint32_t location); std::uint32_t EmitVertexParameterComponentU32(SpirvEmitterState& state, const SpirvInputBinding& input, std::uint32_t component); std::uint32_t EmitInputComponentU32(SpirvEmitterState& state, StageInputKind kind, std::uint32_t component); @@ -98,13 +99,10 @@ std::uint32_t EmitBallotLaneActiveBool(SpirvEmitterState& state, std::uint32_t b std::uint32_t EmitSubgroupLaneActiveBool(SpirvEmitterState& state, std::uint32_t lane); std::uint32_t EmitBinaryU32(SpirvEmitterState& state, std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs); std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dwordIndex); -std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem); -IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem); void EmitMemoryOffsets(SpirvEmitterState& state); std::uint32_t LdsDwordCount(const SpirvEmitterState& state); std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state); MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem); -MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType); std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t rawIndex); std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); std::uint32_t EmitMemoryElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp index f558e5b85..a2f5aa8d5 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp @@ -78,6 +78,7 @@ std::uint32_t EmitWriteLane(SpirvValueEmitContext& ctx, const IrValue& inst); std::uint32_t EmitPermlane16U32(SpirvValueEmitContext& ctx, const IrValue& inst); std::uint32_t EmitGetAttribute(SpirvValueEmitContext& ctx, const IrValue& inst); std::uint32_t EmitGetInterpolationParameter(SpirvValueEmitContext& ctx, const IrValue& inst); +std::uint32_t EmitGetInterpolationParameterF16(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitSetAttribute(SpirvValueEmitContext& ctx, const IrValue& inst); std::uint32_t EmitGetShaderBase(SpirvValueEmitContext& ctx); std::uint32_t EmitShaderClock(SpirvValueEmitContext& ctx, const IrValue& inst); @@ -196,6 +197,8 @@ std::uint32_t EmitFPRsq64(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitFPSqrt64(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitFPTrigPreop64(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1); std::uint32_t EmitFPDot2F32F16(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2); +std::uint32_t EmitFPInterpolateF32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, const IrValue* mode); +std::uint32_t EmitFPInterpolateF16(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, const IrValue* mode); std::uint32_t EmitConvertF32F64(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitConvertF64F32(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitConvertF64S32(SpirvEmitterState& state, std::uint32_t arg0); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index 8fb8e187b..a2498d029 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -68,9 +68,12 @@ struct MemoryResourceAccess { std::uint32_t byteOffset = 0; bool addIndexOffset = false; std::uint32_t memoryAccess = 0; + std::uint32_t misalignment = 0; }; struct SpirvEmitterState { + const ImageResource* runtimeImage = nullptr; + std::uint32_t runtimeImageMetadata = 0u; SpirvEmitterState(const IrProgram& program, const ShaderStageInputInfo& inputInfo); SpirvModule module; @@ -84,6 +87,7 @@ struct SpirvEmitterState { std::uint32_t laneCount = 1; std::uint32_t hostSubgroupSize = 0; bool splitSubgroup = false; + bool narrowSubgroupClock = false; std::unordered_set sharedLaneValues; std::uint32_t laneHalf = 0; // The target's SPIR-V version and what the device accepts, for capabilities an emitter adds @@ -105,6 +109,7 @@ struct SpirvEmitterState { // every read takes the byte path. std::uint32_t bdaProbeFunction = 0; std::uint32_t bdaWritePointerFunction = 0; + std::uint32_t bdaAtomicPointerFunction = 0; std::uint32_t bdaNoteWriteFunction = 0; std::uint32_t bdaFaultFunction = 0; std::array, 2> bdaDwordReadFunctions {}; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp index 2701d0530..ee30b6506 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp @@ -9,12 +9,9 @@ namespace ShaderRecompiler { std::uint32_t EmitBinaryU32(SpirvEmitterState& state, std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs); std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dwordIndex); -std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem); -IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem); void EmitMemoryOffsets(SpirvEmitterState& state); std::uint32_t LdsDwordCount(const SpirvEmitterState& state); std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state); -MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType); MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem); std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t rawIndex); std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp index 404ac85d1..1650714e3 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp @@ -13,12 +13,11 @@ std::uint32_t ImageScalarType(SpirvEmitterState& state, IrTextureNumericClass nu std::uint32_t ImageVectorType(SpirvEmitterState& state, IrTextureNumericClass numericClass, std::uint32_t components); std::uint32_t ImageType(SpirvEmitterState& state, const ImageResource& image); std::uint32_t ImageViewSizeType(SpirvEmitterState& state, RdnaImageDimension dimension); -// `slotId` (a SPIR-V id, 0 for none) selects a bindless table's slot at runtime: element(resource) + slot. std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t slotId); std::uint32_t LoadSamplerDescriptor(SpirvEmitterState& state, std::uint32_t sampler); std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t sampler, std::uint32_t slotId); std::uint32_t StorageImageDescriptorPointer(SpirvEmitterState& state, std::uint32_t resource); -void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t mipLod, std::uint32_t coord, std::uint32_t texel); +void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t mipLod, std::uint32_t coord, std::uint32_t texel, std::uint32_t sample); const RdnaImageDimensionInfo& RdnaImageDimensionInfoFor(RdnaImageDimension dimension); [[noreturn]] void ExitDescriptorBindingFailure(const SpirvEmitterState& state, DescriptorBindingKind kind, std::uint32_t resource, const char* reason); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvInterface.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvInterface.hpp index efbab3988..d6eb2bfac 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvInterface.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvInterface.hpp @@ -8,8 +8,8 @@ namespace ShaderRecompiler { -void EmitModuleHeader(SpirvModule& module, const IrProgram& program, const BindingAllocationResult& bindings); -void EmitModuleHeader(SpirvEmitterState& state, const BindingAllocationResult& bindings); +void EmitModuleHeader(SpirvModule& module, const IrProgram& program, const CompiledBindingLayout& bindings); +void EmitModuleHeader(SpirvEmitterState& state, const CompiledBindingLayout& bindings); void DefineModule(SpirvEmitterState& state); std::uint32_t DefineInterfaceVariable(SpirvEmitterState& state, std::uint32_t type, std::uint32_t storage, const char* name); std::uint32_t ExecutionModelForStage(IrShaderStage stage); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvMemoryAccess.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvMemoryAccess.hpp index 1331877d0..4228fbc05 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvMemoryAccess.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvMemoryAccess.hpp @@ -7,13 +7,10 @@ namespace ShaderRecompiler { std::uint32_t LdsDwordCount(const SpirvEmitterState& state); -std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem); -IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem); void EmitMemoryOffsets(SpirvEmitterState& state); MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem); -MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType); std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t rawIndex); std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvModuleSetup.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvModuleSetup.hpp index 253791eb6..9f9f948d9 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvModuleSetup.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvModuleSetup.hpp @@ -9,8 +9,8 @@ namespace ShaderRecompiler { -void EmitModuleHeader(SpirvModule& module, const IrProgram& program, const BindingAllocationResult& bindings); -void EmitModuleHeader(SpirvEmitterState& state, const BindingAllocationResult& bindings); +void EmitModuleHeader(SpirvModule& module, const IrProgram& program, const CompiledBindingLayout& bindings); +void EmitModuleHeader(SpirvEmitterState& state, const CompiledBindingLayout& bindings); void DefineModule(SpirvEmitterState& state); std::uint32_t ExecutionModelForStage(IrShaderStage stage); std::uint32_t DefineInterfaceVariable(SpirvEmitterState& state, std::uint32_t type, std::uint32_t storage, const char* name); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModule.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModule.hpp index 430a6bd93..15e5f762c 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModule.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModule.hpp @@ -39,6 +39,7 @@ private: public: explicit SpirvModule(std::uint32_t version = 0x00010300u); [[nodiscard]] std::uint32_t AllocateId(); + [[nodiscard]] std::uint32_t SpecializationConstant(std::uint32_t type, std::uint32_t constantId, std::uint32_t defaultValue); void EmitCapability(std::uint32_t capability); void EmitExtension(const std::string& extensionName); void EmitEntryPoint(std::uint32_t executionModel, std::uint32_t entryPointId, const std::string& entryPointName, const std::vector& interfaceIds); @@ -176,6 +177,7 @@ private: return id; } + std::map specializationIds; static void appendOperand(std::vector& words, std::uint32_t value) { words.push_back(value); } diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModuleEmitter.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModuleEmitter.hpp index 6350bda72..c176eca60 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModuleEmitter.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModuleEmitter.hpp @@ -9,9 +9,9 @@ namespace ShaderRecompiler { -void EmitModuleHeader(SpirvModule& module, const IrProgram& program, const BindingAllocationResult& bindings); +void EmitModuleHeader(SpirvModule& module, const IrProgram& program, const CompiledBindingLayout& bindings); -void EmitModuleHeader(SpirvEmitterState& state, const BindingAllocationResult& bindings); +void EmitModuleHeader(SpirvEmitterState& state, const CompiledBindingLayout& bindings); } diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvOptimizer.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvOptimizer.hpp index 670851c15..661caa053 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvOptimizer.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvOptimizer.hpp @@ -7,7 +7,7 @@ namespace ShaderRecompiler { -[[nodiscard]] std::vector ValidateAndOptimizeSpirv(std::span spirv, std::uint32_t vulkanVersion, std::uint32_t spirvVersion, bool allowOffsetTextureOperand = false); +[[nodiscard]] std::vector ValidateAndOptimizeSpirv(std::span spirv, std::uint32_t vulkanVersion, std::uint32_t spirvVersion, bool allowOffsetTextureOperand = false, bool optimize = true, bool specialize = false); } diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvRuntimeBuffer.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvRuntimeBuffer.hpp new file mode 100644 index 000000000..bd7fa1a63 --- /dev/null +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvRuntimeBuffer.hpp @@ -0,0 +1,16 @@ +#ifndef CORE_SHADER_RECOMPILER_SPIRVBACKEND_SPIRVRUNTIMEBUFFER_HPP +#define CORE_SHADER_RECOMPILER_SPIRVBACKEND_SPIRVRUNTIMEBUFFER_HPP + +#include "SpirvBackend/SpirvEmitterHelpers.hpp" +#include + +namespace ShaderRecompiler { + +std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components); +void EmitRuntimeBufferStore(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components); +std::uint32_t EmitRuntimeScalarBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction); +std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t bits, std::uint32_t active, const std::function& operation); + +} + +#endif diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvSpecialization.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvSpecialization.hpp new file mode 100644 index 000000000..782710f10 --- /dev/null +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvSpecialization.hpp @@ -0,0 +1,14 @@ +#ifndef CORE_SHADER_RECOMPILER_SPIRVSPECIALIZATION_HPP +#define CORE_SHADER_RECOMPILER_SPIRVSPECIALIZATION_HPP + +#include +#include +#include + +namespace ShaderRecompiler { + +[[nodiscard]] std::vector SpecializeSpirv(std::span words); + +} + +#endif diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp index 1dec262eb..95c2d7450 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp @@ -197,6 +197,98 @@ std::uint32_t AddRoundToOdd(SpirvEmitterState& state, std::uint32_t lhs, std::ui return ToF64(state, Join(state, SelectBits(state, inexact, odd, rounded))); } +std::uint32_t FromF32Exact(SpirvEmitterState& state, std::uint32_t value) { + const auto u32 = TypeU32(state); + const auto bits = Unary(state, spv::OpBitcast, u32, value); + const auto sign = Binary(state, spv::OpBitwiseAnd, u32, bits, ConstantU32(state, 0x80000000u)); + const auto magnitude = Binary(state, spv::OpBitwiseAnd, u32, bits, ConstantU32(state, 0x7fffffffu)); + const auto subnormal = Binary(state, spv::OpULessThan, TypeBool(state), magnitude, ConstantU32(state, 0x00800000u)); + const auto scaled = Exact(state, spv::OpFMul, Unary(state, spv::OpConvertUToF, TypeF64(state), magnitude), ConstantF64(state, 0x36a0000000000000ull)); + const F64Bits normal{Binary(state, spv::OpShiftLeftLogical, u32, magnitude, ConstantU32(state, 29u)), Binary(state, spv::OpIAdd, u32, Binary(state, spv::OpShiftRightLogical, u32, magnitude, ConstantU32(state, 3u)), ConstantU32(state, 896u << 20u))}; + const F64Bits special{normal.low, Binary(state, spv::OpBitwiseOr, u32, Binary(state, spv::OpShiftRightLogical, u32, Binary(state, spv::OpBitwiseAnd, u32, magnitude, ConstantU32(state, 0x007fffffu)), ConstantU32(state, 3u)), ConstantU32(state, 0x7ff00000u))}; + const auto nonFinite = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), magnitude, ConstantU32(state, 0x7f800000u)); + const auto unsignedBits = SelectBits(state, nonFinite, special, Split(state, FromF64(state, Select(state, TypeF64(state), subnormal, scaled, ToF64(state, Join(state, normal)))))); + return ToF64(state, Join(state, {unsignedBits.low, Binary(state, spv::OpBitwiseOr, u32, unsignedBits.high, sign)})); +} + +std::uint32_t RoundToPrecision(SpirvEmitterState& state, std::uint32_t value, std::uint32_t fractionBits, std::uint32_t smallestStep) { + const auto u32 = TypeU32(state); + const auto bits = Split(state, FromF64(state, value)); + const auto exponent = Unary(state, spv::OpBitcast, TypeI32(state), ExponentField(state, bits.high)); + const auto step = Glsl(state, TypeI32(state), GLSLstd450SMax, Binary(state, spv::OpISub, TypeI32(state), exponent, ConstantI32(state, static_cast(fractionBits))), ConstantI32(state, static_cast(smallestStep))); + const auto sign = Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x80000000u)); + const auto magicExponent = Binary(state, spv::OpShiftLeftLogical, u32, Unary(state, spv::OpBitcast, u32, Binary(state, spv::OpIAdd, TypeI32(state), step, ConstantI32(state, 52))), ConstantU32(state, 20u)); + const auto magic = ToF64(state, Join(state, {ConstantU32(state, 0u), Binary(state, spv::OpBitwiseOr, u32, sign, magicExponent)})); + const auto rounded = Split(state, FromF64(state, Exact(state, spv::OpFSub, Exact(state, spv::OpFAdd, value, magic), magic))); + return ToF64(state, Join(state, {rounded.low, Binary(state, spv::OpBitwiseOr, u32, Binary(state, spv::OpBitwiseAnd, u32, rounded.high, ConstantU32(state, 0x7fffffffu)), sign)})); +} + +std::uint32_t F32BitsOfRounded(SpirvEmitterState& state, std::uint32_t value) { + const auto u32 = TypeU32(state); + const auto bits = Split(state, FromF64(state, value)); + const auto sign = Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x80000000u)); + const auto exponent = ExponentField(state, bits.high); + const auto fraction = Binary(state, spv::OpBitwiseOr, u32, Binary(state, spv::OpShiftLeftLogical, u32, Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x000fffffu)), ConstantU32(state, 3u)), Binary(state, spv::OpShiftRightLogical, u32, bits.low, ConstantU32(state, 29u))); + const auto normal = Binary(state, spv::OpBitwiseOr, u32, Binary(state, spv::OpShiftLeftLogical, u32, Binary(state, spv::OpISub, u32, exponent, ConstantU32(state, 896u)), ConstantU32(state, 23u)), fraction); + const auto magnitude = ToF64(state, Join(state, {bits.low, Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x7fffffffu))})); + const auto subnormal = Unary(state, spv::OpConvertFToU, u32, Exact(state, spv::OpFMul, magnitude, ConstantF64(state, 0x4940000000000000ull))); + auto result = Select(state, u32, Binary(state, spv::OpULessThan, TypeBool(state), exponent, ConstantU32(state, 897u)), subnormal, normal); + result = Select(state, u32, Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), exponent, ConstantU32(state, 1151u)), ConstantU32(state, 0x7f800000u), result); + return Binary(state, spv::OpBitwiseOr, u32, result, sign); +} + +std::uint32_t NonFinite32(SpirvEmitterState& state, std::uint32_t bits) { + const auto exponent = Binary(state, spv::OpBitwiseAnd, TypeU32(state), bits, ConstantU32(state, 0x7f800000u)); + return Binary(state, spv::OpIEqual, TypeBool(state), exponent, ConstantU32(state, 0x7f800000u)); +} + +std::uint32_t Nan32(SpirvEmitterState& state, std::uint32_t bits) { + return Binary(state, spv::OpUGreaterThan, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), bits, ConstantU32(state, 0x7fffffffu)), ConstantU32(state, 0x7f800000u)); +} + +std::uint32_t FlushDenormal32(SpirvEmitterState& state, std::uint32_t value, bool flush, std::uint32_t smallestNormal) { + if (!flush) return value; + const auto u32 = TypeU32(state); + const auto bits = Unary(state, spv::OpBitcast, u32, value); + const auto magnitude = Binary(state, spv::OpBitwiseAnd, u32, bits, ConstantU32(state, 0x7fffffffu)); + const auto denormal = Binary(state, spv::OpLogicalAnd, TypeBool(state), Binary(state, spv::OpINotEqual, TypeBool(state), magnitude, ConstantU32(state, 0u)), Binary(state, spv::OpULessThan, TypeBool(state), magnitude, ConstantU32(state, smallestNormal))); + return Unary(state, spv::OpBitcast, TypeF32(state), Select(state, u32, denormal, Binary(state, spv::OpBitwiseAnd, u32, bits, ConstantU32(state, 0x80000000u)), bits)); +} + +std::uint32_t Interpolate(SpirvEmitterState& state, std::uint32_t delta, std::uint32_t coordinate, std::uint32_t base, bool halfResult, std::uint32_t mode) { + const auto u32 = TypeU32(state); + const bool quietNans = halfResult || (mode & InterpolationQuiet) != 0u; + const bool flush32 = (mode & InterpolationFlush32) != 0u; + const bool flush16 = (mode & InterpolationFlush16) != 0u; + delta = FlushDenormal32(state, delta, flush16, 0x38800000u); + coordinate = FlushDenormal32(state, coordinate, flush32, 0x00800000u); + base = halfResult ? FlushDenormal32(state, base, flush32, 0x00800000u) : FlushDenormal32(state, base, flush16, 0x38800000u); + const auto product = Exact(state, spv::OpFMul, FromF32Exact(state, delta), FromF32Exact(state, coordinate)); + const auto sum = AddRoundToOdd(state, product, FromF32Exact(state, base)); + auto rounded = halfResult ? RoundToPrecision(state, sum, 10u, 999u) : RoundToPrecision(state, sum, 23u, 874u); + if (halfResult) { + const auto bits = Split(state, FromF64(state, rounded)); + const auto sign = Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x80000000u)); + const auto overflow = Binary(state, spv::OpUGreaterThan, TypeBool(state), Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x7fffffffu)), ConstantU32(state, 0x40f00000u)); + rounded = ToF64(state, Join(state, SelectBits(state, overflow, {ConstantU32(state, 0u), Binary(state, spv::OpBitwiseOr, u32, sign, ConstantU32(state, 0x40f00000u))}, bits))); + } + const auto finite = F32BitsOfRounded(state, rounded); + const auto inputs = std::array{Unary(state, spv::OpBitcast, u32, delta), Unary(state, spv::OpBitcast, u32, coordinate), Unary(state, spv::OpBitcast, u32, base)}; + const auto unbounded = Split(state, FromF64(state, Exact(state, spv::OpFAdd, product, FromF32Exact(state, base)))); + auto special = Select(state, u32, Classify(state, Join(state, unbounded)).nan, ConstantU32(state, 0xffc00000u), Binary(state, spv::OpBitwiseOr, u32, Binary(state, spv::OpBitwiseAnd, u32, unbounded.high, ConstantU32(state, 0x80000000u)), ConstantU32(state, 0x7f800000u))); + auto nonFinite = NonFinite32(state, inputs[0]); + for (const auto input : {inputs[1], inputs[2]}) nonFinite = Binary(state, spv::OpLogicalOr, TypeBool(state), nonFinite, NonFinite32(state, input)); + const auto zero = [&](std::uint32_t bits) { return Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, u32, bits, ConstantU32(state, 0x7fffffffu)), ConstantU32(state, 0u)); }; + const auto infinite = [&](std::uint32_t bits) { return Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, u32, bits, ConstantU32(state, 0x7fffffffu)), ConstantU32(state, 0x7f800000u)); }; + const auto invalidProduct = Binary(state, spv::OpLogicalOr, TypeBool(state), Binary(state, spv::OpLogicalAnd, TypeBool(state), infinite(inputs[0]), zero(inputs[1])), Binary(state, spv::OpLogicalAnd, TypeBool(state), zero(inputs[0]), infinite(inputs[1]))); + const auto quiet = [&](std::uint32_t bits) { return quietNans ? Binary(state, spv::OpBitwiseOr, u32, bits, ConstantU32(state, 0x00400000u)) : bits; }; + special = Select(state, u32, Binary(state, spv::OpLogicalAnd, TypeBool(state), Nan32(state, inputs[2]), Unary(state, spv::OpLogicalNot, TypeBool(state), invalidProduct)), quiet(inputs[2]), special); + special = Select(state, u32, Nan32(state, inputs[1]), quiet(inputs[1]), special); + special = Select(state, u32, Nan32(state, inputs[0]), quiet(inputs[0]), special); + const auto result = Unary(state, spv::OpBitcast, TypeF32(state), Select(state, u32, nonFinite, special, finite)); + return halfResult ? FlushDenormal32(state, result, flush16, 0x38800000u) : FlushDenormal32(state, result, flush32, 0x00800000u); +} + std::uint32_t FusedMultiplyAdd(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, std::uint32_t scale) { const auto a = Classify(state, arg0); const auto b = Classify(state, arg1); @@ -532,6 +624,14 @@ std::uint32_t EmitFPDot2F32F16(SpirvEmitterState& state, std::uint32_t arg0, std return pick(aLow.nan, quietHalf(aLow), result); } +std::uint32_t EmitFPInterpolateF32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, const IrValue* mode) { + return Interpolate(state, arg0, arg1, arg2, false, mode->ImmediateU32()); +} + +std::uint32_t EmitFPInterpolateF16(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, const IrValue* mode) { + return Interpolate(state, arg0, arg1, arg2, true, mode->ImmediateU32()); +} + std::uint32_t EmitConvertF32F64(SpirvEmitterState& state, std::uint32_t arg0) { const auto converted = Unary(state, spv::OpBitcast, TypeU32(state), Unary(state, spv::OpFConvert, TypeF32(state), ToF64(state, arg0))); const auto cls = Classify(state, arg0); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp index dd8c915aa..ab3a31dfe 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp @@ -3,8 +3,12 @@ #include "IntermediateRepresentation/IrValue.hpp" #include #include +#include +#include #include +#include #include +#include #include namespace ShaderRecompiler { @@ -226,6 +230,202 @@ bool HostSubgroupNarrowerThanWave(const SpirvEmitterState& state) { ctx.Fail(inst, (access + " is outside the " + std::to_string(state.hostSubgroupSize) + "-lane host subgroup that runs this wave" + std::to_string(state.program.WaveSize()) + " program at one lane per invocation").c_str()); } +struct WaveReduction { + IrOpcode opcode; + std::uint32_t identity; + spv::Op reduce; +}; + +constexpr std::array WaveReductions{{ + {IrOpcode::UMin32, 0xffffffffu, spv::OpGroupNonUniformUMin}, + {IrOpcode::UMax32, 0u, spv::OpGroupNonUniformUMax}, + {IrOpcode::SMin32, 0x7fffffffu, spv::OpGroupNonUniformSMin}, + {IrOpcode::SMax32, 0x80000000u, spv::OpGroupNonUniformSMax}, + {IrOpcode::IAdd32, 0u, spv::OpGroupNonUniformIAdd}, + {IrOpcode::BitwiseAnd32, 0xffffffffu, spv::OpGroupNonUniformBitwiseAnd}, + {IrOpcode::BitwiseOr32, 0u, spv::OpGroupNonUniformBitwiseOr}, +}}; + +struct HalfWaveScan { + const WaveReduction* reduction = nullptr; + const IrValue* source = nullptr; +}; + +const IrValue* Resolved(const IrValue* value) { + return value != nullptr ? value->Resolve() : nullptr; +} + +bool Is(const IrValue* value, IrOpcode opcode) { + return value != nullptr && !value->HasImmediate() && value->Opcode() == opcode; +} + +bool IsU32(const IrValue* value, std::uint32_t expected) { + const auto* resolved = Resolved(value); + return resolved != nullptr && resolved->HasImmediate() && resolved->Type() == IrType::U32 && resolved->ImmediateU32() == expected; +} + +template +bool EitherOrder(const IrValue* value, TMatch&& match) { + return match(Resolved(value->Argument(0)), Resolved(value->Argument(1))) || match(Resolved(value->Argument(1)), Resolved(value->Argument(0))); +} + +std::optional> LaneBitWords(const IrValue* bit) { + const IrValue* masked = nullptr; + const IrValue* shifted = nullptr; + if (!Is(bit, IrOpcode::INotEqual32) || !EitherOrder(bit, [&](const IrValue* value, const IrValue* zero) { masked = value; return IsU32(zero, 0u) && Is(value, IrOpcode::BitwiseAnd32); })) return std::nullopt; + if (!EitherOrder(masked, [&](const IrValue* value, const IrValue* one) { shifted = value; return IsU32(one, 1u) && Is(value, IrOpcode::ShiftRightLogical32); })) return std::nullopt; + const auto* index = Resolved(shifted->Argument(1)); + if (!Is(index, IrOpcode::BitwiseAnd32) || !EitherOrder(index, [](const IrValue* lane, const IrValue* mask) { return Is(lane, IrOpcode::LaneId) && IsU32(mask, 31u); })) return std::nullopt; + const auto* word = Resolved(shifted->Argument(0)); + if (!Is(word, IrOpcode::SelectU32)) return std::array{word, word}; + const auto* low = Resolved(word->Argument(0)); + if (!Is(low, IrOpcode::ULessThan32) || !Is(Resolved(low->Argument(0)), IrOpcode::LaneId) || !IsU32(low->Argument(1), 32u)) return std::nullopt; + return std::array{Resolved(word->Argument(1)), Resolved(word->Argument(2))}; +} + +bool AllOnesWord(const IrValue* word) { + if (IsU32(word, 0xffffffffu)) return true; + return Is(word, IrOpcode::BitwiseOr32) && EitherOrder(word, [](const IrValue* inverted, const IrValue* other) { + return AllOnesWord(inverted) || (Is(inverted, IrOpcode::BitwiseNot32) && Resolved(inverted->Argument(0)) == other); + }); +} + +bool AllLanesBit(const IrValue* bit) { + const auto words = LaneBitWords(Resolved(bit)); + return words && AllOnesWord((*words)[0]) && AllOnesWord((*words)[1]); +} + +class EntryLaneWalk { +public: + bool ZeroBit(const IrValue* bit) { + bit = Resolved(bit); + if (bit == nullptr || !Spend()) return false; + if (Is(bit, IrOpcode::LogicalAnd)) return ZeroBit(bit->Argument(0)) || ZeroBit(bit->Argument(1)); + const auto words = LaneBitWords(bit); + return words && Word((*words)[0], 0u) && Word((*words)[1], 1u); + } + +private: + bool Spend() { + if (budget == 0u) return false; + --budget; + return true; + } + + bool NotHelper(const IrValue* predicate) { + predicate = Resolved(predicate); + if (predicate == nullptr || predicate->HasImmediate() || !Spend()) return false; + switch (predicate->Opcode()) { + case IrOpcode::LogicalAnd: return NotHelper(predicate->Argument(0)) || NotHelper(predicate->Argument(1)); + case IrOpcode::IEqual32: return EitherOrder(predicate, [](const IrValue* builtin, const IrValue* zero) { return IsU32(zero, 0u) && Is(builtin, IrOpcode::GetBuiltin) && IsU32(builtin->Argument(0), static_cast(StageInputKind::HelperInvocation)) && IsU32(builtin->Argument(1), 0u); }); + default: return ZeroBit(predicate); + } + } + + bool Word(const IrValue* word, std::uint32_t half) { + word = Resolved(word); + if (word == nullptr || !Spend()) return false; + if (word->HasImmediate()) return IsU32(word, 0u); + switch (word->Opcode()) { + case IrOpcode::CompositeExtractU32x4: { + const auto* ballot = Resolved(word->Argument(0)); + return IsU32(word->Argument(1), half) && Is(ballot, IrOpcode::Ballot) && NotHelper(ballot->Argument(0)); + } + case IrOpcode::CompositeExtractU64: return IsU32(word->Argument(1), half) && Word64(word->Argument(0), half); + case IrOpcode::BitwiseAnd32: return Word(word->Argument(0), half) || Word(word->Argument(1), half); + case IrOpcode::BitwiseOr32: return Word(word->Argument(0), half) && Word(word->Argument(1), half); + case IrOpcode::Phi: return Incoming(word, [this, half](const IrValue* incoming) { return Word(incoming, half); }); + default: return false; + } + } + + bool Word64(const IrValue* mask, std::uint32_t half) { + mask = Resolved(mask); + if (mask == nullptr || !Spend()) return false; + if (mask->HasImmediate()) return mask->Type() == IrType::U64 && static_cast(mask->ImmediateU64() >> (32u * half)) == 0u; + switch (mask->Opcode()) { + case IrOpcode::CompositeConstructU64: return Word(mask->Argument(half), half); + case IrOpcode::Phi: return Incoming(mask, [this, half](const IrValue* incoming) { return Word64(incoming, half); }); + default: return false; + } + } + + template + bool Incoming(const IrValue* phi, TCheck&& check) { + if (!phis.insert(phi).second) return true; + bool result = phi->ArgumentCount() != 0u; + for (std::size_t index = 0; result && index < phi->ArgumentCount(); ++index) result = check(phi->Argument(index)); + phis.erase(phi); + return result; + } + + std::unordered_set phis; + std::uint32_t budget = 512u; +}; + +bool RowShiftFlags(const IrValue* value, std::uint32_t control) { + const auto flags = value->Flags(); + return flags.control == control && flags.rowMask == 0xfu && flags.bankMask == 0xfu && !flags.boundControl && !flags.fetchInactive; +} + +const IrValue* RowScanStepInput(const IrValue* step, const WaveReduction& reduction, std::uint32_t control) { + if (!Is(step, IrOpcode::DppUpdateU32) || !RowShiftFlags(step, control) || !AllLanesBit(step->Argument(2))) return nullptr; + const auto* previous = Resolved(step->Argument(1)); + const auto* combined = Resolved(step->Argument(0)); + const bool matched = Is(combined, reduction.opcode) && EitherOrder(combined, [&](const IrValue* moved, const IrValue* own) { + return own == previous && Is(moved, IrOpcode::DppMoveU32) && RowShiftFlags(moved, control) && Resolved(moved->Argument(0)) == previous && AllLanesBit(moved->Argument(1)); + }); + return matched ? previous : nullptr; +} + +const IrValue* Unmasked(const IrValue* value) { + while (Is(value, IrOpcode::SelectU32) && AllLanesBit(value->Argument(0))) value = Resolved(value->Argument(1)); + return value; +} + +bool CrossesRowsOf(const IrValue* value, const IrValue* scan) { + const auto* permlane = Unmasked(value); + if (!Is(permlane, IrOpcode::Permlane16U32)) return false; + const auto flags = permlane->Flags(); + return flags.x16 && !flags.fetchInactive && Resolved(permlane->Argument(0)) == scan && IsU32(permlane->Argument(1), 0xffffffffu) && IsU32(permlane->Argument(2), 0xffffffffu) && AllLanesBit(permlane->Argument(3)); +} + +std::optional MatchHalfWaveScan(const IrValue* value) { + const auto* combined = Unmasked(Resolved(value)); + const auto reduction = std::find_if(WaveReductions.begin(), WaveReductions.end(), [&](const WaveReduction& candidate) { return Is(combined, candidate.opcode); }); + const IrValue* scan = nullptr; + if (reduction == WaveReductions.end() || !EitherOrder(combined, [&](const IrValue* own, const IrValue* crossed) { scan = own; return CrossesRowsOf(crossed, own); })) return std::nullopt; + for (const auto control : {0x118u, 0x114u, 0x112u, 0x111u}) { + scan = RowScanStepInput(scan, *reduction, control); + if (scan == nullptr) return std::nullopt; + } + const auto* source = Unmasked(scan); + if (!Is(source, IrOpcode::SelectU32) || !IsU32(source->Argument(2), reduction->identity) || !EntryLaneWalk{}.ZeroBit(source->Argument(0))) return std::nullopt; + return HalfWaveScan{&*reduction, source}; +} + +std::optional EmitHalfWaveReduction(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t lane) { + auto& state = ctx.state; + if (state.program.WaveSize() != 64u || (lane & 31u) != 31u) return std::nullopt; + const auto scan = MatchHalfWaveScan(inst.Argument(0)); + if (!scan) return std::nullopt; + const auto read = "v_readlane_b32 of lane " + std::to_string(lane) + " of a wave64 half-wave reduction scan"; + const auto& capabilities = state.supportedCapabilities; + if (std::find(capabilities.begin(), capabilities.end(), static_cast(spv::CapabilityGroupNonUniformArithmetic)) == capabilities.end()) ctx.Fail(inst, (read + " needs subgroup arithmetic, which the device lacks").c_str()); + if (state.hostSubgroupSize > 64u) ctx.Fail(inst, (read + " on a " + std::to_string(state.hostSubgroupSize) + "-lane host subgroup, which is wider than the wave").c_str()); + const auto identity = ConstantU32(state, scan->reduction->identity); + if (lane >= 32u && state.hostSubgroupSize <= 32u) return identity; + auto keys = ctx.Def(scan->source); + if (state.hostSubgroupSize > 32u) { + const auto upper = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), ConstantU32(state, 32u)); + keys = lane >= 32u ? Select(state, TypeU32(state), upper, keys, identity) : Select(state, TypeU32(state), upper, identity, keys); + } + state.module.EmitCapability(spv::CapabilityGroupNonUniformArithmetic); + const auto result = state.module.AllocateId(); + state.module.AddFunction(scan->reduction->reduce, TypeU32(state), result, ConstantU32(state, spv::ScopeSubgroup), static_cast(spv::GroupOperationReduce), keys); + return result; +} + } std::uint32_t EmitConvertU16U32(SpirvEmitterState& state, std::uint32_t arg0) { @@ -558,9 +758,10 @@ std::uint32_t EmitReadFirstLane(SpirvValueEmitContext& ctx, const IrValue& inst) std::uint32_t EmitReadLane(SpirvValueEmitContext& ctx, const IrValue& inst) { auto& state = ctx.state; const IrValue* selector = inst.Argument(1)->Resolve(); - if (selector != nullptr && selector->HasImmediate() && HostSubgroupNarrowerThanWave(state)) { + if (selector != nullptr && selector->HasImmediate() && state.program.Resources().stage != IrShaderStage::Compute && state.laneCount == 1u) { const auto index = selector->ImmediateU32() & (state.program.WaveSize() - 1u); - if (index >= state.hostSubgroupSize) FailOutsideHostSubgroup(ctx, inst, "v_readlane_b32 of lane " + std::to_string(index)); + if (const auto reduced = EmitHalfWaveReduction(ctx, inst, index)) return *reduced; + if (HostSubgroupNarrowerThanWave(state) && index >= state.hostSubgroupSize) FailOutsideHostSubgroup(ctx, inst, "v_readlane_b32 of lane " + std::to_string(index)); } const auto lane = Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(inst, 1), ConstantU32(state, state.program.WaveSize() - 1u)); return ctx.Shuffle(inst, 0, lane); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp index 9ea921ea8..91b74f5ec 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp @@ -37,6 +37,7 @@ bool LaneSource(const IrProgram& program, const IrValue& value) { case IrOpcode::LaneId: case IrOpcode::GetAttribute: case IrOpcode::GetInterpolationParameter: + case IrOpcode::GetInterpolationParameterF16: case IrOpcode::GetTessellationAttribute: case IrOpcode::DppMoveU32: case IrOpcode::DppUpdateU32: @@ -332,21 +333,8 @@ SpirvRequirements AnalyzeProgramRequirements(const IrProgram& program) { } const auto& memory = program.Resources().memoryInfo.at(memoryIndex); requirements.coherentBuffers = requirements.coherentBuffers || (memory.coherent && !memory.gpuDescriptor); - if (memory.gpuDescriptor) { - if (program.Resources().stage != IrShaderStage::Compute) { - throw std::runtime_error("GPU-selected buffer descriptors outside compute shaders are not implemented"); - } - requirements.subgroupLocalInvocationId = true; - } else if (memory.kind == ResourceKind::Buffer) { - if (memory.resource >= program.Info().buffers.size()) { - throw std::runtime_error("buffer operation has invalid resource metadata"); - } - if ((program.Info().buffers.at(memory.resource).packedStride & (1u << 20u)) != 0u) { - if (program.Resources().stage != IrShaderStage::Compute) { - throw std::runtime_error("buffer ADD_TID is only valid for compute shaders"); - } - requirements.subgroupLocalInvocationId = true; - } + if (!memory.planningOnly) { + requirements.subgroupLocalInvocationId = requirements.subgroupLocalInvocationId || program.Resources().stage == IrShaderStage::Compute; } } const auto sharedAccess = SharedAccessOf(inst->Opcode()); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp index ad34e63e2..ca8c1a267 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp @@ -29,16 +29,20 @@ std::uint32_t BdaLoadAddress(SpirvEmitterState& state, std::uint32_t index) { void DefineBdaFaultFunction(SpirvEmitterState& state) { const auto u32 = TypeU32(state); - const auto u64 = TypeScalarU64(state); + const bool words = !state.program.Info().usesDma; + const auto addressType = words ? u32 : TypeScalarU64(state); state.bdaFaultFunction = state.module.AllocateId(); state.module.AddName(state.bdaFaultFunction, "record_bda_fault"); - state.module.AddFunction(spv::OpFunction, TypeVoid(state), state.bdaFaultFunction, spv::FunctionControlDontInlineMask, state.module.Type(spv::OpTypeFunction, TypeVoid(state), u32, u64, u32, u32)); + const auto type = words ? state.module.Type(spv::OpTypeFunction, TypeVoid(state), u32, addressType, u32, u32, u32) : state.module.Type(spv::OpTypeFunction, TypeVoid(state), u32, addressType, u32, u32); + state.module.AddFunction(spv::OpFunction, TypeVoid(state), state.bdaFaultFunction, spv::FunctionControlDontInlineMask, type); const auto reason = state.module.AllocateId(); const auto address = state.module.AllocateId(); + const auto addressHigh = words ? state.module.AllocateId() : 0u; const auto bytes = state.module.AllocateId(); const auto instruction = state.module.AllocateId(); state.module.AddFunction(spv::OpFunctionParameter, u32, reason); - state.module.AddFunction(spv::OpFunctionParameter, u64, address); + state.module.AddFunction(spv::OpFunctionParameter, addressType, address); + if (words) state.module.AddFunction(spv::OpFunctionParameter, u32, addressHigh); state.module.AddFunction(spv::OpFunctionParameter, u32, bytes); state.module.AddFunction(spv::OpFunctionParameter, u32, instruction); EmitLabel(state, state.module.AllocateId()); @@ -53,8 +57,8 @@ void DefineBdaFaultFunction(SpirvEmitterState& state) { state.module.AddFunction(spv::OpStore, BdaWord(state, state.faultBufferVariable, ConstantU32(state, index)), value); }; store(1, reason); - store(2, Unary(state, spv::OpUConvert, TypeU32(state), address)); - store(3, Unary(state, spv::OpUConvert, TypeU32(state), Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, 32u)))); + store(2, words ? address : Unary(state, spv::OpUConvert, TypeU32(state), address)); + store(3, words ? addressHigh : Unary(state, spv::OpUConvert, TypeU32(state), Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, 32u)))); store(4, bytes); store(5, ConstantU32(state, static_cast(state.program.Resources().stage))); store(6, instruction); @@ -67,10 +71,21 @@ void DefineBdaFaultFunction(SpirvEmitterState& state) { } void RecordBdaFault(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason) { - if (state.bdaFaultFunction == 0) throw std::runtime_error("BDA fault function is missing"); + if (state.bdaFaultFunction == 0 || !state.program.Info().usesDma) throw std::runtime_error("BDA fault function is missing"); state.module.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.module.AllocateId(), state.bdaFaultFunction, ConstantU32(state, static_cast(reason)), address, bytes, instruction); } +void RecordBdaFaultWords(SpirvEmitterState& state, std::uint32_t addressLow, std::uint32_t addressHigh, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason) { + if (state.program.Info().usesDma) { + const auto u64 = TypeScalarU64(state); + const auto high = Binary(state, spv::OpShiftLeftLogical, u64, Unary(state, spv::OpUConvert, u64, addressHigh), BdaConstant(state, 32u)); + RecordBdaFault(state, Binary(state, spv::OpBitwiseOr, u64, Unary(state, spv::OpUConvert, u64, addressLow), high), bytes, instruction, reason); + return; + } + if (state.bdaFaultFunction == 0) throw std::runtime_error("BDA fault function is missing"); + state.module.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.module.AllocateId(), state.bdaFaultFunction, ConstantU32(state, static_cast(reason)), addressLow, addressHigh, bytes, instruction); +} + void ReturnBdaFailureIf(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason) { const auto failed = state.module.AllocateId(); const auto next = state.module.AllocateId(); @@ -92,6 +107,9 @@ void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition) { if (state.bdaStopValue != 0) { state.module.AddFunction(spv::OpReturnValue, state.bdaStopValue); } else { + if (OrderedPixelShader(state)) { + state.module.AddFunction(spv::OpEndInvocationInterlockEXT); + } state.module.AddFunction(state.program.Resources().stage == IrShaderStage::Pixel ? spv::OpKill : spv::OpReturn); } EmitLabel(state, next); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp index 98c1b24d2..77701040f 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp @@ -102,7 +102,7 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r fail(binary(spv::OpINotEqual, boolean, BdaLoadWord(state, at(7)), constant(0)), BdaAbi::FaultReason::InvalidTable); fail(binary(spv::OpUGreaterThanEqual, boolean, begin, finish), BdaAbi::FaultReason::InvalidTable); fail(binary(spv::OpUGreaterThan, boolean, end, finish), BdaAbi::FaultReason::Unmapped); - fail(binary(spv::OpIEqual, boolean, binary(spv::OpBitwiseAnd, u32, permissions, constant(permission)), constant(0)), BdaAbi::FaultReason::Permission); + fail(binary(spv::OpINotEqual, boolean, binary(spv::OpBitwiseAnd, u32, permissions, constant(permission)), constant(permission)), BdaAbi::FaultReason::Permission); fail(binary(spv::OpIEqual, boolean, base, BdaConstant(state, 0)), BdaAbi::FaultReason::InvalidTable); const auto offset = binary(spv::OpISub, u64, address, begin); const auto result = binary(spv::OpIAdd, u64, base, offset); @@ -157,13 +157,14 @@ static std::uint32_t DefineBdaNoteWrite(SpirvEmitterState& state) { } void DefineGetBdaPointer(SpirvEmitterState& state) { + if (state.faultBufferVariable != 0) DefineBdaFaultFunction(state); if (!state.program.Info().usesDma) return; - DefineBdaFaultFunction(state); state.bdaPointerFunction = DefineBdaLookup(state, "get_bda_pointer", true); if (!BdaByteReadsForced()) state.bdaProbeFunction = DefineBdaLookup(state, "probe_bda_pointer", false); DefineBdaDwordReadFunctions(state); if (state.program.Info().bdaWrites) { state.bdaWritePointerFunction = DefineBdaLookup(state, "get_bda_write_pointer", true, BdaAbi::Write); + state.bdaAtomicPointerFunction = DefineBdaLookup(state, "get_bda_atomic_pointer", true, BdaAbi::Read | BdaAbi::Write); state.bdaNoteWriteFunction = DefineBdaNoteWrite(state); } } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index db3fb5db8..5b37506a5 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -167,6 +167,7 @@ std::uint32_t AddBdaImmediate(SpirvValueEmitContext& ctx, const IrValue& inst, s } void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& target) { + if (program.Info().usesFaultBuffer && target.bdaAbiVersion != BdaAbi::Version) throw std::runtime_error("the fault buffer needs the BDA fault ABI, which the target lacks"); if (!program.Info().usesDma) return; if (target.bdaAbiVersion != BdaAbi::Version) throw std::runtime_error("unsupported BDA ABI version"); for (const auto capability : {spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}) { @@ -197,10 +198,28 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std:: return EmitBdaBytes(state, address, bits / 8u, instruction, BdaAccessMask(state, inst)); } -void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value) { +void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits) { auto& state = ctx.state; if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); const auto instruction = ConstantU32(state, inst.Flags().pc); + if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA write width"); + if (bits != 32u) { + EmitBdaOverflowCheck(state, address, bits / 8u, instruction); + for (std::uint32_t byte = 0u; byte < bits / 8u; ++byte) { + const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, BdaConstant(state, byte)); + const auto physical = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaWritePointerFunction, guest, ConstantU32(state, 1u), instruction); + EmitIfCondition(state, Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)), [&] { + const auto byteType = state.module.Type(spv::OpTypeInt, 8u, 0u); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, byteType), pointer, physical); + const auto shifted = Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, ConstantU32(state, byte * 8u)); + state.module.AddFunction(spv::OpStore, pointer, Unary(state, spv::OpUConvert, byteType, shifted), BdaAccessMask(state, inst), 1u); + state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, guest); + }); + } + return; + } const auto unaligned = BdaAddressUnaligned(state, address); EmitIfCondition(state, unaligned, [&] { RecordBdaFault(state, address, ConstantU32(state, 4u), instruction, BdaAbi::FaultReason::Unaligned); }); EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), [&] { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index 596d281b1..b663800a2 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -229,11 +229,11 @@ std::uint32_t SpirvValueEmitContext::Label(const IrBlock* block) const { throw std::runtime_error("SPIR-V emission failed: hash=0x" + std::to_string(state.program.Resources().shaderHash) + " stage=" + std::to_string(static_cast(state.program.Resources().stage)) + " opcode=" + std::string(IrOpcodeName(inst.Opcode())) + " reason=" + reason); } -std::vector SpirvEmitter::Emit(const IrProgram& program, const BindingAllocationResult& bindings, const SpirvTargetOptions& target) const { +std::vector SpirvEmitter::Emit(const IrProgram& program, const CompiledBindingLayout& bindings, const SpirvTargetOptions& target) const { return Emit(program, ShaderStageInputInfo {}, bindings, target); } -std::vector SpirvEmitter::Emit(const IrProgram& program, const ShaderStageInputInfo& inputInfo, const BindingAllocationResult& bindings, const SpirvTargetOptions& target) const { +std::vector SpirvEmitter::Emit(const IrProgram& program, const ShaderStageInputInfo& inputInfo, const CompiledBindingLayout& bindings, const SpirvTargetOptions& target) const { if (program.Resources().stage != IrShaderStage::Compute && program.Resources().stage != IrShaderStage::Vertex && program.Resources().stage != IrShaderStage::Pixel && program.Resources().stage != IrShaderStage::Mesh && program.Resources().stage != IrShaderStage::Local && program.Resources().stage != IrShaderStage::TessellationControl && program.Resources().stage != IrShaderStage::TessellationEvaluation) { FailProgram(program, "binary SPIR-V emitter received an unsupported shader stage"); } @@ -248,6 +248,7 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh state.supportedCapabilities = target.supportedCapabilities; state.supportedExtensions = target.supportedExtensions; state.nonConstantImageOffsets = target.nonConstantImageOffsets; + state.narrowSubgroupClock = target.narrowSubgroupClock; state.hostSubgroupSize = target.subgroupSize; state.splitSubgroup = program.WaveSize() == 32u && target.subgroupSize > 32u; if (state.splitSubgroup && (state.requirements.subgroupBallot || state.requirements.subgroupShuffle)) state.requirements.subgroupLocalInvocationId = true; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 496f4658d..8dc88285f 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -85,9 +85,13 @@ void EmitReturnTerminator(SpirvValueEmitContext& ctx) { const auto pc = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, TypeU32(state), pc, state.loopGuardPc); EmitIfCondition(state, Binary(state, spv::OpINotEqual, TypeBool(state), pc, ConstantU32(state, 0u)), [&] { - RecordBdaFault(state, BdaConstant(state, state.program.Resources().shaderHash), ConstantU32(state, state.loopGuardLimit), EmitBinaryU32(state, spv::OpISub, pc, ConstantU32(state, 1u)), BdaAbi::FaultReason::LoopLimit); + const auto hash = state.program.Resources().shaderHash; + RecordBdaFaultWords(state, ConstantU32(state, static_cast(hash)), ConstantU32(state, static_cast(hash >> 32u)), ConstantU32(state, state.loopGuardLimit), EmitBinaryU32(state, spv::OpISub, pc, ConstantU32(state, 1u)), BdaAbi::FaultReason::LoopLimit); }); } + if (OrderedPixelShader(state)) { + state.module.AddFunction(spv::OpEndInvocationInterlockEXT); + } EmitKillIfPixelValidMaskInactive(state); state.module.AddFunction(spv::OpReturn); } @@ -473,6 +477,8 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::FPSqrt64: return Invoke(EmitFPSqrt64, ctx, inst); case IrOpcode::FPTrigPreop64: return Invoke(EmitFPTrigPreop64, ctx, inst); case IrOpcode::FPDot2F32F16: return Invoke(EmitFPDot2F32F16, ctx, inst); + case IrOpcode::FPInterpolateF32: return Invoke(EmitFPInterpolateF32, ctx, inst); + case IrOpcode::FPInterpolateF16: return Invoke(EmitFPInterpolateF16, ctx, inst); case IrOpcode::ConvertF32F64: return Invoke(EmitConvertF32F64, ctx, inst); case IrOpcode::ConvertF64F32: return Invoke(EmitConvertF64F32, ctx, inst); case IrOpcode::ConvertF64S32: return Invoke(EmitConvertF64S32, ctx, inst); @@ -669,6 +675,7 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::ImageAtomicCmpSwap64: return Invoke(EmitImageAtomicCmpSwap64, ctx, inst); case IrOpcode::GetAttribute: return Invoke(EmitGetAttribute, ctx, inst); case IrOpcode::GetInterpolationParameter: return Invoke(EmitGetInterpolationParameter, ctx, inst); + case IrOpcode::GetInterpolationParameterF16: return Invoke(EmitGetInterpolationParameterF16, ctx, inst); case IrOpcode::SetAttribute: return Invoke(EmitSetAttribute, ctx, inst); case IrOpcode::ControlNop: return Invoke(EmitControlNop, ctx, inst); case IrOpcode::Waitcnt: return Invoke(EmitWaitcnt, ctx, inst); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index 9c8599186..e9fde5d68 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -1,4 +1,6 @@ #include "SpirvBackend/SpirvImageEmitter.hpp" +#include "Optimization/ResourceMaterializer.hpp" +#include "PipelineSpecialization.hpp" #include "SpirvBackend/SpirvBda.hpp" #include "SpirvBackend/SpirvBufferFormat.hpp" #include "SpirvBackend/SpirvEmitterInstructions.hpp" @@ -44,6 +46,28 @@ struct SampleSetup { std::uint32_t coord; }; +std::uint32_t RuntimeImageDword(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t member) { + const auto index = state.runtimeImageMetadata != 0u ? state.runtimeImageMetadata : ConstantU32(state, resource); + const auto dword = Binary(state, spv::OpIAdd, TypeU32(state), ConstantU32(state, state.program.Metadata().bindings.ImageMetadataDword() + member), Binary(state, spv::OpIMul, TypeU32(state), index, ConstantU32(state, sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)))); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.shaderDataStorageVariable, ConstantU32(state, 0u), dword); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); + return value; +} + +std::uint32_t RuntimeImageSwizzle(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t component) { + if (state.program.Info().images.at(resource).indirectRoot == ImageResource::NoIndirectImage) return state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::ImageBase + resource * PipelineSpecialization::ImageWords + 1u + component, component + 4u); + const auto descriptor = RuntimeImageDword(state, resource, offsetof(RuntimeAbi::ResourceMetadata, descriptor) / sizeof(std::uint32_t) + 3u); + return Binary(state, spv::OpBitwiseAnd, TypeU32(state), Binary(state, spv::OpShiftRightLogical, TypeU32(state), descriptor, ConstantU32(state, component * 3u)), ConstantU32(state, 7u)); +} + +std::uint32_t RuntimeSwizzledComponent(SpirvEmitterState& state, std::uint32_t selector, const std::array& components) { + auto result = Select(state, TypeU32(state), Binary(state, spv::OpIEqual, TypeBool(state), selector, ConstantU32(state, 1u)), ConstantU32(state, 1u), ConstantU32(state, 0u)); + for (std::uint32_t index = 0u; index < components.size(); ++index) result = Select(state, TypeU32(state), Binary(state, spv::OpIEqual, TypeBool(state), selector, ConstantU32(state, index + 4u)), components[index], result); + return result; +} + std::uint32_t EffectiveDmask(const MemoryInfo& mem) { return mem.dmask != 0u ? mem.dmask : 1u; } @@ -190,17 +214,19 @@ std::uint32_t CoordF32(SpirvValueEmitContext& ctx, const ImageEmitAccess& access std::uint32_t CoordU32(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { const auto components = RdnaImageDimensionInfoFor(access.image.dimension).coordinateComponents; - if (AddressDimension(access).coordinateComponents < components || access.mem.imageAddressComponents < components) { + const auto encoded = std::min(components, AddressDimension(access).coordinateComponents); + if (access.mem.imageAddressComponents < encoded) { ctx.Fail(access.inst, "has an image address with too few coordinate components"); } - const auto x = AddressU32(ctx, access, 0); + const auto component = [&](std::uint32_t index) { return index < encoded ? AddressU32(ctx, access, index) : ConstantU32(ctx.state, 0); }; + const auto x = component(0); if (components == 1u) { return x; } - const auto y = AddressU32(ctx, access, 1); + const auto y = component(1); const auto result = ctx.state.module.AllocateId(); if (components == 3u) { - const auto z = AddressU32(ctx, access, 2); + const auto z = component(2); ctx.state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 3), result, x, y, z); } else { ctx.state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(ctx.state, 2), result, x, y); @@ -308,19 +334,19 @@ std::uint32_t ResultVector(SpirvValueEmitContext& ctx, const ImageEmitAccess& ac component[index] = index == 0u ? F32BitsToU32(ctx, value) : ConstantU32(state, 0); continue; } - const auto selector = (access.image.shaderSwizzle >> (index * 3u)) & 7u; - if (access.image.depthBits && !gather && selector != 4u) { - component[index] = ConstantU32(state, selector == 1u || selector == 7u ? 1u : 0u); - continue; - } const auto scalar = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeExtract, ImageScalarType(state, valueClass), scalar, value, index); if (access.image.depthUnorm16) { const auto scaled = Binary(state, spv::OpFMul, TypeF32(state), scalar, ConstantF32Value(state, 65535.0f)); component[index] = Unary(state, spv::OpConvertFToU, TypeU32(state), Binary(state, spv::OpFAdd, TypeF32(state), scaled, ConstantF32Value(state, 0.5f))); - continue; + } else { + component[index] = SampledComponentBits(ctx, scalar, valueClass); + } + if (access.image.depthBits && !gather) { + const auto selector = RuntimeImageSwizzle(state, mem.resource, index); + const auto one = Binary(state, spv::OpLogicalOr, TypeBool(state), Binary(state, spv::OpIEqual, TypeBool(state), selector, ConstantU32(state, 1u)), Binary(state, spv::OpIEqual, TypeBool(state), selector, ConstantU32(state, 7u))); + component[index] = Select(state, TypeU32(state), Binary(state, spv::OpIEqual, TypeBool(state), selector, ConstantU32(state, 4u)), component[index], Select(state, TypeU32(state), one, ConstantU32(state, 1u), ConstantU32(state, 0u))); } - component[index] = SampledComponentBits(ctx, scalar, valueClass); } const auto result = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), result, component[0], component[1], component[2], component[3]); @@ -462,30 +488,21 @@ std::uint32_t HorizontalOffsets(SpirvValueEmitContext& ctx, const ImageEmitAcces return state.module.Constant(spv::OpConstantComposite, arrayType, offsets[0], offsets[1], offsets[2], offsets[3]); } -std::uint32_t InverseSwizzle(std::uint32_t swizzle, std::uint32_t component) { - for (std::uint32_t source = 0; source < 4u; source++) { - if (((swizzle >> (source * 3u)) & 7u) == 4u + component) { - return source; - } - } - return NoImageComponent; -} - SpirvBufferFormatInfo ImageConversionFormat(const ImageResource& image) { const auto format = image.conversionFormat; if (format == IrBufferFormat::Invalid) { return {}; } const auto info = GetFormatInfo(format); - if (SampledTextureNumericClass(format) != IrTextureNumericClass::Uint || RemapTextureFormat(format) == format || (info.type != SpirvFormatComponentType::Uint && info.type != SpirvFormatComponentType::Unorm) || !info.packedBitfield || info.byteSize != sizeof(std::uint32_t) || info.componentCount == 0u || info.componentCount > 4u) { - throw std::runtime_error("image conversion format is not a packed 32-bit unsigned integer or unorm format"); + if (SampledTextureNumericClass(format) != IrTextureNumericClass::Uint || RemapTextureFormat(format) == format || (info.type != SpirvFormatComponentType::Uint && info.type != SpirvFormatComponentType::Unorm && info.type != SpirvFormatComponentType::Float) || !info.packedBitfield || info.byteSize != sizeof(std::uint32_t) || info.componentCount == 0u || info.componentCount > 4u) { + throw std::runtime_error("image conversion format is not a packed 32-bit unsigned integer, unorm or float format"); } return info; } void RequireConvertedUnormAccess(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, const SpirvBufferFormatInfo& info) { if (access.mem.dataBits == 16u) { - ctx.Fail(access.inst, "reads or writes a converted unorm image with 16-bit data, which is not implemented"); + ctx.Fail(access.inst, info.type == SpirvFormatComponentType::Float ? "reads or writes a converted float image with 16-bit data, which is not implemented" : "reads or writes a converted unorm image with 16-bit data, which is not implemented"); } for (std::uint32_t component = 0; component < 4u; component++) { const auto selector = (access.image.shaderSwizzle >> (component * 3u)) & 7u; @@ -504,13 +521,13 @@ std::uint32_t UnpackImageTexel(SpirvValueEmitContext& ctx, const ImageEmitAccess if (info.format == IrBufferFormat::Invalid) { return texel; } - const bool unorm = info.type == SpirvFormatComponentType::Unorm; + const bool unorm = info.type == SpirvFormatComponentType::Unorm || info.type == SpirvFormatComponentType::Float; if (unorm) { RequireConvertedUnormAccess(ctx, access, info); } const auto packed = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), packed, texel, 0u); - std::uint32_t components[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0)}; + std::array components{ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0)}; for (std::uint32_t component = 0; component < info.componentCount; component++) { components[component] = state.module.AllocateId(); state.module.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), components[component], packed, ConstantU32(state, info.componentBitOffset[component]), ConstantU32(state, info.componentBits[component])); @@ -521,17 +538,11 @@ std::uint32_t UnpackImageTexel(SpirvValueEmitContext& ctx, const ImageEmitAccess for (std::uint32_t component = info.componentCount; component < 4u; component++) { components[component] = components[component % info.componentCount]; } - const auto swizzle = access.image.shaderSwizzle; std::uint32_t selected[4] = {}; for (std::uint32_t component = 0; component < 4u; component++) { - const auto selector = (swizzle >> (component * 3u)) & 7u; - if (selector == 1u) { - selected[component] = ConstantU32(state, FormattedConstantBits(info, SpirvFormattedSourceKind::One)); - } else if (selector >= 4u) { - selected[component] = components[selector - 4u]; - } else { - selected[component] = ConstantU32(state, 0u); - } + const auto selector = RuntimeImageSwizzle(state, access.mem.resource, component); + const auto value = RuntimeSwizzledComponent(state, selector, components); + selected[component] = Select(state, TypeU32(state), Binary(state, spv::OpIEqual, TypeBool(state), selector, ConstantU32(state, 1u)), ConstantU32(state, FormattedConstantBits(info, SpirvFormattedSourceKind::One)), value); } const auto result = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), result, selected[0], selected[1], selected[2], selected[3]); @@ -602,18 +613,18 @@ std::uint32_t UnpackImageGather(SpirvValueEmitContext& ctx, const ImageEmitAcces return gathered; } const auto component = ImageGatherComponent(EffectiveDmask(access.mem)); - const auto selector = (access.image.shaderSwizzle >> (component * 3u)) & 7u; - if (selector < 4u) { - const auto value = ConstantU32(state, selector == 1u ? 1u : 0u); - return state.module.Constant(spv::OpConstantComposite, TypeU32Vector(state, 4), value, value, value, value); - } + const auto selector = RuntimeImageSwizzle(state, access.mem.resource, component); std::uint32_t values[4] = {}; for (std::uint32_t lane = 0; lane < 4u; lane++) { const auto packed = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), packed, gathered, lane); - values[lane] = state.module.AllocateId(); - const auto physical = (selector - 4u) % info.componentCount; - state.module.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), values[lane], packed, ConstantU32(state, info.componentBitOffset[physical]), ConstantU32(state, info.componentBits[physical])); + std::array components{}; + for (std::uint32_t index = 0u; index < components.size(); ++index) { + const auto physical = index % info.componentCount; + components[index] = state.module.AllocateId(); + state.module.AddFunction(spv::OpBitFieldUExtract, TypeU32(state), components[index], packed, ConstantU32(state, info.componentBitOffset[physical]), ConstantU32(state, info.componentBits[physical])); + } + values[lane] = RuntimeSwizzledComponent(state, selector, components); } const auto result = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), result, values[0], values[1], values[2], values[3]); @@ -674,7 +685,7 @@ std::uint32_t PackImageTexel(SpirvValueEmitContext& ctx, const ImageEmitAccess& if (info.format == IrBufferFormat::Invalid) { return texel; } - const bool unorm = info.type == SpirvFormatComponentType::Unorm; + const bool unorm = info.type == SpirvFormatComponentType::Unorm || info.type == SpirvFormatComponentType::Float; if (unorm) { RequireConvertedUnormAccess(ctx, access, info); } @@ -718,22 +729,25 @@ std::uint32_t StoreTexel(SpirvValueEmitContext& ctx, const ImageEmitAccess& acce if (sint.format != IrBufferFormat::Invalid && mem.dataBits == 16u) { ctx.Fail(access.inst, "stores 16-bit data to an image of a SINT format"); } - const auto swizzle = access.image.shaderSwizzle; std::uint32_t values[4] = {}; const auto dmask = EffectiveDmask(mem); for (std::uint32_t component = 0; component < 4u; component++) { - const auto source = InverseSwizzle(swizzle, component); auto raw = ConstantU32(state, 0); - if (source < 4u && ((dmask >> source) & 1u) != 0u) { + for (std::uint32_t reverse = 4u; reverse != 0u; --reverse) { + const auto source = reverse - 1u; const auto packedIndex = DmaskComponentIndex(dmask, source); - raw = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), raw, data, mem.dataBits == 16u ? packedIndex / 2u : packedIndex); - if (mem.dataBits == 16u) { - if ((packedIndex & 1u) != 0u) { - raw = Binary(state, spv::OpShiftRightLogical, TypeU32(state), raw, ConstantU32(state, 16u)); - } - raw = Binary(state, spv::OpBitwiseAnd, TypeU32(state), raw, ConstantU32(state, 0xffffu)); + auto candidate = ConstantU32(state, 0u); + if (((dmask >> source) & 1u) != 0u) { + candidate = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), candidate, data, mem.dataBits == 16u ? packedIndex / 2u : packedIndex); } + if (mem.dataBits == 16u && ((dmask >> source) & 1u) != 0u) { + if ((packedIndex & 1u) != 0u) { + candidate = Binary(state, spv::OpShiftRightLogical, TypeU32(state), candidate, ConstantU32(state, 16u)); + } + candidate = Binary(state, spv::OpBitwiseAnd, TypeU32(state), candidate, ConstantU32(state, 0xffffu)); + } + raw = Select(state, TypeU32(state), Binary(state, spv::OpIEqual, TypeBool(state), RuntimeImageSwizzle(state, mem.resource, source), ConstantU32(state, component + 4u)), candidate, raw); } values[component] = integer ? raw : mem.dataBits == 16u ? EmitF16BitsToF32(state, raw) : Unary(state, spv::OpBitcast, TypeF32(state), raw); if (component < sint.componentCount) { @@ -838,6 +852,9 @@ void EmitQueryLodOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { if (ImageConversionFormat(access.image).type == SpirvFormatComponentType::Unorm) { ctx.Fail(access.inst, "queries the level of detail of a converted unorm image, which is not implemented"); } + if (ImageConversionFormat(access.image).type == SpirvFormatComponentType::Float) { + ctx.Fail(access.inst, "queries the level of detail of a converted float image, which is not implemented"); + } state.module.EmitCapability(spv::CapabilityImageQuery); const auto sampled = MakeSampledImage(state, access.mem.resource, access.mem.sampler, access.slot); const auto coord = CoordF32(ctx, access, 0, RdnaImageDimensionInfoFor(access.image.dimension).spatialComponents, AddressDimension(access).spatialComponents); @@ -854,7 +871,82 @@ void EmitQueryLodOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { ctx.Define(access.inst, TableResult(ctx, access, result)); } +void EmitByReadOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { + auto& state = ctx.state; + const auto elements = access.mem.imageByElements; + const bool packed = access.mem.imagePacked; + const auto components = packed ? 1u : access.mem.dataDwords / elements; + if (access.image.dimension != RdnaImageDimension::Dim2D || components == 0u || (packed ? access.mem.dataDwords != 1u : components * elements != access.mem.dataDwords) || access.mem.dataBits != 32u) { + ctx.Fail(access.inst, "is a MIMG BY2/BY4 load outside the measured 2D, 32-bit data subset"); + } + state.module.EmitCapability(spv::CapabilityImageQuery); + const auto condition = ctx.Arg(access.inst, 2); + ctx.Define(access.inst, EmitValueOrDefaultIfCondition(state, condition, TypeU32Vector(state, 4), ConstantU32CompositeZero(state, 4), [&]() { + const auto image = LoadSampledImageDescriptor(state, access.mem.resource, access.slot); + const auto x = AddressU32(ctx, access, 0); + const auto y = AddressU32(ctx, access, 1); + const auto lod = LodU32(ctx, access); + const auto levels = state.module.AllocateId(); + state.module.AddFunction(spv::OpImageQueryLevels, TypeU32(state), levels, image); + const auto lodInside = Binary(state, spv::OpULessThan, TypeBool(state), lod, levels); + const auto queryLod = Select(state, TypeU32(state), lodInside, lod, ConstantU32(state, 0)); + const auto size = state.module.AllocateId(); + state.module.AddFunction(spv::OpImageQuerySizeLod, ImageViewSizeType(state, access.image.dimension), size, image, ConstantU32(state, 0)); + const auto levelExtent = [&](std::uint32_t axis) { + const auto base = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), base, size, axis); + const auto shifted = Binary(state, spv::OpShiftRightLogical, TypeU32(state), base, queryLod); + return Select(state, TypeU32(state), Binary(state, spv::OpUGreaterThan, TypeBool(state), shifted, ConstantU32(state, 1)), shifted, ConstantU32(state, 1)); + }; + const auto width = levelExtent(0u); + const auto height = levelExtent(1u); + auto inside = Binary(state, spv::OpLogicalAnd, TypeBool(state), lodInside, Binary(state, spv::OpULessThan, TypeBool(state), x, width)); + inside = Binary(state, spv::OpLogicalAnd, TypeBool(state), inside, Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), Binary(state, spv::OpISub, TypeU32(state), width, x), ConstantU32(state, elements))); + inside = Binary(state, spv::OpLogicalAnd, TypeBool(state), inside, Binary(state, spv::OpULessThan, TypeBool(state), y, height)); + const auto first = Select(state, TypeU32(state), inside, Binary(state, spv::OpBitwiseAnd, TypeU32(state), x, ConstantU32(state, ~(elements - 1u))), ConstantU32(state, 0)); + const auto row = Select(state, TypeU32(state), inside, y, ConstantU32(state, 0)); + MemoryInfo texelMemory = access.mem; + texelMemory.dmask = (1u << components) - 1u; + texelMemory.dataDwords = components; + texelMemory.componentCount = components; + const ImageEmitAccess texelAccess{access.inst, texelMemory, access.image, access.address, access.table, access.slot}; + std::uint32_t words[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0)}; + for (std::uint32_t element = 0; element < elements; ++element) { + const auto column = Binary(state, spv::OpIAdd, TypeU32(state), first, ConstantU32(state, element)); + const auto coord = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 2), coord, column, row); + const auto color = state.module.AllocateId(); + state.module.AddFunction(spv::OpImageFetch, ImageVectorType(state, access.image.numericClass, 4), color, image, coord, spv::ImageOperandsLodMask, queryLod); + if (packed) { + const auto info = PackedTexelFormat(ctx, access); + std::uint32_t texelBits = 0; + for (std::uint32_t component = 0; component < info.componentCount; ++component) texelBits += info.componentBits[component]; + if (texelBits * elements > 32u) ctx.Fail(access.inst, "is a MIMG PCK2/PCK4 load whose elements do not fit one dword"); + const auto raw = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), raw, PackedImageTexel(ctx, access, color), 0u); + const auto bits = Binary(state, spv::OpBitwiseAnd, TypeU32(state), raw, ConstantU32(state, texelBits == 32u ? 0xffffffffu : (1u << texelBits) - 1u)); + const auto shifted = element == 0u ? bits : Binary(state, spv::OpShiftLeftLogical, TypeU32(state), bits, ConstantU32(state, element * texelBits)); + words[0] = Binary(state, spv::OpBitwiseOr, TypeU32(state), words[0], Select(state, TypeU32(state), inside, shifted, ConstantU32(state, 0))); + continue; + } + const auto texel = ResultVector(ctx, texelAccess, UnpackImageTexel(ctx, texelAccess, color), access.image.numericClass, false, false); + for (std::uint32_t component = 0; component < components; ++component) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, texel, component); + words[element * components + component] = Select(state, TypeU32(state), inside, value, ConstantU32(state, 0)); + } + } + const auto result = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), result, words[0], words[1], words[2], words[3]); + return TableResult(ctx, access, result); + })); +} + void EmitReadOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { + if (access.mem.imageByElements != 0u) { + EmitByReadOp(ctx, access); + return; + } auto& state = ctx.state; const auto& dimensionInfo = RdnaImageDimensionInfoFor(access.image.dimension); const auto numericClass = access.image.numericClass; @@ -879,6 +971,13 @@ void EmitReadOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { })); } +std::uint32_t ImageSampleIndex(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { + if (RdnaImageDimensionInfoFor(access.image.dimension).multisampled == 0u) return ConstantU32(ctx.state, 0u); + const auto& addressInfo = AddressDimension(access); + if (addressInfo.multisampled == 0u || access.mem.imageAddressComponents <= addressInfo.coordinateComponents) ctx.Fail(access.inst, "has no sample index in the image address"); + return AddressU32(ctx, access, addressInfo.coordinateComponents); +} + void EmitWriteOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { auto& state = ctx.state; if (access.slot != 0) { @@ -889,7 +988,7 @@ void EmitWriteOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { const auto mipLod = access.image.mipMode == ImageMipMode::DynamicStorage ? LodU32(ctx, access) : 0u; const auto coord = CoordU32(ctx, access); const auto texel = access.mem.imagePacked ? PackedStoreTexel(ctx, access, ctx.Arg(access.inst, 2)) : StoreTexel(ctx, access, ctx.Arg(access.inst, 2), uintImage); - EmitStorageImageWrite(state, access.mem.resource, mipLod, coord, texel); + EmitStorageImageWrite(state, access.mem.resource, mipLod, coord, texel, ImageSampleIndex(ctx, access)); }); } @@ -922,7 +1021,7 @@ void EmitAtomicOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { ctx.Define(access.inst, EmitValueOrZeroIfCondition(state, condition, [&]() { const auto pointer = state.module.AllocateId(); const auto pointerType = TypePointer(state, spv::StorageClassImage, TypeU32(state)); - state.module.AddFunction(spv::OpImageTexelPointer, pointerType, pointer, StorageImageDescriptorPointer(state, access.mem.resource), CoordU32(ctx, access), ConstantU32(state, 0)); + state.module.AddFunction(spv::OpImageTexelPointer, pointerType, pointer, StorageImageDescriptorPointer(state, access.mem.resource), CoordU32(ctx, access), ImageSampleIndex(ctx, access)); if (opcode == IrOpcode::ImageAtomicInc32 || opcode == IrOpcode::ImageAtomicDec32) { return AtomicUpdate(state, pointer, ResourceKind::Image, [&](std::uint32_t current) { return opcode == IrOpcode::ImageAtomicInc32 ? AtomicIncrement(state, current, value) : AtomicDecrement(state, current, value); @@ -960,10 +1059,108 @@ SampleSetup MakeSampleSetup(SpirvValueEmitContext& ctx, const ImageEmitAccess& a if (ImageConversionFormat(access.image).type == SpirvFormatComponentType::Unorm) { ctx.Fail(access.inst, "samples or gathers a converted unorm image, which needs filtering in the shader and is not implemented"); } + if (ImageConversionFormat(access.image).type == SpirvFormatComponentType::Float) { + ctx.Fail(access.inst, "samples or gathers a converted float image, which needs filtering in the shader and is not implemented"); + } const auto coord = CoordF32(ctx, access, layout.coord, dimensionInfo.coordinateComponents, AddressDimension(access).coordinateComponents); return {dimensionInfo, layout, access.image.numericClass, dref, coord}; } +void EmitPackedHorizontalGatherOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { + auto& state = ctx.state; + const auto& mem = access.mem; + const auto dimension = access.image.dimension; + if (dimension != RdnaImageDimension::Dim1D && dimension != RdnaImageDimension::Dim2D) { + ctx.Fail(access.inst, "is a packed horizontal gather of an image dimension whose row and slice rules are not measured"); + } + if (Layout(mem).clamp != NoImageComponent || HasFlag(mem, RdnaImageSampleFlagCompare)) { + ctx.Fail(access.inst, "is a packed horizontal gather with depth comparison or an LOD clamp, which is not implemented"); + } + if (AddressDimension(access).coordinateComponents < RdnaImageDimensionInfoFor(dimension).coordinateComponents) { + ctx.Fail(access.inst, "has an image address with too few coordinate components"); + } + const auto info = PackedTexelFormat(ctx, access); + std::uint32_t texelBits = 0; + for (std::uint32_t component = 0; component < info.componentCount; component++) { + texelBits += info.componentBits[component]; + } + std::uint32_t streamWords[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0)}; + std::uint32_t rowInside = 0; + const bool twoDimensional = dimension == RdnaImageDimension::Dim2D; + if (texelBits <= 32u) { + state.module.EmitCapability(spv::CapabilityImageQuery); + const auto image = LoadSampledImageDescriptor(state, mem.resource, access.slot); + const auto size = state.module.AllocateId(); + state.module.AddFunction(spv::OpImageQuerySizeLod, twoDimensional ? TypeI32Vector(state, 2) : TypeI32(state), size, image, ConstantU32(state, 0)); + const auto extent = [&](std::uint32_t axis) { + if (!twoDimensional) { + return size; + } + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeI32(state), value, size, axis); + return value; + }; + const auto texelFloor = [&](std::uint32_t address, std::uint32_t length, bool centre) { + auto scaled = Binary(state, spv::OpFMul, TypeF32(state), AddressF32(ctx, access, address), Unary(state, spv::OpConvertSToF, TypeF32(state), length)); + state.module.AddAnnotation(spv::OpDecorate, scaled, spv::DecorationNoContraction); + if (centre) { + scaled = Binary(state, spv::OpFSub, TypeF32(state), scaled, ConstantF32(state, 0x3f000000u)); + } + const auto floored = state.module.AllocateId(); + state.module.AddFunction(spv::OpExtInst, TypeF32(state), floored, GlslStd450(state), GLSLstd450Floor, scaled); + return Unary(state, spv::OpConvertFToS, TypeI32(state), floored); + }; + const auto clampInto = [&](std::uint32_t value, std::uint32_t length) { + const auto last = Binary(state, spv::OpISub, TypeI32(state), length, ConstantI32(state, 1)); + const auto low = Select(state, TypeI32(state), Binary(state, spv::OpSLessThan, TypeBool(state), value, ConstantI32(state, 0)), ConstantI32(state, 0), value); + return Select(state, TypeI32(state), Binary(state, spv::OpSLessThan, TypeBool(state), last, low), last, low); + }; + const auto width = extent(0u); + const auto anchor = texelFloor(0u, width, true); + std::uint32_t row = 0; + if (twoDimensional) { + const auto height = extent(1u); + const auto rawRow = texelFloor(1u, height, false); + rowInside = Binary(state, spv::OpSLessThan, TypeBool(state), rawRow, height); + row = clampInto(rawRow, height); + } + for (std::uint32_t element = 0; element < 4u; element++) { + const auto column = clampInto(Binary(state, spv::OpIAdd, TypeI32(state), anchor, ConstantI32(state, static_cast(element) - 1)), width); + std::uint32_t coord = column; + if (twoDimensional) { + coord = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeI32Vector(state, 2), coord, column, row); + } + const auto color = state.module.AllocateId(); + state.module.AddFunction(spv::OpImageFetch, ImageVectorType(state, access.image.numericClass, 4), color, image, coord, spv::ImageOperandsLodMask, ConstantI32(state, 0)); + const auto packed = PackedImageTexel(ctx, access, color); + const auto bits = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), bits, packed, 0u); + const auto position = element * texelBits; + const auto word = position / 32u; + const auto shift = position % 32u; + streamWords[word] = Binary(state, spv::OpBitwiseOr, TypeU32(state), streamWords[word], shift == 0u ? bits : Binary(state, spv::OpShiftLeftLogical, TypeU32(state), bits, ConstantU32(state, shift))); + if (shift != 0u && shift + texelBits > 32u) { + streamWords[word + 1u] = Binary(state, spv::OpBitwiseOr, TypeU32(state), streamWords[word + 1u], Binary(state, spv::OpShiftRightLogical, TypeU32(state), bits, ConstantU32(state, 32u - shift))); + } + } + } + std::uint32_t selected[4] = {ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0), ConstantU32(state, 0)}; + std::uint32_t next = 0; + for (std::uint32_t bit = 0; bit < 4u; bit++) { + if (((mem.dmask >> bit) & 1u) == 0u) { + continue; + } + if (texelBits <= 32u) { + selected[next] = rowInside != 0u ? Select(state, TypeU32(state), rowInside, streamWords[bit], ConstantU32(state, 0)) : streamWords[bit]; + } + next++; + } + const auto result = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4), result, selected[0], selected[1], selected[2], selected[3]); + ctx.Define(access.inst, TableResult(ctx, access, result)); +} + void EmitGatherOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, const SampleSetup& setup) { auto& state = ctx.state; const auto& mem = access.mem; @@ -1021,10 +1218,6 @@ void EmitGatherOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access, con ctx.Define(access.inst, TableResult(ctx, access, ResultVector(ctx, access, UnpackImageGather(ctx, access, gathered), resultNumericClass, false, true))); } -// The bound slot of a bindless table for the runtime key: a binary search over the (key, slot) -// pairs the materializer appended to the flattened SRT. A key the mapping lacks (past the table, -// or a null or unusable entry) selects slot 0 and reports unmapped: its result is zeroed, as a -// null T# samples on hardware. TableSelection EmitIndirectImageSelector(SpirvValueEmitContext& ctx, const ImageResource& image, std::uint32_t key) { auto& state = ctx.state; const auto loadMapping = [&](std::uint32_t index) { @@ -1037,7 +1230,7 @@ TableSelection EmitIndirectImageSelector(SpirvValueEmitContext& ctx, const Image const auto mapping = ConstantU32(state, image.indirectMappingOffset); auto low = ConstantU32(state, 0u); auto high = loadMapping(mapping); - auto selected = ConstantU32(state, 0u); + auto selected = ConstantU32(state, image.indirectRoot); auto mapped = ConstantBool(state, false); for (std::uint32_t iteration = 0; iteration < image.indirectSearchIterations; iteration++) { const auto active = Binary(state, spv::OpULessThan, TypeBool(state), low, high); @@ -1063,10 +1256,10 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess auto& state = ctx.state; const auto& mem = access.mem; const auto& image = access.image; - const auto compare = image.emulatedCompare; + const auto parameter = [&](std::uint32_t word) { return state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::CompareBase + mem.resource * PipelineSpecialization::CompareWords + word, 0u); }; + const auto equal = [&](std::uint32_t value, std::uint32_t literal) { return Binary(state, spv::OpIEqual, TypeBool(state), value, ConstantU32(state, literal)); }; if (access.slot != 0) ctx.Fail(access.inst, "compares against a color texture through a bindless image table, which is not implemented"); if (HasFlag(mem, RdnaImageSampleFlagDerivative) || HasFlag(mem, RdnaImageSampleFlagLod)) ctx.Fail(access.inst, "compares against a color texture with gradients or an explicit LOD, which is not implemented"); - if (!HasFlag(mem, RdnaImageSampleFlagLevelZero) && (compare & EmulatedCompare::SingleLevel) == 0u) ctx.Fail(access.inst, "compares against a color texture across mip levels, which is not implemented"); const bool arrayed = image.dimension == RdnaImageDimension::Dim2DArray; if (image.dimension != RdnaImageDimension::Dim2D && !arrayed) ctx.Fail(access.inst, "compares against a color texture that is not a 2D or 2D array view, which is not implemented"); const auto f32 = TypeF32(state); @@ -1098,11 +1291,10 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess layer = Unary(state, spv::OpBitcast, u32, ext(i32, GLSLstd450SClamp, {rounded, ConstantI32(state, 0), Binary(state, spv::OpISub, i32, layers, ConstantI32(state, 1))})); } auto reference = DrefValueF32(ctx, access, setup.layout); - if (EmulatedCompare::Reference(compare) == EmulatedCompare::ReferenceUnorm) reference = ext(f32, GLSLstd450FClamp, {reference, f32Constant(0.0f), f32Constant(1.0f)}); - if (EmulatedCompare::Reference(compare) == EmulatedCompare::ReferenceSnorm) reference = ext(f32, GLSLstd450FClamp, {reference, f32Constant(-1.0f), f32Constant(1.0f)}); + reference = Select(state, f32, equal(parameter(4u), EmulatedCompare::ReferenceUnorm), ext(f32, GLSLstd450FClamp, {reference, f32Constant(0.0f), f32Constant(1.0f)}), reference); + reference = Select(state, f32, equal(parameter(4u), EmulatedCompare::ReferenceSnorm), ext(f32, GLSLstd450FClamp, {reference, f32Constant(-1.0f), f32Constant(1.0f)}), reference); const auto address = [&](std::uint32_t index, std::uint32_t extent, std::uint32_t mode) { - if (mode == EmulatedCompare::AddressWrap) return Binary(state, spv::OpSMod, i32, index, extent); - return ext(i32, GLSLstd450SClamp, {index, ConstantI32(state, 0), Binary(state, spv::OpISub, i32, extent, ConstantI32(state, 1))}); + return Select(state, i32, equal(mode, EmulatedCompare::AddressWrap), Binary(state, spv::OpSMod, i32, index, extent), ext(i32, GLSLstd450SClamp, {index, ConstantI32(state, 0), Binary(state, spv::OpISub, i32, extent, ConstantI32(state, 1))})); }; const auto inside = [&](std::uint32_t index, std::uint32_t extent) { const auto notBelow = Binary(state, spv::OpSGreaterThanEqual, TypeBool(state), index, ConstantI32(state, 0)); @@ -1111,10 +1303,10 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess }; const auto one = f32Constant(1.0f); const auto zero = f32Constant(0.0f); - const auto borderRed = (compare & EmulatedCompare::BorderWhite) != 0u ? one : zero; + const auto borderRed = Select(state, f32, equal(parameter(5u), 1u), one, zero); const auto compareTexel = [&](std::uint32_t x, std::uint32_t y) { - const auto modeX = EmulatedCompare::AddressX(compare); - const auto modeY = EmulatedCompare::AddressY(compare); + const auto modeX = parameter(2u); + const auto modeY = parameter(3u); const auto ux = Unary(state, spv::OpBitcast, u32, address(x, width, modeX)); const auto uy = Unary(state, spv::OpBitcast, u32, address(y, height, modeY)); const auto coord = state.module.AllocateId(); @@ -1123,23 +1315,15 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess const auto texel = state.module.AllocateId(); state.module.AddFunction(spv::OpImageFetch, ImageVectorType(state, IrTextureNumericClass::Float, 4), texel, descriptor, coord, spv::ImageOperandsLodMask, ConstantU32(state, 0u)); auto red = extract(f32, texel, 0u); - std::uint32_t inBorder = 0u; - if (modeX == EmulatedCompare::AddressBorder) inBorder = Unary(state, spv::OpLogicalNot, TypeBool(state), inside(x, width)); - if (modeY == EmulatedCompare::AddressBorder) { - const auto outsideY = Unary(state, spv::OpLogicalNot, TypeBool(state), inside(y, height)); - inBorder = inBorder == 0u ? outsideY : Binary(state, spv::OpLogicalOr, TypeBool(state), inBorder, outsideY); - } - if (inBorder != 0u) red = Select(state, f32, inBorder, borderRed, red); - switch (EmulatedCompare::Function(compare)) { - case 0u: return zero; - case 1u: return Select(state, f32, Binary(state, spv::OpFOrdLessThan, TypeBool(state), reference, red), one, zero); - case 2u: return Select(state, f32, Binary(state, spv::OpFOrdEqual, TypeBool(state), reference, red), one, zero); - case 3u: return Select(state, f32, Binary(state, spv::OpFOrdLessThanEqual, TypeBool(state), reference, red), one, zero); - case 4u: return Select(state, f32, Binary(state, spv::OpFOrdGreaterThan, TypeBool(state), reference, red), one, zero); - case 5u: return Select(state, f32, Binary(state, spv::OpFOrdNotEqual, TypeBool(state), reference, red), one, zero); - case 6u: return Select(state, f32, Binary(state, spv::OpFOrdGreaterThanEqual, TypeBool(state), reference, red), one, zero); - default: return one; + red = Select(state, f32, equal(modeX, EmulatedCompare::AddressBorder), Select(state, f32, inside(x, width), red, borderRed), red); + red = Select(state, f32, equal(modeY, EmulatedCompare::AddressBorder), Select(state, f32, inside(y, height), red, borderRed), red); + constexpr std::array operations{spv::OpFOrdLessThan, spv::OpFOrdEqual, spv::OpFOrdLessThanEqual, spv::OpFOrdGreaterThan, spv::OpFOrdNotEqual, spv::OpFOrdGreaterThanEqual}; + auto result = Select(state, f32, equal(parameter(0u), 7u), one, zero); + for (std::uint32_t index = 0; index < operations.size(); ++index) { + const auto compared = Select(state, f32, Binary(state, operations[index], TypeBool(state), reference, red), one, zero); + result = Select(state, f32, equal(parameter(0u), index + 1u), compared, result); } + return result; }; const auto scaledU = Binary(state, spv::OpFMul, f32, extract(f32, setup.coord, 0u), Unary(state, spv::OpConvertSToF, f32, width)); const auto scaledV = Binary(state, spv::OpFMul, f32, extract(f32, setup.coord, 1u), Unary(state, spv::OpConvertSToF, f32, height)); @@ -1153,10 +1337,9 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess const auto texelIndex = [&](std::uint32_t coordinate, std::uint32_t offset) { return Binary(state, spv::OpIAdd, i32, Unary(state, spv::OpConvertFToS, i32, ext(f32, GLSLstd450Floor, {coordinate})), offset); }; - std::uint32_t result; - if ((compare & EmulatedCompare::Linear) == 0u) { - result = compareTexel(texelIndex(scaledU, offsetX), texelIndex(scaledV, offsetY)); - } else { + const auto result = EmitValueIfElse(state, equal(parameter(1u), 0u), f32, [&] { + return compareTexel(texelIndex(scaledU, offsetX), texelIndex(scaledV, offsetY)); + }, [&] { const auto half = f32Constant(0.5f); const auto centreU = Binary(state, spv::OpFSub, f32, scaledU, half); const auto centreV = Binary(state, spv::OpFSub, f32, scaledV, half); @@ -1170,8 +1353,8 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess const auto y1 = Binary(state, spv::OpIAdd, i32, y0, ConstantI32(state, 1)); const auto top = ext(f32, GLSLstd450FMix, {compareTexel(x0, y0), compareTexel(x1, y0), weightU}); const auto bottom = ext(f32, GLSLstd450FMix, {compareTexel(x0, y1), compareTexel(x1, y1), weightU}); - result = ext(f32, GLSLstd450FMix, {top, bottom, weightV}); - } + return ext(f32, GLSLstd450FMix, {top, bottom, weightV}); + }); ctx.Define(access.inst, TableResult(ctx, access, ResultVector(ctx, access, result, setup.numericClass, true, false))); } @@ -1291,6 +1474,10 @@ void EmitSamplingOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { if (access.image.srgbDecode) { ctx.Fail(access.inst, "samples or gathers an sRGB image the device cannot sample, which is not implemented"); } + if (access.mem.imagePacked) { + EmitPackedHorizontalGatherOp(ctx, access); + return; + } const auto setup = MakeSampleSetup(ctx, access); if (access.inst.Opcode() == IrOpcode::ImageGatherRaw) { EmitGatherOp(ctx, access, setup); @@ -1309,12 +1496,11 @@ void EmitImageOperation(SpirvValueEmitContext& context, const IrValue& value, st throw std::logic_error("EmitImageOperation(SpirvValueEmitContext&) is not used: image instructions are emitted per opcode through EmitImage"); } -void EmitImage(SpirvValueEmitContext& ctx, const IrValue& inst) { +void EmitImageMode(SpirvValueEmitContext& ctx, const IrValue& inst, const ImageResource& image) { const auto irOpcode = inst.Opcode(); const auto imageInfo = ImageOpcodeInfoOf(irOpcode); const auto& mem = ctx.Memory(inst); ctx.ResourceIndex(inst.Argument(0), IrOpcode::GetImageResource); - const auto& image = ImageResourceOf(ctx.state, mem); const auto* address = ctx.ImageAddress(inst.Argument(imageInfo.needsSampler ? 2 : 1)); if (address == nullptr) { ctx.Fail(inst, "has no image address"); @@ -1346,6 +1532,90 @@ void EmitImage(SpirvValueEmitContext& ctx, const IrValue& inst) { } } +void EmitImage(SpirvValueEmitContext& ctx, const IrValue& inst) { + auto& state = ctx.state; + const auto& memory = ctx.Memory(inst); + const auto& base = ImageResourceOf(state, memory); + const auto modes = ResourceMaterializer::RuntimeImageModes(base); + state.runtimeImageMetadata = ConstantU32(state, memory.resource); + const auto table = TableSlot(ctx, inst, memory, base); + if (table.slot != 0u) { + state.runtimeImageMetadata = table.slot; + } + const auto emitMode = [&](const ImageResource& mode) { + state.runtimeImage = &mode; + if (mode.packedFormat == IrBufferFormat::Fmask8_S2_F1) { + if (inst.Opcode() == IrOpcode::ImageRead && memory.dataBits == 32u) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4u), value, ConstantU32(state, 0x76543210u), ConstantU32(state, 0xfedcba98u), ConstantU32(state, 0u), ConstantU32(state, 0u)); + ctx.Define(inst, Select(state, TypeU32Vector(state, 4u), ctx.Arg(inst, 2u), value, ConstantU32CompositeZero(state, 4u))); + } else { + ctx.Fail(inst, "FMASK requires a 32-bit image read"); + } + } else if (mode.depthBits && memory.dataBits == 16u) { + ctx.Fail(inst, "depth bits require a 32-bit image result"); + } else { + EmitImageMode(ctx, inst, mode); + } + }; + if (modes.size() == 1u) { + emitMode(modes.front()); + state.runtimeImage = nullptr; + state.runtimeImageMetadata = 0u; + return; + } + const auto selector = base.indirectRoot == ImageResource::NoIndirectImage ? state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::ImageBase + memory.resource * PipelineSpecialization::ImageWords, 0u) : Binary(state, spv::OpShiftRightLogical, TypeU32(state), RuntimeImageDword(state, memory.resource, offsetof(RuntimeAbi::ResourceMetadata, flags) / sizeof(std::uint32_t)), ConstantU32(state, 1u)); + const bool returnsValue = inst.Opcode() != IrOpcode::ImageWrite; + const auto resultId = returnsValue ? ctx.Result(inst) : 0u; + ctx.definitions.erase(&inst); + const auto merge = state.module.AllocateId(); + const auto invalid = state.module.AllocateId(); + std::vector labels(modes.size()); + std::vector words{spv::OpSwitch, selector, invalid}; + for (std::uint32_t index = 0u; index < modes.size(); ++index) { + labels[index] = state.module.AllocateId(); + words.insert(words.end(), {index, labels[index]}); + } + state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + state.module.AddFunction(words); + std::vector incoming; + for (std::uint32_t index = 0u; index < modes.size(); ++index) { + EmitLabel(state, labels[index]); + std::uint32_t modeMerge = 0u; + if (base.indirectRoot != ImageResource::NoIndirectImage) { + const auto enabled = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::ImageModeBase + memory.resource * PipelineSpecialization::ImageModeStride + index, 1u); + const auto active = Binary(state, spv::OpINotEqual, TypeBool(state), enabled, ConstantU32(state, 0u)); + const auto activeLabel = state.module.AllocateId(); + const auto inactiveLabel = state.module.AllocateId(); + modeMerge = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, modeMerge, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, active, activeLabel, inactiveLabel); + EmitLabel(state, inactiveLabel); + state.module.AddFunction(spv::OpUnreachable); + EmitLabel(state, activeLabel); + } + emitMode(modes[index]); + if (modeMerge != 0u) { + state.module.AddFunction(spv::OpBranch, modeMerge); + EmitLabel(state, modeMerge); + } + if (returnsValue) incoming.insert(incoming.end(), {ctx.Def(&inst), state.currentLabel}); + ctx.definitions.erase(&inst); + state.module.AddFunction(spv::OpBranch, merge); + } + state.runtimeImage = nullptr; + state.runtimeImageMetadata = 0u; + EmitLabel(state, invalid); + state.module.AddFunction(spv::OpUnreachable); + EmitLabel(state, merge); + if (returnsValue) { + std::vector phi{spv::OpPhi, TypeId(state, inst.Type()), resultId}; + phi.insert(phi.end(), incoming.begin(), incoming.end()); + state.module.AddFunction(phi); + ctx.definitions.emplace(&inst, resultId); + } +} + void EmitGetImageResource(SpirvValueEmitContext& context) { EmitVoid(context); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp index 5185ac073..7416ff383 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp @@ -1,3 +1,4 @@ +#include "PipelineSpecialization.hpp" #include "SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" @@ -36,7 +37,9 @@ std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dw const auto pointer = state.module.AllocateId(); const auto value = state.module.AllocateId(); if (layout.UsesPushData()) { - state.module.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer, state.pushConstantVariable, ConstantU32(state, 0), ConstantU32(state, dwordIndex + layout.pushDataStartDword)); + const auto base = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::PushDataOffset, layout.pushDataStartDword); + const auto index = Binary(state, spv::OpIAdd, TypeU32(state), base, ConstantU32(state, dwordIndex)); + state.module.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer, state.pushConstantVariable, ConstantU32(state, 0), index); } else if (state.shaderDataStorageVariable != 0) { state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.shaderDataStorageVariable, ConstantU32(state, 0), ConstantU32(state, dwordIndex)); } else { @@ -46,20 +49,6 @@ std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dw return value; } -std::uint32_t StorageBufferPackedStride(const SpirvEmitterState& state, const MemoryInfo& mem) { - if (mem.resource >= state.program.Info().buffers.size()) { - ExitDescriptorBindingFailure(state, DescriptorBindingKind::Buffers, mem.resource, "buffer specialization is missing"); - } - return state.program.Info().buffers[mem.resource].packedStride; -} - -IrBufferFormat StorageBufferFormat(const SpirvEmitterState& state, const MemoryInfo& mem) { - if (mem.resource >= state.program.Info().buffers.size()) { - ExitDescriptorBindingFailure(state, DescriptorBindingKind::Buffers, mem.resource, "buffer specialization is missing"); - } - return state.program.Info().buffers[mem.resource].descriptorFormat; -} - void EmitMemoryOffsets(SpirvEmitterState& state) { const IrBindingLayout& layout = state.program.Metadata().bindings; state.memoryByteOffsets.assign(layout.memoryOffsetCount, 0u); @@ -86,25 +75,6 @@ std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state) { return pointer; } -MemoryResourceAccess PrepareStorageBufferResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem, std::uint32_t variable, std::uint32_t pointerType) { - if (variable == 0) { - ExitDescriptorBindingFailure(state, DescriptorBindingKind::Buffers, mem.resource, "storage buffer descriptor array was not emitted"); - } - const auto arrayIndex = ResourceForDescriptor(state, DescriptorBindingKind::Buffers, mem.resource); - MemoryResourceAccess access; - access.kind = mem.kind; - access.objectPointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, pointerType, access.objectPointer, variable, ConstantU32(state, arrayIndex)); - access.byteOffset = state.memoryByteOffsets.at(arrayIndex); - access.memoryAccess = mem.coherent ? spv::MemoryAccessVolatileMask : 0u; - access.length = state.module.AllocateId(); - state.module.AddFunction(spv::OpArrayLength, TypeU32(state), access.length, access.objectPointer, 0u); - if (mem.resource < state.program.Info().buffers.size() && state.program.Info().buffers[mem.resource].empty) { - access.length = ConstantU32(state, 0u); - } - return access; -} - MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem) { MemoryResourceAccess access; access.kind = mem.kind; @@ -137,10 +107,7 @@ MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const FailEmit("physical address memory must use the BDA emitter"); case ResourceKind::ScalarBuffer: case ResourceKind::Buffer: - access = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferVariable, TypeStorageBufferPointer(state)); - access.indexOffset = EmitBinaryU32(state, spv::OpShiftRightLogical, access.byteOffset, ConstantU32(state, 2u)); - access.addIndexOffset = true; - return access; + FailEmit("buffer memory must use the runtime V# emitter"); default: FailEmit("unsupported memory resource kind " + std::to_string(static_cast(mem.kind))); } @@ -151,8 +118,9 @@ std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResou } std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index) { + const auto last = access.misalignment != 0u ? EmitAddU32(state, index, ConstantU32(state, 1u)) : index; const auto inBounds = state.module.AllocateId(); - state.module.AddFunction(spv::OpULessThan, TypeBool(state), inBounds, index, access.length); + state.module.AddFunction(spv::OpULessThan, TypeBool(state), inBounds, last, access.length); return inBounds; } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp index f74ae1b0b..b1a37bb57 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp @@ -3,6 +3,7 @@ #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "Optimization/ResourceMaterializer.hpp" +#include "PipelineSpecialization.hpp" #include #include #include @@ -48,15 +49,13 @@ std::uint32_t DescriptorElementPointer(SpirvEmitterState& state, std::uint32_t r ExitDescriptorBindingFailure(state, kind, resource, variableName); } const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, resultPtrType, pointer, variableId, ConstantU32(state, arrayIndex)); + const bool heap = kind == DescriptorBindingKind::Samplers || ImageBindingResourceClass(kind) != ImageResourceClass::None; + const auto index = heap ? state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::DescriptorIndex(static_cast(kind), arrayIndex), arrayIndex) : ConstantU32(state, arrayIndex); + state.module.AddFunction(spv::OpAccessChain, resultPtrType, pointer, variableId, index); return pointer; } -// The element of a bindless table's runtime slot, `arrayIndex` (the root's element) + `slotId`. -// The slot is wave-uniform; where that is not uniform over the invocation group (see -// SpirvEmitterState::tableIndexNonUniform) the index carries NonUniform, which needs -// VK_EXT_descriptor_indexing on the device. -std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind kind, std::uint32_t resource, std::uint32_t arrayIndex, std::uint32_t slotId) { +std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind kind, std::uint32_t resource) { const bool storage = ImageBindingResourceClass(kind) == ImageResourceClass::Storage; const auto supported = [&](std::uint32_t capability) { return std::find(state.supportedCapabilities.begin(), state.supportedCapabilities.end(), capability) != state.supportedCapabilities.end(); @@ -66,7 +65,12 @@ std::uint32_t TableImageIndex(SpirvEmitterState& state, DescriptorBindingKind ki ExitDescriptorBindingFailure(state, kind, resource, "bindless image table needs image array dynamic indexing, which the device lacks"); } state.module.EmitCapability(dynamic); - const auto index = Binary(state, spv::OpIAdd, TypeU32(state), ConstantU32(state, arrayIndex), slotId); + if (state.runtimeImageMetadata == 0u) FailEmit("bindless image has no runtime metadata index"); + const auto offset = Binary(state, spv::OpIAdd, TypeU32(state), ConstantU32(state, state.program.Metadata().bindings.ImageMetadataDword() + offsetof(RuntimeAbi::ResourceMetadata, firstElement) / sizeof(std::uint32_t)), Binary(state, spv::OpIMul, TypeU32(state), state.runtimeImageMetadata, ConstantU32(state, sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)))); + const auto metadata = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), metadata, state.shaderDataStorageVariable, ConstantU32(state, 0u), offset); + const auto index = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), index, metadata); if (!state.tableIndexNonUniform) { return index; } @@ -124,6 +128,13 @@ std::uint32_t ImageType(SpirvEmitterState& state, const ImageResource& image) { FailEmit("invalid image resource class"); } const auto& info = RdnaImageDimensionInfoFor(image.dimension); + if (info.multisampled != 0u) { + if (sampled == 2u) { + if (std::ranges::find(state.supportedCapabilities, spv::CapabilityStorageImageMultisample) == state.supportedCapabilities.end()) FailEmit("storage image multisampling is unavailable on the target device"); + state.module.EmitCapability(spv::CapabilityStorageImageMultisample); + } + if (info.arrayed != 0u) state.module.EmitCapability(spv::CapabilityImageMSArray); + } const auto scalar = image.atomic64 ? TypeScalarU64(state) : ImageScalarType(state, image.numericClass); return state.module.Type(spv::OpTypeImage, scalar, info.spirvDimension, image.depthCompare ? 1u : 0u, info.arrayed, info.multisampled, sampled, format); } @@ -138,7 +149,7 @@ std::uint32_t ImageViewSizeType(SpirvEmitterState& state, RdnaImageDimension dim } std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t slotId) { - const auto& imageResource = state.program.Info().images.at(resource); + const auto& imageResource = state.runtimeImage != nullptr ? *state.runtimeImage : state.program.Info().images.at(resource); if (imageResource.resourceClass != ImageResourceClass::Sampled) { FailEmit("sampled image descriptor requested for a non-sampled image"); } @@ -155,7 +166,7 @@ std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t pointer = DescriptorElementPointer(state, pointerType, variable, arrayIndex, kind, resource, "sampled image descriptor array was not emitted"); } else { pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, variable, TableImageIndex(state, kind, resource, arrayIndex, slotId)); + state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, variable, TableImageIndex(state, kind, resource)); DecorateTableNonUniform(state, slotId, pointer); } const auto image = state.module.AllocateId(); @@ -165,7 +176,8 @@ std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t } std::uint32_t LoadSamplerDescriptor(SpirvEmitterState& state, std::uint32_t sampler) { - const auto arrayIndex = ResourceForDescriptor(state, DescriptorBindingKind::Samplers, sampler); + const bool point = state.runtimeImage != nullptr && (state.runtimeImage->numericClass == IrTextureNumericClass::Sint || state.runtimeImage->conversionFormat != IrBufferFormat::Invalid || state.runtimeImage->depthBits); + const auto arrayIndex = ResourceForDescriptor(state, DescriptorBindingKind::Samplers, sampler) + (point ? 1u : 0u); const auto samplerType = state.module.Type(spv::OpTypeSampler); const auto pointerType = state.module.Type(spv::OpTypePointer, spv::StorageClassUniformConstant, samplerType); const auto pointer = DescriptorElementPointer(state, pointerType, state.samplerVariable, arrayIndex, DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted"); @@ -175,7 +187,7 @@ std::uint32_t LoadSamplerDescriptor(SpirvEmitterState& state, std::uint32_t samp } std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t sampler, std::uint32_t slotId) { - const auto& imageResource = state.program.Info().images.at(resource); + const auto& imageResource = state.runtimeImage != nullptr ? *state.runtimeImage : state.program.Info().images.at(resource); const auto image = LoadSampledImageDescriptor(state, resource, slotId); const auto samplerId = LoadSamplerDescriptor(state, sampler); const auto sampledImage = state.module.AllocateId(); @@ -186,7 +198,7 @@ std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, } std::uint32_t StorageImageDescriptorPointer(SpirvEmitterState& state, std::uint32_t resource) { - const auto& image = state.program.Info().images.at(resource); + const auto& image = state.runtimeImage != nullptr ? *state.runtimeImage : state.program.Info().images.at(resource); if (image.resourceClass != ImageResourceClass::Storage) { FailEmit("storage image descriptor requested for a non-storage image"); } @@ -197,8 +209,8 @@ std::uint32_t StorageImageDescriptorPointer(SpirvEmitterState& state, std::uint3 return DescriptorElementPointer(state, pointerType, variable, arrayIndex, kind, resource, "storage image descriptor array was not emitted"); } -void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t mipLod, std::uint32_t coord, std::uint32_t texel) { - const auto& image = state.program.Info().images.at(resource); +void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t mipLod, std::uint32_t coord, std::uint32_t texel, std::uint32_t sample) { + const auto& image = state.runtimeImage != nullptr ? *state.runtimeImage : state.program.Info().images.at(resource); if (image.resourceClass != ImageResourceClass::Storage) { FailEmit("storage image write requested for a non-storage image"); } @@ -216,6 +228,11 @@ void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std state.module.AddFunction(spv::OpLoad, imageType, descriptor, pointer); return descriptor; }; + if (RdnaImageDimensionInfoFor(image.dimension).multisampled != 0u) { + if (sample == 0u || image.mipMode == ImageMipMode::DynamicStorage) FailEmit("multisampled storage image requires a sample and cannot have mip levels"); + state.module.AddFunction(spv::OpImageWrite, loadAt(arrayIndex), coord, texel, spv::ImageOperandsSampleMask, sample); + return; + } if (image.mipMode != ImageMipMode::DynamicStorage) { state.module.AddFunction(spv::OpImageWrite, loadAt(arrayIndex), coord, texel); return; @@ -235,7 +252,16 @@ void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std state.module.AddFunction(std::span(words)); for (std::uint32_t mip = 0; mip < image.mipCount; mip++) { EmitLabel(state, labels[mip]); + const auto count = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::MipCountBase + resource, image.mipCount); + const auto active = Binary(state, spv::OpULessThan, TypeBool(state), ConstantU32(state, mip), count); + const auto writeLabel = state.module.AllocateId(); + const auto nextLabel = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, nextLabel, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, active, writeLabel, nextLabel); + EmitLabel(state, writeLabel); state.module.AddFunction(spv::OpImageWrite, loadAt(arrayIndex + mip), coord, texel); + state.module.AddFunction(spv::OpBranch, nextLabel); + EmitLabel(state, nextLabel); state.module.AddFunction(spv::OpBranch, mergeLabel); } EmitLabel(state, mergeLabel); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp index 28a7bdf07..efb0d8e3f 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp @@ -1,4 +1,6 @@ +#include "Optimization/ResourceMaterializer.hpp" #include "BdaAbi.hpp" +#include "PipelineSpecialization.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" @@ -124,7 +126,7 @@ std::uint32_t PushConstantBlockType(SpirvEmitterState& state) { } -void CheckBindings(const IrProgram& program, const BindingAllocationResult& bindings) { +void CheckBindings(const IrProgram& program, const CompiledBindingLayout& bindings) { const IrProgramMetadata& metadata = program.Metadata(); if (!metadata.bindingLayoutComplete) { FailEmit("shader binding layout has not been allocated for this program"); @@ -132,59 +134,6 @@ void CheckBindings(const IrProgram& program, const BindingAllocationResult& bind if (!(bindings.layout == metadata.bindings)) { FailEmit("binding allocation result does not match the program's committed binding layout"); } - if (bindings.bindings.size() != bindings.layout.descriptors.size()) { - FailEmit("binding allocation result has a different descriptor count than the binding layout"); - } - const IrShaderStage stage = program.Resources().stage; - for (std::size_t index = 0; index < bindings.layout.descriptors.size(); index++) { - const IrDescriptorBinding& logical = bindings.layout.descriptors[index]; - const DescriptorBinding& physical = bindings.bindings[index]; - const std::uint32_t expectedCount = logical.resources.empty() ? 1u : static_cast(logical.resources.size()); - if (physical.count != expectedCount) { - FailEmit("descriptor binding " + std::to_string(index) + " has an incorrect descriptor count"); - } - if (physical.descriptorSet != 0u) { - FailEmit("descriptor binding " + std::to_string(index) + " is bound to the wrong descriptor set"); - } - if (physical.binding != NativeBinding(stage, logical.kind)) { - FailEmit("descriptor binding " + std::to_string(index) + " is bound to the wrong native binding slot"); - } - DescriptorKind expectedKind = DescriptorKind::StorageBuffer; - DescriptorRole expectedRole = DescriptorRole::GuestBuffers; - if (logical.kind == DescriptorBindingKind::Samplers) { - expectedKind = DescriptorKind::Sampler; - expectedRole = DescriptorRole::GuestSamplers; - } else if (logical.kind == DescriptorBindingKind::Gds) { - expectedRole = DescriptorRole::Gds; - } else if (logical.kind == DescriptorBindingKind::BdaPagetable) { - expectedRole = DescriptorRole::BdaPagetable; - } else if (logical.kind == DescriptorBindingKind::FaultBuffer) { - expectedRole = DescriptorRole::FaultBuffer; - } else if (logical.kind == DescriptorBindingKind::FlattenedSrt) { - expectedRole = DescriptorRole::FlattenedSrt; - } else if (logical.kind == DescriptorBindingKind::ShaderData) { - expectedRole = DescriptorRole::ShaderData; - } else { - const auto imageClass = ImageBindingResourceClass(logical.kind); - if (imageClass == ImageResourceClass::Sampled) { - expectedKind = DescriptorKind::SampledImage; - expectedRole = DescriptorRole::GuestImages; - } else if (imageClass == ImageResourceClass::Storage) { - expectedKind = DescriptorKind::StorageImage; - expectedRole = DescriptorRole::GuestImages; - } else if (logical.kind == DescriptorBindingKind::Buffers) { - expectedRole = DescriptorRole::GuestBuffers; - } else { - FailEmit("descriptor binding " + std::to_string(index) + " has an unmapped binding kind"); - } - } - if (physical.kind != expectedKind) { - FailEmit("descriptor binding " + std::to_string(index) + " has an incorrect descriptor kind"); - } - if (physical.role != expectedRole) { - FailEmit("descriptor binding " + std::to_string(index) + " has an incorrect descriptor role"); - } - } } void EmitBaseHeader(SpirvModule& module, const IrProgram& program) { @@ -393,7 +342,10 @@ void DefineDescriptors(SpirvEmitterState& state) { return variable; }; const auto ArrayType = [&](std::uint32_t type) { - return state.module.Type(spv::OpTypeArray, type, ConstantU32(state, static_cast(binding.resources.size()))); + const bool heap = binding.kind == DescriptorBindingKind::Samplers || ImageBindingResourceClass(binding.kind) != ImageResourceClass::None; + const auto count = static_cast(binding.resources.size()); + const auto length = heap ? state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::HeapCountBase + static_cast(binding.kind), count) : ConstantU32(state, count); + return state.module.Type(spv::OpTypeArray, type, length); }; switch (binding.kind) { case DescriptorBindingKind::Buffers: @@ -430,7 +382,10 @@ void DefineDescriptors(SpirvEmitterState& state) { if (ImageBindingResourceClass(binding.kind) == ImageResourceClass::None) { FailEmit("descriptor binding has an unmapped image resource class"); } - const ImageResource& image = state.program.Info().images.at(binding.resources.front()); + const auto modes = ResourceMaterializer::RuntimeImageModes(state.program.Info().images.at(binding.resources.front())); + const auto selected = std::ranges::find_if(modes, [&](const ImageResource& mode) { return DescriptorBindingForImage(mode) == binding.kind; }); + if (selected == modes.end()) FailEmit("static image heap has no runtime mode"); + const auto& image = *selected; const auto name = "image_" + std::to_string(static_cast(binding.kind)); state.imageVariables.at(ImageBindingIndex(binding.kind)) = Define(ArrayType(ImageType(state, image)), name.c_str(), spv::StorageClassUniformConstant); if (image.dimension == RdnaImageDimension::Dim1D || image.dimension == RdnaImageDimension::Dim1DArray) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp index 72f796336..2c3eb07f5 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp @@ -1,3 +1,4 @@ +#include "RdnaDecoder/RdnaDescriptorFormat.hpp" #include "SpirvBackend/SpirvMemory/SpirvInputOutput.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" @@ -27,7 +28,7 @@ namespace ShaderRecompiler if (input >= PixelParameterLimit || input >= info.inputNum) { FailEmit("pixel interpolator index is out of range"); } - if (info.InputIsDefault(input)) { + if (info.InputIsDefault(input) && (!info.InputIsFp16(input) || info.InputHalfIsDefault(input, true))) { FailEmit("pixel input " + std::to_string(input) + " reads a default value, not a parameter"); } return info.InputSlot(input); @@ -88,29 +89,10 @@ namespace ShaderRecompiler if (location >= static_cast(ShaderVertexInputInfo::MaxResources) || location >= static_cast(vertex.resourcesNum)) { return VertexInputScalarKind::Float; } - switch (vertex.resources[location].Format()) { - case IrBufferFormat::Format8UInt: - case IrBufferFormat::Format16UInt: - case IrBufferFormat::Format8_8UInt: - case IrBufferFormat::Format32UInt: - case IrBufferFormat::Format16_16UInt: - case IrBufferFormat::Format8_8_8_8UInt: - case IrBufferFormat::Format32_32UInt: - case IrBufferFormat::Format16_16_16_16UInt: - case IrBufferFormat::Format32_32_32UInt: - case IrBufferFormat::Format32_32_32_32UInt: return VertexInputScalarKind::Uint; - case IrBufferFormat::Format8SInt: - case IrBufferFormat::Format16SInt: - case IrBufferFormat::Format8_8SInt: - case IrBufferFormat::Format32SInt: - case IrBufferFormat::Format16_16SInt: - case IrBufferFormat::Format8_8_8_8SInt: - case IrBufferFormat::Format32_32SInt: - case IrBufferFormat::Format16_16_16_16SInt: - case IrBufferFormat::Format32_32_32SInt: - case IrBufferFormat::Format32_32_32_32SInt: return VertexInputScalarKind::Sint; - default: return VertexInputScalarKind::Float; - } + const auto numeric = VertexInputNumericClass(vertex.resources[location].Format()); + if (numeric == IrTextureNumericClass::Uint) return VertexInputScalarKind::Uint; + if (numeric == IrTextureNumericClass::Sint) return VertexInputScalarKind::Sint; + return VertexInputScalarKind::Float; } std::uint32_t VertexParameterComponentCount(const SpirvInputBinding& input) { @@ -151,6 +133,10 @@ namespace ShaderRecompiler return 0; } + bool OrderedPixelShader(const SpirvEmitterState& state) { + return state.program.Resources().stage == IrShaderStage::Pixel && state.inputInfo.pixel != nullptr && state.inputInfo.pixel->psOrderedPixelShader; + } + const SpirvInputBinding* SpirvInputBindingForParameter(const SpirvEmitterState& state, std::uint32_t location) { for (const auto& input : state.inputs) { if (input.kind == StageInputKind::Parameter && input.location == location) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index dee96aab9..a0ab53817 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -1,4 +1,5 @@ #include "SpirvBackend/SpirvBda.hpp" +#include "SpirvBackend/SpirvRuntimeBuffer.hpp" #include "SpirvBackend/SpirvEmitterInstructions.hpp" #include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp" #include "SpirvBackend/SpirvBufferFormat.hpp" @@ -20,45 +21,6 @@ std::uint32_t AndCondition(SpirvEmitterState& state, std::uint32_t lhs, std::uin return Binary(state, spv::OpLogicalAnd, TypeBool(state), lhs, rhs); } -std::uint32_t BufferByteAddress(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t index, std::uint32_t offset, std::uint32_t soffset) { - auto& state = ctx.state; - const std::uint32_t packed = StorageBufferPackedStride(state, mem); - const std::uint32_t stride = packed & 0x3fffu; - const bool swizzle = stride != 0u && ((packed >> 14u) & 1u) != 0u; - if (((packed >> 20u) & 1u) != 0u) { - const auto lane = Binary(state, spv::OpBitwiseAnd, TypeU32(state), EmitSubgroupLocalInvocationId(state), ConstantU32(state, 63u)); - index = Binary(state, spv::OpIAdd, TypeU32(state), index, lane); - } - if (mem.offset != 0u) { - offset = Binary(state, spv::OpIAdd, TypeU32(state), offset, ConstantU32(state, mem.offset)); - } - std::uint32_t address = 0; - if (!swizzle) { - if (stride == 0u) { - address = offset; - } else { - const auto indexed = stride == 1u ? index : Binary(state, spv::OpIMul, TypeU32(state), index, ConstantU32(state, stride)); - address = Binary(state, spv::OpIAdd, TypeU32(state), indexed, offset); - } - } else { - const std::uint32_t strideEnum = (packed >> 16u) & 3u; - const std::uint32_t indexStride = 8u << strideEnum; - const auto indexMsb = Binary(state, spv::OpShiftRightLogical, TypeU32(state), index, ConstantU32(state, strideEnum + 3u)); - const auto indexLsb = Binary(state, spv::OpBitwiseAnd, TypeU32(state), index, ConstantU32(state, indexStride - 1u)); - const auto offsetMsb = Binary(state, spv::OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, ~3u)); - const auto offsetLsb = Binary(state, spv::OpBitwiseAnd, TypeU32(state), offset, ConstantU32(state, 3u)); - const auto indexedMsb = stride == 1u ? indexMsb : Binary(state, spv::OpIMul, TypeU32(state), indexMsb, ConstantU32(state, stride)); - const auto msb = Binary(state, spv::OpIMul, TypeU32(state), Binary(state, spv::OpIAdd, TypeU32(state), indexedMsb, offsetMsb), ConstantU32(state, indexStride)); - const auto lsb = Binary(state, spv::OpIAdd, TypeU32(state), Binary(state, spv::OpShiftLeftLogical, TypeU32(state), indexLsb, ConstantU32(state, 2u)), offsetLsb); - address = Binary(state, spv::OpIAdd, TypeU32(state), msb, lsb); - } - const IrValue* soffsetValue = inst.Argument(3)->Resolve(); - if (soffsetValue->HasImmediate() && soffsetValue->Type() == IrType::U32 && soffsetValue->ImmediateU32() == 0u) { - return address; - } - return Binary(state, spv::OpIAdd, TypeU32(state), address, soffset); -} - struct WordPair { std::uint32_t low = 0; std::uint32_t high = 0; @@ -84,8 +46,6 @@ std::uint32_t ScratchByteAddress(SpirvValueEmitContext& ctx, const MemoryInfo& m std::uint32_t ByteAddress(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { auto& state = ctx.state; switch (mem.kind) { - case ResourceKind::Buffer: - return BufferByteAddress(ctx, inst, mem, ctx.Arg(inst, 1), ctx.Arg(inst, 2), ctx.Arg(inst, 3)); case ResourceKind::Lds: case ResourceKind::Gds: if (mem.kind == ResourceKind::Lds) { @@ -114,11 +74,47 @@ std::uint32_t ActiveArgument(SpirvValueEmitContext& ctx, const IrValue& inst) { std::uint32_t LoadWordInBounds(SpirvValueEmitContext& ctx, const MemoryResourceAccess& resource, std::uint32_t index) { auto& state = ctx.state; - const auto pointer = EmitMemoryElementPointer(state, resource, index); - const auto value = state.module.AllocateId(); - if (resource.memoryAccess != 0u) state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, resource.memoryAccess); - else state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); - return value; + const auto load = [&](std::uint32_t element) { + const auto pointer = EmitMemoryElementPointer(state, resource, element); + const auto value = state.module.AllocateId(); + if (resource.memoryAccess != 0u) state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, resource.memoryAccess); + else state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); + return value; + }; + if (resource.misalignment == 0u) return load(index); + const auto shift = resource.misalignment * 8u; + const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(index), ConstantU32(state, shift)); + const auto high = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), load(EmitAddU32(state, index, ConstantU32(state, 1u))), ConstantU32(state, 32u - shift)); + return Binary(state, spv::OpBitwiseOr, TypeU32(state), low, high); +} + +void StoreMisalignedBits(SpirvValueEmitContext& ctx, const MemoryResourceAccess& resource, std::uint32_t index, std::uint32_t mask, std::uint32_t value) { + auto& state = ctx.state; + const auto shift = resource.misalignment * 8u; + const auto merge = [&](std::uint32_t element, spv::Op direction, std::uint32_t amount) { + const auto partMask = Binary(state, direction, TypeU32(state), mask, ConstantU32(state, amount)); + const auto partValue = Binary(state, direction, TypeU32(state), value, ConstantU32(state, amount)); + AtomicUpdate(state, EmitMemoryElementPointer(state, resource, element), resource.kind, [&](std::uint32_t old) { + return Binary(state, spv::OpBitwiseOr, TypeU32(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), old, Unary(state, spv::OpNot, TypeU32(state), partMask)), partValue); + }); + }; + merge(index, spv::OpShiftLeftLogical, shift); + merge(EmitAddU32(state, index, ConstantU32(state, 1u)), spv::OpShiftRightLogical, 32u - shift); +} + +void StoreMisalignedSubword(SpirvValueEmitContext& ctx, const MemoryResourceAccess& resource, std::uint32_t address, std::uint32_t index, std::uint32_t bits, std::uint32_t data) { + auto& state = ctx.state; + if (bits != 8u && bits != 16u) { + ctx.Fail("subword store has an unsupported width"); + } + const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), address, ConstantU32(state, 3u)), ConstantU32(state, 3u)); + const auto fieldMask = ConstantU32(state, bits == 8u ? 0xffu : 0xffffu); + const auto mask = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), fieldMask, shift); + StoreMisalignedBits(ctx, resource, index, mask, Binary(state, spv::OpShiftLeftLogical, TypeU32(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), data, fieldMask), shift)); +} + +void RequireAlignedAtomic(SpirvValueEmitContext& ctx, const MemoryResourceAccess& resource) { + if (resource.misalignment != 0u) ctx.Fail("buffer atomic on a V# whose base is not DWORD aligned"); } std::uint32_t LoadSubwordInBounds(SpirvValueEmitContext& ctx, const MemoryResourceAccess& resource, std::uint32_t address, std::uint32_t index, std::uint32_t bits, bool signExtend) { @@ -179,84 +175,6 @@ std::uint32_t DeviceAddressFromWords(SpirvEmitterState& state, std::uint32_t low return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), low64, high64); } -std::pair RuntimeBufferByteAddress(SpirvEmitterState& state, std::uint32_t index, std::uint32_t offset, std::uint32_t soffset, std::uint32_t immediate, std::uint32_t stride, std::uint32_t swizzle, std::uint32_t indexStride) { - const auto u32 = TypeU32(state); - const auto add = [&](std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, spv::OpIAdd, u32, lhs, rhs); }; - const auto mul = [&](std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, spv::OpIMul, u32, lhs, rhs); }; - if (immediate != 0u) offset = add(offset, ConstantU32(state, immediate)); - const auto linear = add(mul(index, stride), offset); - const auto indexShift = add(indexStride, ConstantU32(state, 3u)); - const auto indices = Binary(state, spv::OpShiftLeftLogical, u32, ConstantU32(state, 1u), indexShift); - const auto indexMsb = Binary(state, spv::OpShiftRightLogical, u32, index, indexShift); - const auto indexLsb = Binary(state, spv::OpBitwiseAnd, u32, index, Binary(state, spv::OpISub, u32, indices, ConstantU32(state, 1u))); - const auto offsetMsb = Binary(state, spv::OpBitwiseAnd, u32, offset, ConstantU32(state, ~3u)); - const auto offsetLsb = Binary(state, spv::OpBitwiseAnd, u32, offset, ConstantU32(state, 3u)); - const auto swizzled = add(mul(add(mul(indexMsb, stride), offsetMsb), indices), add(Binary(state, spv::OpShiftLeftLogical, u32, indexLsb, ConstantU32(state, 2u)), offsetLsb)); - return {offset, add(Select(state, u32, swizzle, swizzled, linear), soffset)}; -} - -template -void ForEachGpuDescriptorDword(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components, TFunction&& function) { - auto& state = ctx.state; - const IrValue* handle = inst.Argument(0)->Resolve(); - if (handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) { - ctx.Fail(inst, "has no GPU-selected V#"); - } - const auto u32 = TypeU32(state); - const auto boolean = TypeBool(state); - const auto word1 = ctx.Arg(*handle, 1); - const auto records = ctx.Arg(*handle, 2); - const auto word3 = ctx.Arg(*handle, 3); - const auto field = [&](std::uint32_t word, std::uint32_t first, std::uint32_t count) { return EmitBitFieldUExtract(state, word, ConstantU32(state, first), ConstantU32(state, count)); }; - const auto nonzero = [&](std::uint32_t value) { return Binary(state, spv::OpINotEqual, boolean, value, ConstantU32(state, 0u)); }; - const auto hasDword = [&](std::uint32_t offset, std::uint32_t size) { - return AndCondition(state, Binary(state, spv::OpUGreaterThanEqual, boolean, size, ConstantU32(state, 4u)), Binary(state, spv::OpULessThanEqual, boolean, offset, Binary(state, spv::OpISub, u32, size, ConstantU32(state, 4u)))); - }; - const auto stride = field(word1, 16u, 14u); - const auto swizzle = AndCondition(state, nonzero(stride), nonzero(field(word1, 31u, 1u))); - const auto lane = Binary(state, spv::OpBitwiseAnd, u32, EmitSubgroupLocalInvocationId(state), ConstantU32(state, 63u)); - const auto index = Binary(state, spv::OpIAdd, u32, ctx.Arg(inst, 1), Select(state, u32, nonzero(field(word3, 23u, 1u)), lane, ConstantU32(state, 0u))); - const auto soffset = ctx.Arg(inst, 3); - const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), field(word1, 0u, 16u)); - const auto mode = field(word3, 28u, 2u); - const auto indexInBounds = Binary(state, spv::OpULessThan, boolean, index, records); - const auto scalarInBounds = Binary(state, spv::OpULessThanEqual, boolean, soffset, records); - const auto rawRecords = Binary(state, spv::OpISub, u32, records, soffset); - const auto rawIndexInBounds = Binary(state, spv::OpULessThan, boolean, index, rawRecords); - for (std::uint32_t component = 0; component < components; component++) { - const auto [offset, byte] = RuntimeBufferByteAddress(state, index, ctx.Arg(inst, 2), soffset, mem.offset + component * 4u, stride, swizzle, field(word3, 21u, 2u)); - const auto structured = AndCondition(state, indexInBounds, Binary(state, spv::OpULessThan, boolean, offset, stride)); - const auto raw = AndCondition(state, scalarInBounds, Select(state, boolean, swizzle, AndCondition(state, rawIndexInBounds, hasDword(offset, stride)), hasDword(offset, rawRecords))); - auto inBounds = Select(state, boolean, Binary(state, spv::OpIEqual, boolean, mode, ConstantU32(state, 0u)), structured, indexInBounds); - inBounds = Select(state, boolean, Binary(state, spv::OpULessThan, boolean, mode, ConstantU32(state, 2u)), inBounds, Select(state, boolean, Binary(state, spv::OpIEqual, boolean, mode, ConstantU32(state, 2u)), nonzero(records), raw)); - const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), base, Unary(state, spv::OpUConvert, TypeScalarU64(state), byte)); - function(component, guest, AndCondition(state, nonzero(field(word3, 12u, 7u)), inBounds)); - } -} - -std::uint32_t ConstructU32Composite(SpirvEmitterState& state, std::uint32_t components, const std::array& values); - -std::uint32_t LoadGpuDescriptor(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components) { - std::array values{}; - ForEachGpuDescriptorDword(ctx, inst, ctx.Memory(inst), components, [&](std::uint32_t component, std::uint32_t guest, std::uint32_t inBounds) { - values[component] = EmitValueOrZeroIfCondition(ctx.state, inBounds, [&] { return EmitBdaRead(ctx, inst, guest, 32u); }); - }); - return components == 1u ? values[0] : ConstructU32Composite(ctx.state, components, values); -} - -void StoreGpuDescriptor(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components) { - auto& state = ctx.state; - const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u); - ForEachGpuDescriptorDword(ctx, inst, ctx.Memory(inst), components, [&](std::uint32_t component, std::uint32_t guest, std::uint32_t inBounds) { - auto value = data; - if (components != 1u) { - value = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), value, data, component); - } - EmitIfCondition(state, inBounds, [&] { EmitBdaWrite(ctx, inst, guest, value); }); - }); -} - std::uint32_t GuestAddressBase(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { auto& state = ctx.state; auto low = ctx.Arg(inst, 1); @@ -290,93 +208,12 @@ std::uint32_t LoadBda(SpirvValueEmitContext& ctx, const IrValue& inst, const Mem }); } -IrBufferFormat BufferFormatOf(const SpirvEmitterState& state, const MemoryInfo& mem) { - return mem.typed ? DecodeTBufferFormat(mem.dataFormat, mem.numberFormat) : StorageBufferFormat(state, mem); -} - -SpirvBufferFormatInfo MemoryFormatInfo(const SpirvEmitterState& state, const MemoryInfo& mem) { - return GetFormatInfo(mem.formatted ? BufferFormatOf(state, mem) : IrBufferFormat::Invalid); -} - -MemoryInfo RebaseFormattedComponent(MemoryInfo mem, const SpirvBufferFormatInfo& info, std::uint32_t component) { - mem.offset += GetFormatComponentByteOffset(info, component); - mem.dataDwords = 1u; - mem.componentIndex = component; - return mem; -} - -MemoryInfo RebaseRawComponent(MemoryInfo mem, std::uint32_t component) { - mem.offset += component * 4u; - mem.dataDwords = 1u; - mem.componentIndex = component; - return mem; -} - -SpirvFormattedSource ResolveOutputSource(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const SpirvBufferFormatInfo& info, std::uint32_t outputComponent) { - if (mem.typed) { - if (outputComponent < info.componentCount) { - return {SpirvFormattedSourceKind::Memory, outputComponent}; - } - return {}; - } - const auto swizzle = ctx.state.program.Info().buffers.at(mem.resource).descriptorSwizzle; - return ResolveFormattedSource(info, (swizzle >> (outputComponent * 3u)) & 0x7u); -} - -std::uint32_t FormattedConstant(SpirvValueEmitContext& ctx, const SpirvBufferFormatInfo& info, SpirvFormattedSourceKind kind) { - return ConstantU32(ctx.state, FormattedConstantBits(info, kind)); -} - -template -std::uint32_t LoadFormattedComponent(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const SpirvBufferFormatInfo& info, std::uint32_t outputComponent, TLoadWord&& loadWord, TLoadSubword&& loadSubword) { - auto& state = ctx.state; - const auto source = ResolveOutputSource(ctx, mem, info, outputComponent); - if (source.kind != SpirvFormattedSourceKind::Memory) { - const bool integer = info.type == SpirvFormatComponentType::Uint || info.type == SpirvFormatComponentType::Sint; - if (mem.d16 && source.kind == SpirvFormattedSourceKind::One && !integer) return ConstantU32(state, 0x3c00u); - return FormattedConstant(ctx, info, source.kind); - } - const auto component = source.component; - const auto bits = info.componentBits[component]; - const bool signedType = IsSignedFormatComponent(info.type); - std::uint32_t raw = 0; - if (info.packedBitfield) { - raw = loadWord(component); - const auto type = signedType ? TypeI32(state) : TypeU32(state); - const auto sourceValue = signedType ? Unary(state, spv::OpBitcast, type, raw) : raw; - const auto extracted = state.module.AllocateId(); - state.module.AddFunction(signedType ? spv::OpBitFieldSExtract : spv::OpBitFieldUExtract, type, extracted, sourceValue, ConstantU32(state, info.componentBitOffset[component]), ConstantU32(state, bits)); - raw = signedType ? Unary(state, spv::OpBitcast, TypeU32(state), extracted) : extracted; - } else if (bits == 32u) { - raw = loadWord(component); - } else { - raw = loadSubword(component, bits, signedType); - } - return mem.d16 ? EmitD16FormatComponent(state, info, component, raw) : NormalizeFormatComponent(state, info, component, raw); -} - -std::uint32_t FormattedLoadPrepared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t outputComponent, const MemoryResourceAccess& resource) { - const auto info = MemoryFormatInfo(ctx.state, mem); - if (info.type == SpirvFormatComponentType::Unknown) { - if (mem.d16) ctx.Fail(inst, "is a D16 format load with an unknown buffer format"); - return LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, outputComponent), resource); - } - return LoadFormattedComponent(ctx, mem, info, outputComponent, [&](std::uint32_t component) { - return LoadWordPrepared(ctx, inst, RebaseFormattedComponent(mem, info, component), resource); - }, [&](std::uint32_t component, std::uint32_t bits, bool signExtend) { - return LoadSubwordPrepared(ctx, inst, RebaseFormattedComponent(mem, info, component), resource, bits, signExtend); - }); -} - -std::uint32_t FormattedLoad(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { - return EmitValueOrZeroIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - return FormattedLoadPrepared(ctx, inst, mem, 0u, resource); - }); -} - void StoreWordInBounds(SpirvValueEmitContext& ctx, const MemoryResourceAccess& resource, std::uint32_t index, std::uint32_t data) { auto& state = ctx.state; + if (resource.misalignment != 0u) { + StoreMisalignedBits(ctx, resource, index, ConstantU32(state, 0xffffffffu), data); + return; + } if (resource.memoryAccess != 0u) state.module.AddFunction(spv::OpStore, EmitMemoryElementPointer(state, resource, index), data, resource.memoryAccess); else state.module.AddFunction(spv::OpStore, EmitMemoryElementPointer(state, resource, index), data); } @@ -414,7 +251,8 @@ void StoreSubwordPrepared(SpirvValueEmitContext& ctx, const IrValue& inst, const const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); const auto index = EmitMemoryElementIndex(state, resource, rawIndex); EmitIfCondition(state, EmitMemoryElementInBounds(state, resource, index), [&]() { - StoreSubwordInBounds(ctx, mem, resource, address, index, bits, data); + if (resource.misalignment != 0u) StoreMisalignedSubword(ctx, resource, address, index, bits, data); + else StoreSubwordInBounds(ctx, mem, resource, address, index, bits, data); }); } @@ -440,81 +278,6 @@ void StoreWord(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo }); } -struct PreparedFormattedMemory { - SpirvBufferFormatInfo info; - MemoryResourceAccess resource; - std::array addresses{}; - std::array indices{}; - std::uint32_t inBounds = 0; -}; - -enum class FormattedAccess { - Load, - Store -}; - -PreparedFormattedMemory PrepareFormattedMemory(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource, const SpirvBufferFormatInfo& info, std::uint32_t components, FormattedAccess access) { - auto& state = ctx.state; - PreparedFormattedMemory plan; - plan.info = info; - plan.resource = resource; - std::array required{}; - if (access == FormattedAccess::Load) { - for (std::uint32_t output = 0; output < components; output++) { - const auto source = ResolveOutputSource(ctx, mem, plan.info, output); - if (source.kind == SpirvFormattedSourceKind::Memory) { - required.at(source.component) = true; - } - } - } else { - for (std::uint32_t component = 0; component < plan.info.componentCount; component++) { - required.at(component) = true; - } - } - bool firstBound = true; - for (std::uint32_t component = 0; component < plan.info.componentCount; component++) { - if (!required.at(component)) { - continue; - } - const auto byteOffset = GetFormatComponentByteOffset(plan.info, component); - bool reused = false; - for (std::uint32_t previous = 0; previous < component; previous++) { - if (required.at(previous) && GetFormatComponentByteOffset(plan.info, previous) == byteOffset) { - plan.addresses.at(component) = plan.addresses.at(previous); - plan.indices.at(component) = plan.indices.at(previous); - reused = true; - break; - } - } - if (reused) { - continue; - } - const auto componentMem = RebaseFormattedComponent(mem, plan.info, component); - plan.addresses.at(component) = ByteAddress(ctx, inst, componentMem); - const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), plan.addresses.at(component), ConstantU32(state, 2u)); - plan.indices.at(component) = EmitMemoryElementIndex(state, resource, rawIndex); - const auto componentBound = EmitMemoryElementInBounds(state, resource, plan.indices.at(component)); - if (firstBound) { - plan.inBounds = componentBound; - firstBound = false; - } else { - plan.inBounds = AndCondition(state, plan.inBounds, componentBound); - } - } - if (firstBound) { - plan.inBounds = ConstantBool(state, true); - } - return plan; -} - -std::uint32_t LoadFormattedInBounds(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, std::uint32_t outputComponent) { - return LoadFormattedComponent(ctx, mem, plan.info, outputComponent, [&](std::uint32_t component) { - return LoadWordInBounds(ctx, plan.resource, plan.indices.at(component)); - }, [&](std::uint32_t component, std::uint32_t bits, bool signExtend) { - return LoadSubwordInBounds(ctx, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, signExtend); - }); -} - std::uint32_t ConstructU32Composite(SpirvEmitterState& state, std::uint32_t components, const std::array& values) { const auto result = state.module.AllocateId(); std::vector words{spv::OpCompositeConstruct, TypeU32Composite(state, components), result}; @@ -523,134 +286,6 @@ std::uint32_t ConstructU32Composite(SpirvEmitterState& state, std::uint32_t comp return result; } -std::uint32_t FormattedOutOfBoundsValue(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, std::uint32_t components) { - auto& state = ctx.state; - std::array values{}; - for (std::uint32_t component = 0; component < components; component++) { - const auto source = ResolveOutputSource(ctx, mem, plan.info, component); - values.at(component) = source.kind == SpirvFormattedSourceKind::Memory ? ConstantU32(state, 0u) : FormattedConstant(ctx, plan.info, source.kind); - } - return ConstructU32Composite(state, components, values); -} - -void StoreFormattedElement(SpirvValueEmitContext& ctx, const MemoryInfo& mem, const PreparedFormattedMemory& plan, const std::array& data, std::uint32_t components) { - auto& state = ctx.state; - const auto& info = plan.info; - std::array encoded{}; - for (std::uint32_t component = 0; component < info.componentCount; component++) { - encoded.at(component) = component >= components ? ConstantU32(state, 0u) : mem.d16 ? EmitD16StoreComponent(state, info, component, data.at(component)) : EmitFormatStoreComponent(state, info, component, data.at(component)); - } - if (info.packedBitfield) { - auto word = ConstantU32(state, 0u); - for (std::uint32_t component = 0; component < info.componentCount; component++) { - const std::uint32_t mask = (1u << info.componentBits[component]) - 1u; - word = EmitOrU32(state, word, EmitBinaryU32(state, spv::OpShiftLeftLogical, EmitAndConstant(state, encoded.at(component), mask), ConstantU32(state, info.componentBitOffset[component]))); - } - StoreWordInBounds(ctx, plan.resource, plan.indices.at(0), word); - return; - } - for (std::uint32_t component = 0; component < info.componentCount; component++) { - const auto bits = info.componentBits[component]; - if (bits == 8u || bits == 16u) { - StoreSubwordInBounds(ctx, mem, plan.resource, plan.addresses.at(component), plan.indices.at(component), bits, encoded.at(component)); - } else { - StoreWordInBounds(ctx, plan.resource, plan.indices.at(component), encoded.at(component)); - } - } -} - -void StoreFormattedValues(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource, const SpirvBufferFormatInfo& info, const std::array& data, std::uint32_t components) { - const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, FormattedAccess::Store); - EmitIfCondition(ctx.state, plan.inBounds, [&]() { - StoreFormattedElement(ctx, mem, plan, data, components); - }); -} - -void FormattedStore(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem) { - EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() { - const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); - const auto info = MemoryFormatInfo(ctx.state, mem); - const auto data = ctx.Arg(inst, inst.ArgumentCount() - 2u); - if (info.type == SpirvFormatComponentType::Unknown) { - if (mem.d16) ctx.Fail(inst, "is a D16 format store with an unknown buffer format"); - StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, 0u), resource, data); - return; - } - StoreFormattedValues(ctx, inst, mem, resource, info, {data, 0u, 0u, 0u}, 1u); - }); -} - -std::uint32_t LoadCoherentPair(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, const MemoryResourceAccess& resource) { - auto& state = ctx.state; - const auto wide = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferU64Variable, TypeStorageBufferU64Pointer(state)); - const auto byteAddress = Binary(state, spv::OpIAdd, TypeU32(state), ByteAddress(ctx, inst, mem), wide.byteOffset); - const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteAddress, ConstantU32(state, 3u)); - const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), byteAddress, ConstantU32(state, 7u)), ConstantU32(state, 0u)); - return EmitValueIfElse(state, AndCondition(state, aligned, EmitMemoryElementInBounds(state, wide, index)), TypeU32Composite(state, 2u), [&]() { - const auto pointer = EmitStorageBufferElementPointer(state, wide, index, TypeStorageBufferU64ElementPointer(state)); - const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpAtomicLoad, TypeScalarU64(state), value, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone)); - const auto high = Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), value, BdaConstant(state, 32u)); - return ConstructU32Composite(state, 2u, {Unary(state, spv::OpUConvert, TypeU32(state), value), Unary(state, spv::OpUConvert, TypeU32(state), high), 0u, 0u}); - }, [&]() { - std::array values{}; - for (std::uint32_t component = 0; component < 2u; component++) { - values.at(component) = LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource); - } - return ConstructU32Composite(state, 2u, values); - }); -} - -std::uint32_t LoadWideBuffer(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) { - auto& state = ctx.state; - return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() { - const auto resource = PrepareMemoryResourceAccess(state, mem); - const auto info = MemoryFormatInfo(state, mem); - if (info.type == SpirvFormatComponentType::Unknown && components == 2u && mem.coherent && state.storageBufferU64Variable != 0u) { - return LoadCoherentPair(ctx, inst, mem, resource); - } - if (info.type != SpirvFormatComponentType::Unknown) { - const auto plan = PrepareFormattedMemory(ctx, inst, mem, resource, info, components, FormattedAccess::Load); - return EmitValueOrDefaultIfCondition(state, plan.inBounds, TypeU32Composite(state, components), FormattedOutOfBoundsValue(ctx, mem, plan, components), [&]() { - std::array values{}; - for (std::uint32_t component = 0; component < components; component++) { - values.at(component) = LoadFormattedInBounds(ctx, mem, plan, component); - } - return ConstructU32Composite(state, components, values); - }); - } - std::array values{}; - for (std::uint32_t component = 0; component < components; component++) { - values.at(component) = LoadWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource); - } - return ConstructU32Composite(state, components, values); - }); -} - -void StoreWideBuffer(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) { - auto& state = ctx.state; - EmitIfCondition(state, ActiveArgument(ctx, inst), [&]() { - const auto resource = PrepareMemoryResourceAccess(state, mem); - const auto composite = ctx.Arg(inst, inst.ArgumentCount() - 2u); - const auto info = MemoryFormatInfo(state, mem); - if (info.type != SpirvFormatComponentType::Unknown) { - std::array data{}; - for (std::uint32_t component = 0; component < components; component++) { - data.at(component) = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), data.at(component), composite, component); - } - StoreFormattedValues(ctx, inst, mem, resource, info, data, components); - return; - } - if (mem.d16) ctx.Fail(inst, "is a D16 format store with an unknown buffer format"); - for (std::uint32_t component = 0; component < components; component++) { - const auto data = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), data, composite, component); - StoreWordPrepared(ctx, inst, RebaseRawComponent(mem, component), resource, data); - } - }); -} - std::uint32_t LoadWideShared(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, std::uint32_t components) { auto& state = ctx.state; return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU32Composite(state, components), ConstantU32CompositeZero(state, components), [&]() { @@ -690,27 +325,9 @@ const MemoryInfo& BufferMemory(SpirvValueEmitContext& ctx, const IrValue& inst) if (mem.kind != ResourceKind::Buffer) { ctx.Fail(inst, "must access a buffer resource"); } - if (mem.gpuDescriptor) { - ctx.Fail(inst, "accesses a GPU-selected V# in a way only raw dword loads and stores support"); - } return mem; } -bool EmitGpuDescriptorAccess(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components, bool store) { - if (!ctx.Memory(inst).gpuDescriptor) { - return false; - } - auto& state = ctx.state; - if (store) { - EmitIfCondition(state, ActiveArgument(ctx, inst), [&] { StoreGpuDescriptor(ctx, inst, components); }); - return true; - } - const auto type = components == 1u ? TypeU32(state) : TypeU32Composite(state, components); - const auto zero = components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); - ctx.Define(inst, EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), type, zero, [&] { return LoadGpuDescriptor(ctx, inst, components); })); - return true; -} - const MemoryInfo& SharedMemory(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto& mem = ctx.Memory(inst); if (mem.kind != ResourceKind::Lds && mem.kind != ResourceKind::Gds) { @@ -788,6 +405,7 @@ struct PreparedMemoryElement { PreparedMemoryElement PrepareMemoryElement(SpirvValueEmitContext& ctx, const MemoryInfo& mem, std::uint32_t rawIndex) { const auto resource = PrepareMemoryResourceAccess(ctx.state, mem); + RequireAlignedAtomic(ctx, resource); return {resource, EmitMemoryElementIndex(ctx.state, resource, rawIndex)}; } @@ -856,6 +474,7 @@ std::uint32_t EmitAtomicOperation(SpirvValueEmitContext& ctx, const IrValue& ins template std::uint32_t EmitAtomicAccess(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& mem, TOperation&& operation) { + if (mem.kind == ResourceKind::Buffer) return EmitRuntimeBufferAtomic(ctx, inst, 32u, ActiveArgument(ctx, inst), operation); auto& state = ctx.state; return EmitValueOrZeroIfCondition(state, ActiveArgument(ctx, inst), [&]() { const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); @@ -939,52 +558,46 @@ std::uint32_t Atomic32(SpirvValueEmitContext& ctx, const IrValue& inst, const Me if (zeroSkip) active = AndCondition(state, active, nonZero); } if (mem.kind == ResourceKind::Buffer) NoteBufferAtomicSite(zeroSkip, zeroIdentity && !zeroSkip); - return EmitValueOrZeroIfCondition(state, active, [&]() { - const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); - return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access.resource, access.index), [&]() { - const auto pointer = EmitMemoryElementPointer(state, access.resource, access.index); - const auto operation = [&]() { - const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); - if (lds) { - const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; - state.module.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)); - } else { - EmitDeviceAtomicMemoryBarrier(state); - } - return old; - }; - if (!zeroIdentity || zeroSkip) return operation(); - return EmitValueIfElse(state, nonZero, TypeU32(state), operation, [&]() { - // The barrier stays so an "add 0" used as a coherent read keeps its acquire. - const auto current = state.module.AllocateId(); - state.module.AddFunction(spv::OpAtomicLoad, TypeU32(state), current, pointer, ConstantU32(state, scope), ConstantU32(state, spv::MemorySemanticsMaskNone)); + const auto apply = [&](std::uint32_t pointer) { + const auto operation = [&]() { + const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); + if (lds) { + const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; + state.module.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)); + } else { EmitDeviceAtomicMemoryBarrier(state); - return current; - }); + } + return old; + }; + if (!zeroIdentity || zeroSkip) return operation(); + return EmitValueIfElse(state, nonZero, TypeU32(state), operation, [&]() { + const auto current = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicLoad, TypeU32(state), current, pointer, ConstantU32(state, scope), ConstantU32(state, spv::MemorySemanticsMaskNone)); + EmitDeviceAtomicMemoryBarrier(state); + return current; }); + }; + if (mem.kind == ResourceKind::Buffer) return EmitRuntimeBufferAtomic(ctx, inst, 32u, active, apply); + return EmitValueOrZeroIfCondition(state, active, [&] { + const auto access = PrepareMemoryElement(ctx, mem, DwordIndex(ctx, inst, mem)); + return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access.resource, access.index), [&] { return apply(EmitMemoryElementPointer(state, access.resource, access.index)); }); }); } std::uint32_t BufferAtomic64(SpirvValueEmitContext& ctx, const IrValue& inst) { auto& state = ctx.state; - const auto& mem = BufferMemory(ctx, inst); - return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU64(state), ConstantU64(state, 0u), [&]() { - const auto resource = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferU64Variable, TypeStorageBufferU64Pointer(state)); - const auto byteAddress = Binary(state, spv::OpIAdd, TypeU32(state), ByteAddress(ctx, inst, mem), resource.byteOffset); - const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteAddress, ConstantU32(state, 3u)); - return EmitValueOrDefaultIfCondition(state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state), ConstantU64(state, 0u), [&]() { - const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 2u)); - const auto old = state.module.AllocateId(); - const auto pointer = EmitStorageBufferElementPointer(state, resource, index, TypeStorageBufferU64ElementPointer(state)); - if (inst.Opcode() == IrOpcode::BufferAtomicCmpSwap64) { - const auto desired = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 3u)); - state.module.AddFunction(spv::OpAtomicCompareExchange, TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), ConstantU32(state, spv::MemorySemanticsMaskNone), desired, value); - } else { - state.module.AddFunction(SpirvAtomicOpcode(inst.Opcode()), TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), value); - } - EmitDeviceAtomicMemoryBarrier(state); - return Unary(state, spv::OpBitcast, TypeU64(state), old); - }); + static_cast(BufferMemory(ctx, inst)); + return EmitRuntimeBufferAtomic(ctx, inst, 64u, ActiveArgument(ctx, inst), [&](std::uint32_t pointer) { + const auto value = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 2u)); + const auto old = state.module.AllocateId(); + if (inst.Opcode() == IrOpcode::BufferAtomicCmpSwap64) { + const auto desired = Unary(state, spv::OpBitcast, TypeScalarU64(state), ctx.Arg(inst, inst.ArgumentCount() - 3u)); + state.module.AddFunction(spv::OpAtomicCompareExchange, TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), ConstantU32(state, spv::MemorySemanticsMaskNone), desired, value); + } else { + state.module.AddFunction(SpirvAtomicOpcode(inst.Opcode()), TypeScalarU64(state), old, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), value); + } + EmitDeviceAtomicMemoryBarrier(state); + return Unary(state, spv::OpBitcast, TypeU64(state), old); }); } @@ -1045,18 +658,10 @@ std::uint32_t BufferFloatAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, template std::uint32_t BufferAtomic64Update(SpirvValueEmitContext& ctx, const IrValue& inst, TReplacement&& replacement) { auto& state = ctx.state; - const auto& mem = BufferMemory(ctx, inst); - return EmitValueOrDefaultIfCondition(state, ActiveArgument(ctx, inst), TypeU64(state), ConstantU64(state, 0u), [&]() { - const auto resource = PrepareStorageBufferResourceAccess(state, mem, state.storageBufferU64Variable, TypeStorageBufferU64Pointer(state)); - const auto byteAddress = Binary(state, spv::OpIAdd, TypeU32(state), ByteAddress(ctx, inst, mem), resource.byteOffset); - const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byteAddress, ConstantU32(state, 3u)); - return EmitValueOrDefaultIfCondition(state, EmitMemoryElementInBounds(state, resource, index), TypeU64(state), ConstantU64(state, 0u), [&]() { - const auto pointer = EmitStorageBufferElementPointer(state, resource, index, TypeStorageBufferU64ElementPointer(state)); - const auto old = AtomicUpdateTyped(state, pointer, ResourceKind::Buffer, TypeScalarU64(state), [&](std::uint32_t observed) { - return replacement(observed); - }); - return Unary(state, spv::OpBitcast, TypeU64(state), old); - }); + static_cast(BufferMemory(ctx, inst)); + return EmitRuntimeBufferAtomic(ctx, inst, 64u, ActiveArgument(ctx, inst), [&](std::uint32_t pointer) { + const auto old = AtomicUpdateTyped(state, pointer, ResourceKind::Buffer, TypeScalarU64(state), [&](std::uint32_t observed) { return replacement(observed); }); + return Unary(state, spv::OpBitcast, TypeU64(state), old); }); } @@ -1415,6 +1020,41 @@ std::uint32_t EmitReadConst(SpirvValueEmitContext& ctx, const IrValue& inst) { if (state.flattenedSrtVariable == 0) { ctx.Fail(inst, "requires the flattened SRT descriptor"); } + const auto& resources = state.program.Resources(); + const auto& guarded = resources.guardedSrtSlots; + if (!guarded.empty()) { + const IrValue* slot = inst.Argument(1)->Resolve(); + if (!slot->HasImmediate()) { + ctx.Fail(inst, "reads the flattened SRT at a runtime slot while some slots are guarded"); + } + const auto found = std::lower_bound(guarded.begin(), guarded.end(), slot->ImmediateU32()); + if (found != guarded.end() && *found == slot->ImmediateU32()) { + if (state.faultBufferVariable == 0) { + ctx.Fail(inst, "reads a guarded SRT slot without a fault buffer"); + } + const auto u32 = TypeU32(state); + const auto load = [&](std::uint32_t index) { + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.flattenedSrtVariable, ConstantU32(state, 0u), index); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, u32, value, pointer); + return value; + }; + const auto guard = static_cast(found - guarded.begin()); + const auto flag = load(ConstantU32(state, resources.srtGuardOffset + guard)); + const auto poisoned = Binary(state, spv::OpINotEqual, TypeBool(state), flag, ConstantU32(state, 0u)); + EmitIfCondition(state, poisoned, [&] { + const auto records = resources.srtGuardOffset + static_cast(guarded.size()); + const auto record = Binary(state, spv::OpIAdd, u32, ConstantU32(state, records), Binary(state, spv::OpIMul, u32, Binary(state, spv::OpISub, u32, flag, ConstantU32(state, 1u)), ConstantU32(state, 3u))); + const auto word = [&](std::uint32_t offset) { return load(Binary(state, spv::OpIAdd, u32, record, ConstantU32(state, offset))); }; + const auto pc = word(0u); + const auto low = word(1u); + const auto high = word(2u); + RecordBdaFaultWords(state, low, high, ConstantU32(state, 4u), pc, BdaAbi::FaultReason::Unmapped); + }); + StopBdaInvocationIf(state, poisoned); + } + } const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.flattenedSrtVariable, ConstantU32(state, 0u), ctx.Arg(inst, 1)); const auto value = state.module.AllocateId(); @@ -1424,41 +1064,9 @@ std::uint32_t EmitReadConst(SpirvValueEmitContext& ctx, const IrValue& inst) { void EmitReadConstBuffer(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto& mem = ctx.Memory(inst); - if (mem.planningOnly) { - return; - } - if (mem.kind != ResourceKind::ScalarBuffer) { - ctx.Fail(inst, "must read a scalar buffer resource"); - } - auto& state = ctx.state; - const auto address = Binary(state, spv::OpIAdd, TypeU32(state), ctx.Arg(inst, 1), ConstantU32(state, mem.offset)); - const auto noCarry = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address, ctx.Arg(inst, 1)); - if (mem.gpuDescriptor) { - const IrValue* handle = inst.Argument(0)->Resolve(); - if (handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) { - ctx.Fail(inst, "has no GPU-selected V#"); - } - const auto u32 = TypeU32(state); - const auto word1 = ctx.Arg(*handle, 1); - const auto records = ctx.Arg(*handle, 2); - const auto stride = EmitBitFieldUExtract(state, word1, ConstantU32(state, 16u), ConstantU32(state, 14u)); - const auto size = Select(state, u32, Binary(state, spv::OpIEqual, TypeBool(state), stride, ConstantU32(state, 0u)), records, Binary(state, spv::OpIMul, u32, stride, records)); - const auto byte = Binary(state, spv::OpBitwiseAnd, u32, address, ConstantU32(state, ~3u)); - const auto inBounds = AndCondition(state, noCarry, Binary(state, spv::OpULessThan, TypeBool(state), byte, size)); - const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), EmitBitFieldUExtract(state, word1, ConstantU32(state, 0u), ConstantU32(state, 16u))); - const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), base, Unary(state, spv::OpUConvert, TypeScalarU64(state), byte)); - ctx.Define(inst, EmitValueOrZeroIfCondition(state, inBounds, [&] { return EmitBdaRead(ctx, inst, guest, 32u); })); - return; - } - const auto rawIndex = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); - const auto access = PrepareMemoryResourceAccess(state, mem); - const auto element = EmitMemoryElementIndex(state, access, rawIndex); - const auto inBounds = AndCondition(state, noCarry, EmitMemoryElementInBounds(state, access, element)); - ctx.Define(inst, EmitValueOrZeroIfCondition(state, inBounds, [&]() { - const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, TypeU32(state), value, EmitMemoryElementPointer(state, access, element)); - return value; - })); + if (mem.planningOnly) return; + if (mem.kind != ResourceKind::ScalarBuffer) ctx.Fail(inst, "must read a scalar buffer resource"); + ctx.Define(inst, EmitRuntimeScalarBufferLoad(ctx, inst)); } void EmitGetSrtResource(SpirvValueEmitContext& context) { @@ -1518,65 +1126,51 @@ std::uint32_t EmitAddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst) } void EmitLoadBufferU8(SpirvValueEmitContext& ctx, const IrValue& inst) { - ctx.Define(inst, LoadSubword(ctx, inst, BufferMemory(ctx, inst), 8u)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 1u)); } void EmitLoadBufferU16(SpirvValueEmitContext& ctx, const IrValue& inst) { - ctx.Define(inst, LoadSubword(ctx, inst, BufferMemory(ctx, inst), 16u)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 1u)); } void EmitLoadBufferU32(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 1u, false)) return; - const auto& mem = BufferMemory(ctx, inst); - ctx.Define(inst, mem.formatted ? FormattedLoad(ctx, inst, mem) : LoadWord(ctx, inst, mem)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 1u)); } void EmitLoadBufferU32x2(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 2u, false)) return; - ctx.Define(inst, LoadWideBuffer(ctx, inst, BufferMemory(ctx, inst), 2u)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 2u)); } void EmitLoadBufferU32x3(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 3u, false)) return; - ctx.Define(inst, LoadWideBuffer(ctx, inst, BufferMemory(ctx, inst), 3u)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 3u)); } void EmitLoadBufferU32x4(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 4u, false)) return; - ctx.Define(inst, LoadWideBuffer(ctx, inst, BufferMemory(ctx, inst), 4u)); + ctx.Define(inst, EmitRuntimeBufferLoad(ctx, inst, 4u)); } void EmitStoreBufferU8(SpirvValueEmitContext& ctx, const IrValue& inst) { - StoreSubword(ctx, inst, BufferMemory(ctx, inst), 8u); + EmitRuntimeBufferStore(ctx, inst, 1u); } void EmitStoreBufferU16(SpirvValueEmitContext& ctx, const IrValue& inst) { - StoreSubword(ctx, inst, BufferMemory(ctx, inst), 16u); + EmitRuntimeBufferStore(ctx, inst, 1u); } void EmitStoreBufferU32(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 1u, true)) return; - const auto& mem = BufferMemory(ctx, inst); - if (mem.formatted) { - FormattedStore(ctx, inst, mem); - } else { - StoreWord(ctx, inst, mem); - } + EmitRuntimeBufferStore(ctx, inst, 1u); } void EmitStoreBufferU32x2(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 2u, true)) return; - StoreWideBuffer(ctx, inst, BufferMemory(ctx, inst), 2u); + EmitRuntimeBufferStore(ctx, inst, 2u); } void EmitStoreBufferU32x3(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 3u, true)) return; - StoreWideBuffer(ctx, inst, BufferMemory(ctx, inst), 3u); + EmitRuntimeBufferStore(ctx, inst, 3u); } void EmitStoreBufferU32x4(SpirvValueEmitContext& ctx, const IrValue& inst) { - if (EmitGpuDescriptorAccess(ctx, inst, 4u, true)) return; - StoreWideBuffer(ctx, inst, BufferMemory(ctx, inst), 4u); + EmitRuntimeBufferStore(ctx, inst, 4u); } std::uint32_t EmitBufferAtomicSwap32(SpirvValueEmitContext& ctx, const IrValue& inst) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp index 395c88156..2689e24de 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp @@ -11,12 +11,12 @@ namespace ShaderRecompiler { -void EmitModuleHeader(SpirvModule& module, const IrProgram& program, const BindingAllocationResult& bindings) { +void EmitModuleHeader(SpirvModule& module, const IrProgram& program, const CompiledBindingLayout& bindings) { CheckBindings(program, bindings); EmitBaseHeader(module, program); } -void EmitModuleHeader(SpirvEmitterState& state, const BindingAllocationResult& bindings) { +void EmitModuleHeader(SpirvEmitterState& state, const CompiledBindingLayout& bindings) { CheckBindings(state.program, bindings); DefineModule(state); } @@ -77,6 +77,9 @@ void DefineModule(SpirvEmitterState& state) { state.module.EmitExtension(extension); } if (state.requirements.bufferInt64Atomics) { + if (std::find(state.supportedCapabilities.begin(), state.supportedCapabilities.end(), static_cast(spv::CapabilityInt64Atomics)) == state.supportedCapabilities.end()) { + FailEmit("64-bit buffer atomics need shaderBufferInt64Atomics"); + } state.module.EmitCapability(spv::CapabilityInt64); state.module.EmitCapability(spv::CapabilityInt64Atomics); } @@ -176,6 +179,14 @@ void DefineModule(SpirvEmitterState& state) { if (pixel.psEarlyZ && !pixel.psPixelKillEnable && !pixel.psDepthExportEnable && !pixel.psSampleMaskExportEnable) { state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeEarlyFragmentTests); } + if (pixel.psOrderedPixelShader) { + if (std::find(state.supportedCapabilities.begin(), state.supportedCapabilities.end(), static_cast(spv::CapabilityFragmentShaderPixelInterlockEXT)) == state.supportedCapabilities.end()) { + throw std::runtime_error("a primitive-ordered pixel shader needs the fragmentShaderPixelInterlock feature, which the device lacks"); + } + state.module.EmitCapability(spv::CapabilityFragmentShaderPixelInterlockEXT); + state.module.EmitExtension("SPV_EXT_fragment_shader_interlock"); + state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModePixelInterlockOrderedEXT); + } } if (state.requirements.computeDerivatives && StageOf(state) == IrShaderStage::Compute) { state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeDerivativeGroupQuadsKHR); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModule.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModule.cpp index 0b5adeb3b..9b6d701cc 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModule.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModule.cpp @@ -158,6 +158,15 @@ std::uint32_t SpirvModule::declareDecoratedType(std::uint32_t opcode, std::vecto return id; } +std::uint32_t SpirvModule::SpecializationConstant(std::uint32_t type, std::uint32_t constantId, std::uint32_t defaultValue) { + if (const auto found = specializationIds.find(constantId); found != specializationIds.end()) return found->second; + const auto id = AllocateId(); + appendInstruction(declarations, spv::OpSpecConstant, type, id, defaultValue); + AddAnnotation(spv::OpDecorate, id, spv::DecorationSpecId, constantId); + specializationIds.emplace(constantId, id); + return id; +} + std::uint32_t SpirvModule::declareConstant(std::uint32_t opcode, std::vector key) { if (const auto it = declarationIds.find(key); it != declarationIds.end()) { return it->second; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index 1fb490ff1..4fb0e39aa 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -5,6 +5,7 @@ #include "SpirvBackend/SpirvBda.hpp" #include #include +#include "PipelineSpecialization.hpp" #include #include #include @@ -199,6 +200,38 @@ std::uint32_t EmitInterpolationParameterValue(SpirvEmitterState& state, std::uin return bits; } +std::uint32_t EmitInterpolationParameterF16Value(SpirvEmitterState& state, std::uint32_t attr, std::uint32_t chan, std::uint32_t mode, bool high) { + const auto& pixel = *state.inputInfo.pixel; + if (pixel.InputHalfIsDefault(attr, high)) { + return ConstantF32(state, mode == 2u ? pixel.InputHalfDefaultBits(attr, chan & 3u, high) : 0u); + } + const auto* input = SpirvInputBindingForParameter(state, attr); + if (input == nullptr || !input->perVertex) { + throw std::runtime_error("16-bit interpolation parameter is not read per vertex"); + } + const auto loadVertex = [&](std::uint32_t vertex) { + const auto pointer = state.module.AllocateId(); + const auto value = state.module.AllocateId(); + auto bits = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypePointer(state, spv::StorageClassInput, TypeF32(state)), pointer, input->variableId, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)); + state.module.AddFunction(spv::OpLoad, TypeF32(state), value, pointer); + state.module.AddFunction(spv::OpBitcast, TypeU32(state), bits, value); + if (high) { + const auto shifted = state.module.AllocateId(); + state.module.AddFunction(spv::OpShiftRightLogical, TypeU32(state), shifted, bits, ConstantU32(state, 16u)); + bits = shifted; + } + return EmitF16BitsToF32(state, bits); + }; + const auto value = loadVertex((mode + 1u) % 3u); + if (mode == 2u) { + return value; + } + const auto delta = state.module.AllocateId(); + state.module.AddFunction(spv::OpFSub, TypeF32(state), delta, value, loadVertex(0u)); + return EmitF16BitsToF32(state, EmitConvertF16F32(state, delta)); +} + std::uint32_t MrtOutputMode(const SpirvEmitterState& state, const ExportInfo& exp) { if (state.program.Resources().stage != IrShaderStage::Pixel || exp.kind != ExportTargetKind::Mrt) { return 0u; @@ -927,8 +960,14 @@ void EmitProgram(SpirvEmitterState& state) { if (state.pixelValidMaskVariable != 0u) { state.module.AddFunction(spv::OpStore, state.pixelValidMaskVariable, ConstantU32(state, 1u)); } + if (OrderedPixelShader(state)) { + state.module.AddFunction(spv::OpBeginInvocationInterlockEXT); + } EmitMemoryOffsets(state); if (program.BlockOrder().empty()) { + if (OrderedPixelShader(state)) { + state.module.AddFunction(spv::OpEndInvocationInterlockEXT); + } if (state.pixelValidMaskVariable != 0u) { const auto maskValue = state.module.AllocateId(); const auto active = state.module.AllocateId(); @@ -1059,6 +1098,18 @@ std::uint32_t EmitGetBuiltin(SpirvValueEmitContext& ctx, const IrValue* kind, co } std::uint32_t EmitGetAttribute(SpirvValueEmitContext& ctx, const IrValue& inst) { + if (inst.Flags() == 1u) { + auto& state = ctx.state; + const auto attribute = inst.Argument(0)->ImmediateU32(); + const auto component = inst.Argument(1)->ImmediateU32(); + if (state.inputInfo.vertex == nullptr || !state.inputInfo.vertex->fetchEmbedded || component >= 4u) ctx.Fail(inst, "invalid prepared vertex attribute"); + const auto first = PipelineSpecialization::VertexBase + attribute * PipelineSpecialization::VertexWords; + const auto selector = state.module.SpecializationConstant(TypeU32(state), first + component, component + 4u); + const auto one = state.module.SpecializationConstant(TypeU32(state), first + 4u, 0x3f800000u); + auto value = Select(state, TypeU32(state), Binary(state, spv::OpIEqual, TypeBool(state), selector, ConstantU32(state, 1u)), one, ConstantU32(state, 0u)); + for (std::uint32_t channel = 0; channel < 4u; ++channel) value = Select(state, TypeU32(state), Binary(state, spv::OpIEqual, TypeBool(state), selector, ConstantU32(state, channel + 4u)), EmitAttributeValue(state, attribute, channel), value); + return value; + } return EmitAttributeValue(ctx.state, inst.Argument(0)->ImmediateU32(), inst.Argument(1)->ImmediateU32()); } @@ -1066,6 +1117,10 @@ std::uint32_t EmitGetInterpolationParameter(SpirvValueEmitContext& ctx, const Ir return EmitInterpolationParameterValue(ctx.state, inst.Argument(0)->ImmediateU32(), inst.Argument(1)->ImmediateU32(), inst.Argument(2)->ImmediateU32()); } +std::uint32_t EmitGetInterpolationParameterF16(SpirvValueEmitContext& ctx, const IrValue& inst) { + return EmitInterpolationParameterF16Value(ctx.state, inst.Argument(0)->ImmediateU32(), inst.Argument(1)->ImmediateU32(), inst.Argument(2)->ImmediateU32(), inst.Argument(3)->ImmediateU32() != 0u); +} + void EmitSetAttribute(SpirvValueEmitContext& ctx, const IrValue& inst) { auto& state = ctx.state; const auto& exp = ctx.Export(inst); @@ -1114,13 +1169,17 @@ void EmitSetAttribute(SpirvValueEmitContext& ctx, const IrValue& inst) { const bool uintOutput = MrtOutputMode(state, exp) == 7u; const auto vectorType = uintOutput ? TypeU32Vector(state, 4u) : TypeF32Vector(state, 4u); auto value = ExportVector(ctx, data, exp, uintOutput); - if (state.program.Resources().stage == IrShaderStage::Pixel && exp.kind == ExportTargetKind::Mrt && exp.index < state.inputInfo.pixel->targetExportMapping.size()) { - const auto& mapping = state.inputInfo.pixel->targetExportMapping.at(exp.index); - if (!mapping.IsIdentity()) { - const auto mapped = state.module.AllocateId(); - state.module.AddFunction(spv::OpVectorShuffle, vectorType, mapped, value, value, mapping.Map(0), mapping.Map(1), mapping.Map(2), mapping.Map(3)); - value = mapped; + if (state.program.Resources().stage == IrShaderStage::Pixel && exp.kind == ExportTargetKind::Mrt) { + if (exp.index >= 8u) throw std::runtime_error("fragment export target exceeds the runtime mapping table"); + std::array components{}; + for (std::uint32_t component = 0; component < components.size(); ++component) { + const auto index = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::ExportBase + exp.index * 4u + component, component); + components[component] = state.module.AllocateId(); + state.module.AddFunction(spv::OpVectorExtractDynamic, uintOutput ? TypeU32(state) : TypeF32(state), components[component], value, index); } + const auto mapped = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, vectorType, mapped, components[0], components[1], components[2], components[3]); + value = mapped; } if (exp.kind == ExportTargetKind::Position) { if (state.inputInfo.vertex == nullptr) { @@ -1164,7 +1223,7 @@ std::uint32_t EmitReadClock(SpirvValueEmitContext& ctx, const IrValue& inst, spv } std::uint32_t EmitShaderClock(SpirvValueEmitContext& ctx, const IrValue& inst) { - return EmitReadClock(ctx, inst, spv::ScopeSubgroup); + return EmitReadClock(ctx, inst, ctx.state.narrowSubgroupClock ? spv::ScopeDevice : spv::ScopeSubgroup); } std::uint32_t EmitRealtimeClock(SpirvValueEmitContext& ctx, const IrValue& inst) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp index e29ca3e36..a1b5e245b 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp @@ -1,13 +1,62 @@ #include "SpirvBackend/SpirvOptimizer.hpp" #include #include +#include #include #include #include namespace ShaderRecompiler { -std::vector ValidateAndOptimizeSpirv(std::span spirv, std::uint32_t vulkanVersion, std::uint32_t spirvVersion, bool allowOffsetTextureOperand) { +namespace { + +bool HasLimitedUseTypes(std::span spirv) { + if (spirv.size() < 5) throw std::runtime_error("SPIRV-Tools: incomplete module header"); + bool int8 = false, int16 = false, float16 = false; + bool int8Capability = false, int16Capability = false, float16Capability = false; + const bool swapped = spirv[0] != spv::MagicNumber; + const auto wordAt = [spirv, swapped](std::size_t index) { + const auto word = spirv[index]; + return swapped ? (word >> 24u) | ((word >> 8u) & 0x0000ff00u) | + ((word << 8u) & 0x00ff0000u) | (word << 24u) + : word; + }; + if (wordAt(0) != spv::MagicNumber) throw std::runtime_error("SPIRV-Tools: invalid module magic"); + for (std::size_t offset = 5; offset < spirv.size();) { + const auto instruction = wordAt(offset); + const auto wordCount = instruction >> spv::WordCountShift; + const auto opcode = instruction & spv::OpCodeMask; + if (wordCount == 0 || wordCount > spirv.size() - offset || + (opcode == spv::OpCapability && wordCount < 2) || + (opcode == spv::OpTypeInt && wordCount < 4) || + (opcode == spv::OpTypeFloat && wordCount < 3)) { + throw std::runtime_error("SPIRV-Tools: malformed instruction while scanning narrow types"); + } + switch (opcode) { + case spv::OpCapability: + int8Capability |= wordAt(offset + 1) == spv::CapabilityInt8 || + wordAt(offset + 1) == spv::CapabilityDotProductInput4x8Bit; + int16Capability |= wordAt(offset + 1) == spv::CapabilityInt16; + float16Capability |= wordAt(offset + 1) == spv::CapabilityFloat16; + break; + case spv::OpTypeInt: + int8 |= wordAt(offset + 2) == 8; + int16 |= wordAt(offset + 2) == 16; + break; + case spv::OpTypeFloat: + float16 |= wordCount == 3 && wordAt(offset + 2) == 16; + break; + case spv::OpFunction: + return (int8 && !int8Capability) || (int16 && !int16Capability) || (float16 && !float16Capability); + } + offset += wordCount; + } + return (int8 && !int8Capability) || (int16 && !int16Capability) || (float16 && !float16Capability); +} + +} + +std::vector ValidateAndOptimizeSpirv(std::span spirv, std::uint32_t vulkanVersion, std::uint32_t spirvVersion, bool allowOffsetTextureOperand, bool optimize, bool specialize) { spv_target_env environment; const auto apiVersion = vulkanVersion & ~0xfffu; std::uint32_t maxSpirvVersion = 0; @@ -37,21 +86,32 @@ std::vector ValidateAndOptimizeSpirv(std::span(spirv.begin(), spirv.end()); spvtools::Optimizer optimizer(environment); optimizer.SetMessageConsumer(consumer); static const char* mode = std::getenv("APS5_SPIRV_OPT"); - if (mode != nullptr && std::string(mode) == "none") return std::vector(spirv.begin(), spirv.end()); + if (!specialize && mode != nullptr && std::string(mode) == "none") return std::vector(spirv.begin(), spirv.end()); // The recompiler emits helpers (BDA lookup, fault reporting) as functions called from every memory // access. Exhaustive inlining multiplies module size by ~10x and optimization time by ~20x, so the // default pipeline keeps the performance passes that work per function and leaves inlining to the driver. // APS5_SPIRV_OPT=full restores the SPIRV-Tools performance pipeline, =none skips optimization. - if (mode == nullptr || std::string(mode) != "full") { + const bool allowFolding = !HasLimitedUseTypes(spirv); + const bool full = mode != nullptr && std::string(mode) == "full"; + if (full && allowFolding) { + optimizer.RegisterPerformancePasses(true); + } else { optimizer.RegisterPass(spvtools::CreateWrapOpKillPass()) - .RegisterPass(spvtools::CreateDeadBranchElimPass()) - .RegisterPass(spvtools::CreateAggressiveDCEPass(true)) + .RegisterPass(spvtools::CreateDeadBranchElimPass()); + if (full) { + optimizer.RegisterPass(spvtools::CreateMergeReturnPass()) + .RegisterPass(spvtools::CreateInlineExhaustivePass()) + .RegisterPass(spvtools::CreateEliminateDeadFunctionsPass()); + } + optimizer.RegisterPass(spvtools::CreateAggressiveDCEPass(true)) .RegisterPass(spvtools::CreatePrivateToLocalPass()) .RegisterPass(spvtools::CreateLocalSingleBlockLoadStoreElimPass()) .RegisterPass(spvtools::CreateLocalSingleStoreElimPass()) @@ -62,23 +122,43 @@ std::vector ValidateAndOptimizeSpirv(std::span optimized; diagnostics.clear(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp new file mode 100644 index 000000000..59279223c --- /dev/null +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -0,0 +1,486 @@ +#include "SpirvBackend/SpirvRuntimeBuffer.hpp" +#include "SpirvBackend/SpirvEmitterInstructions.hpp" +#include "SpirvBackend/SpirvBda.hpp" +#include "SpirvBackend/SpirvBufferFormat.hpp" +#include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp" +#include "SpirvBackend/SpirvMemory/SpirvInputOutput.hpp" +#include "PipelineSpecialization.hpp" +#include +#include +#include + +namespace ShaderRecompiler { +namespace { + +struct RuntimeDescriptor { + std::uint32_t base; + std::uint32_t records; + std::uint32_t stride; + std::uint32_t word3; + std::uint32_t swizzle; + std::uint32_t index; + std::uint32_t valid; +}; + +struct RuntimeAddress { + std::uint32_t guest; + std::uint32_t inBounds; +}; + +std::uint32_t field(SpirvEmitterState& state, std::uint32_t word, std::uint32_t offset, std::uint32_t count) { + return EmitBitFieldUExtract(state, word, ConstantU32(state, offset), ConstantU32(state, count)); +} + +std::uint32_t both(SpirvEmitterState& state, std::uint32_t left, std::uint32_t right) { + return Binary(state, spv::OpLogicalAnd, TypeBool(state), left, right); +} + +std::uint32_t equal(SpirvEmitterState& state, std::uint32_t left, std::uint32_t right) { + return Binary(state, spv::OpIEqual, TypeBool(state), left, right); +} + +std::uint32_t nonzero(SpirvEmitterState& state, std::uint32_t value) { + return Binary(state, spv::OpINotEqual, TypeBool(state), value, ConstantU32(state, 0u)); +} + +void faultIf(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, BdaAbi::FaultReason reason) { + auto& state = context.state; + EmitIfCondition(state, condition, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), ConstantU32(state, instruction.Flags().pc), reason); }); + StopBdaInvocationIf(state, condition); +} + +RuntimeDescriptor descriptor(SpirvValueEmitContext& context, const IrValue& instruction) { + auto& state = context.state; + const auto* handle = instruction.Argument(0)->Resolve(); + if (handle->Opcode() != IrOpcode::GetBufferResource || handle->ArgumentCount() != 4u) context.Fail(instruction, "requires a runtime four-dword V#"); + const auto& memory = context.Memory(instruction); + const auto first = PipelineSpecialization::BufferBase + memory.resource * PipelineSpecialization::BufferWords; + const auto word1 = memory.gpuDescriptor ? context.Arg(*handle, 1u) : state.module.SpecializationConstant(TypeU32(state), first, 0u); + const auto word3 = memory.gpuDescriptor ? context.Arg(*handle, 3u) : state.module.SpecializationConstant(TypeU32(state), first + 1u, 0u); + const auto u32 = TypeU32(state); + auto base = ConstantU32(state, 0u); + if (memory.gpuDescriptor || BufferAccessOf(instruction.Opcode()) == BufferAccess::Atomic) { + const auto u64 = TypeScalarU64(state); + const auto high = Binary(state, spv::OpShiftLeftLogical, u64, Unary(state, spv::OpUConvert, u64, field(state, context.Arg(*handle, 1u), 0u, 16u)), BdaConstant(state, 32u)); + base = Binary(state, spv::OpBitwiseOr, u64, Unary(state, spv::OpUConvert, u64, context.Arg(*handle, 0u)), high); + } + auto invalid = nonzero(state, field(state, word3, 30u, 2u)); + auto index = ConstantU32(state, 0u); + if (context.Memory(instruction).kind == ResourceKind::Buffer) { + index = context.Arg(instruction, 1u); + const auto addTid = nonzero(state, field(state, word3, 23u, 1u)); + if (state.program.Resources().stage == IrShaderStage::Compute) { + const auto lane = state.laneCount == 1u && state.hostSubgroupSize < state.program.WaveSize() ? + Binary(state, spv::OpBitwiseAnd, u32, EmitLocalInvocationIndex(state), ConstantU32(state, state.program.WaveSize() - 1u)) : EmitSubgroupLocalInvocationId(state); + index = Binary(state, spv::OpIAdd, u32, index, Select(state, u32, addTid, lane, ConstantU32(state, 0u))); + } else { + invalid = Binary(state, spv::OpLogicalOr, TypeBool(state), invalid, addTid); + } + } + if (memory.gpuDescriptor) faultIf(context, instruction, invalid, base, 0u, BdaAbi::FaultReason::InvalidDescriptor); + const auto stride = field(state, word1, 16u, 14u); + const auto valid = memory.gpuDescriptor ? both(state, Unary(state, spv::OpLogicalNot, TypeBool(state), invalid), Binary(state, spv::OpINotEqual, TypeBool(state), base, BdaConstant(state, 0u))) : nonzero(state, state.module.SpecializationConstant(u32, first + 2u, 0u)); + return {base, context.Arg(*handle, 2u), stride, word3, both(state, nonzero(state, stride), nonzero(state, field(state, word1, 31u, 1u))), index, valid}; +} + +RuntimeAddress address(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t componentOffset, std::uint32_t bytes) { + auto& state = context.state; + const auto u32 = TypeU32(state); + const auto boolean = TypeBool(state); + const auto constant = [&](std::uint32_t value) { return ConstantU32(state, value); }; + const auto add = [&](std::uint32_t left, std::uint32_t right) { return Binary(state, spv::OpIAdd, u32, left, right); }; + const auto mul = [&](std::uint32_t left, std::uint32_t right) { return Binary(state, spv::OpIMul, u32, left, right); }; + const auto less = [&](std::uint32_t left, std::uint32_t right) { return Binary(state, spv::OpULessThan, boolean, left, right); }; + const auto fits = [&](std::uint32_t offset, std::uint32_t size) { + return both(state, Binary(state, spv::OpUGreaterThanEqual, boolean, size, constant(bytes)), Binary(state, spv::OpULessThanEqual, boolean, offset, Binary(state, spv::OpISub, u32, size, constant(bytes)))); + }; + const auto offset = add(context.Arg(instruction, 2u), constant(context.Memory(instruction).offset + componentOffset)); + const auto soffset = context.Arg(instruction, 3u); + const auto indexShift = add(field(state, resource.word3, 21u, 2u), constant(3u)); + const auto indices = Binary(state, spv::OpShiftLeftLogical, u32, constant(1u), indexShift); + const auto indexMsb = Binary(state, spv::OpShiftRightLogical, u32, resource.index, indexShift); + const auto indexLsb = Binary(state, spv::OpBitwiseAnd, u32, resource.index, Binary(state, spv::OpISub, u32, indices, constant(1u))); + const auto offsetMsb = Binary(state, spv::OpBitwiseAnd, u32, offset, constant(~3u)); + const auto offsetLsb = Binary(state, spv::OpBitwiseAnd, u32, offset, constant(3u)); + const auto swizzled = add(mul(add(mul(indexMsb, resource.stride), offsetMsb), indices), add(Binary(state, spv::OpShiftLeftLogical, u32, indexLsb, constant(2u)), offsetLsb)); + const auto linear = add(mul(resource.index, resource.stride), offset); + const auto byte = add(Select(state, u32, resource.swizzle, swizzled, linear), soffset); + const auto mode = field(state, resource.word3, 28u, 2u); + const auto indexInBounds = less(resource.index, resource.records); + const auto structured = both(state, indexInBounds, less(offset, resource.stride)); + const auto rawRecords = Binary(state, spv::OpISub, u32, resource.records, soffset); + const auto raw = both(state, Binary(state, spv::OpULessThanEqual, boolean, soffset, resource.records), Select(state, boolean, resource.swizzle, both(state, less(resource.index, rawRecords), fits(offset, resource.stride)), fits(offset, rawRecords))); + auto inBounds = Select(state, boolean, equal(state, mode, constant(0u)), structured, indexInBounds); + inBounds = Select(state, boolean, less(mode, constant(2u)), inBounds, Select(state, boolean, equal(state, mode, constant(2u)), nonzero(state, resource.records), raw)); + inBounds = both(state, resource.valid, both(state, nonzero(state, field(state, resource.word3, 12u, 7u)), inBounds)); + if (!context.Memory(instruction).gpuDescriptor) return {byte, inBounds}; + return {AddBdaAddress(context, instruction, resource.base, Unary(state, spv::OpUConvert, TypeScalarU64(state), byte), false), inBounds}; +} + +MemoryResourceAccess directAccess(SpirvValueEmitContext& context, const IrValue& instruction, bool wide = false) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + const auto variable = wide ? state.storageBufferU64Variable : state.storageBufferVariable; + if (variable == 0u) context.Fail(instruction, "direct buffer descriptor array is missing"); + const auto element = ResourceForDescriptor(state, DescriptorBindingKind::Buffers, memory.resource); + MemoryResourceAccess access; + access.kind = memory.kind; + access.objectPointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, wide ? TypeStorageBufferU64Pointer(state) : TypeStorageBufferPointer(state), access.objectPointer, variable, ConstantU32(state, element)); + access.byteOffset = state.memoryByteOffsets.at(element); + access.memoryAccess = memory.coherent ? spv::MemoryAccessVolatileMask : 0u; + access.length = state.module.AllocateId(); + state.module.AddFunction(spv::OpArrayLength, TypeU32(state), access.length, access.objectPointer, 0u); + return access; +} + +std::uint32_t baseMisalignment(SpirvValueEmitContext& context, const IrValue& instruction) { + const auto& memory = context.Memory(instruction); + if (memory.kind == ResourceKind::ScalarBuffer) return ConstantU32(context.state, 0u); + const auto first = PipelineSpecialization::BufferBase + memory.resource * PipelineSpecialization::BufferWords; + return context.state.module.SpecializationConstant(TypeU32(context.state), first + 3u, 0u); +} + +std::uint32_t directWordIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t address) { + const auto relative = Binary(state, spv::OpShiftRightLogical, TypeU32(state), address, ConstantU32(state, 2u)); + const auto base = Binary(state, spv::OpShiftRightLogical, TypeU32(state), access.byteOffset, ConstantU32(state, 2u)); + return EmitAddU32(state, relative, base); +} + +std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t bits) { + if (context.Memory(instruction).gpuDescriptor) return EmitBdaRead(context, instruction, address, bits); + auto& state = context.state; + const auto access = directAccess(context, instruction); + const auto index = directWordIndex(state, access, address); + const auto misalignment = baseMisalignment(context, instruction); + const auto unaligned = nonzero(state, misalignment); + const auto next = EmitAddU32(state, index, ConstantU32(state, 1u)); + const auto last = Select(state, TypeU32(state), unaligned, next, index); + return EmitValueOrZeroIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] { + const auto load = [&](std::uint32_t word) { + const auto value = state.module.AllocateId(); + const auto pointer = EmitMemoryElementPointer(state, access, word); + if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, access.memoryAccess); + else state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); + return value; + }; + const auto shift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), misalignment, ConstantU32(state, 3u)); + const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(index), shift); + const auto high = EmitValueOrZeroIfCondition(state, unaligned, [&] { + return Binary(state, spv::OpShiftLeftLogical, TypeU32(state), load(next), Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32u), shift)); + }); + auto value = EmitOrU32(state, low, high); + if (bits != 32u) { + const auto subwordShift = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), EmitAndConstant(state, address, 3u), ConstantU32(state, 3u)); + value = Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, subwordShift); + } + return EmitAndConstant(state, value, bits == 32u ? 0xffffffffu : (1u << bits) - 1u); + }); +} + +void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u) { + if (context.Memory(instruction).gpuDescriptor) { + EmitBdaWrite(context, instruction, address, value, bits); + return; + } + auto& state = context.state; + const auto access = directAccess(context, instruction); + const auto u32 = TypeU32(state); + const auto index = directWordIndex(state, access, address); + const auto misalignment = baseMisalignment(context, instruction); + const auto unaligned = nonzero(state, misalignment); + const auto next = EmitAddU32(state, index, ConstantU32(state, 1u)); + const auto last = Select(state, u32, unaligned, next, index); + EmitIfCondition(state, EmitMemoryElementInBounds(state, access, last), [&] { + auto mask = ConstantU32(state, bits == 32u ? 0xffffffffu : (1u << bits) - 1u); + auto masked = Binary(state, spv::OpBitwiseAnd, u32, value, mask); + if (bits != 32u) { + const auto subwordShift = Binary(state, spv::OpShiftLeftLogical, u32, EmitAndConstant(state, address, 3u), ConstantU32(state, 3u)); + mask = Binary(state, spv::OpShiftLeftLogical, u32, mask, subwordShift); + masked = Binary(state, spv::OpShiftLeftLogical, u32, masked, subwordShift); + } + const auto update = [&](std::uint32_t word, std::uint32_t wordMask, std::uint32_t replacement) { + AtomicUpdate(state, EmitMemoryElementPointer(state, access, word), access.kind, [&](std::uint32_t old) { + return EmitOrU32(state, Binary(state, spv::OpBitwiseAnd, u32, old, Unary(state, spv::OpNot, u32, wordMask)), replacement); + }); + }; + if (bits == 32u) { + EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), [&] { + const auto pointer = EmitMemoryElementPointer(state, access, index); + if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpStore, pointer, value, access.memoryAccess); + else state.module.AddFunction(spv::OpStore, pointer, value); + }); + } + EmitIfCondition(state, bits == 32u ? unaligned : ConstantBool(state, true), [&] { + const auto shift = Binary(state, spv::OpShiftLeftLogical, u32, misalignment, ConstantU32(state, 3u)); + update(index, Binary(state, spv::OpShiftLeftLogical, u32, mask, shift), Binary(state, spv::OpShiftLeftLogical, u32, masked, shift)); + EmitIfCondition(state, unaligned, [&] { + const auto remaining = Binary(state, spv::OpISub, u32, ConstantU32(state, 32u), shift); + update(next, Binary(state, spv::OpShiftRightLogical, u32, mask, remaining), Binary(state, spv::OpShiftRightLogical, u32, masked, remaining)); + }); + }); + }); +} + +std::uint32_t composite(SpirvEmitterState& state, std::uint32_t components, const std::array& values) { + if (components == 1u) return values[0]; + const auto result = state.module.AllocateId(); + std::vector words{spv::OpCompositeConstruct, TypeU32Composite(state, components), result}; + words.insert(words.end(), values.begin(), values.begin() + components); + state.module.AddFunction(words); + return result; +} + +std::array storeValues(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) { + auto& state = context.state; + std::array result{}; + const auto data = context.Arg(instruction, instruction.ArgumentCount() - 2u); + for (std::uint32_t component = 0; component < components; ++component) { + result[component] = data; + if (components != 1u) { + result[component] = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), result[component], data, component); + } + } + return result; +} + +std::uint32_t formatted(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, const SpirvBufferFormatInfo& info, std::uint32_t components, bool store) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + const auto u32 = TypeU32(state); + std::array values{}; + std::array selectors{}; + std::array selected{}; + std::array addresses{}; + auto inBounds = ConstantBool(state, true); + for (std::uint32_t output = 0; output < components; ++output) { + selectors[output] = memory.typed ? ConstantU32(state, output < info.componentCount ? 4u + output : 0u) : field(state, resource.word3, output * 3u, 3u); + selected[output] = Binary(state, spv::OpUMod, u32, Binary(state, spv::OpISub, u32, selectors[output], ConstantU32(state, 4u)), ConstantU32(state, info.componentCount)); + } + std::array required{}; + for (std::uint32_t component = 0; component < info.componentCount; ++component) { + required[component] = ConstantBool(state, store); + if (!store) { + for (std::uint32_t output = 0; output < components; ++output) { + const auto fromMemory = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), selectors[output], ConstantU32(state, 4u)); + required[component] = Binary(state, spv::OpLogicalOr, TypeBool(state), required[component], both(state, fromMemory, equal(state, selected[output], ConstantU32(state, component)))); + } + } + addresses[component] = address(context, instruction, resource, GetFormatComponentByteOffset(info, component), info.packedBitfield ? 4u : info.componentBits[component] / 8u); + inBounds = both(state, inBounds, Binary(state, spv::OpLogicalOr, TypeBool(state), Unary(state, spv::OpLogicalNot, TypeBool(state), required[component]), addresses[component].inBounds)); + } + if (store) { + const auto data = storeValues(context, instruction, components); + EmitIfCondition(state, inBounds, [&] { + auto packed = ConstantU32(state, 0u); + for (std::uint32_t component = 0; component < info.componentCount; ++component) { + const auto encoded = component >= components ? ConstantU32(state, 0u) : memory.d16 ? EmitD16StoreComponent(state, info, component, data[component]) : EmitFormatStoreComponent(state, info, component, data[component]); + if (info.packedBitfield) { + packed = EmitOrU32(state, packed, Binary(state, spv::OpShiftLeftLogical, u32, EmitAndConstant(state, encoded, (1u << info.componentBits[component]) - 1u), ConstantU32(state, info.componentBitOffset[component]))); + } else { + writeBuffer(context, instruction, addresses[component].guest, encoded, info.componentBits[component]); + } + } + if (info.packedBitfield) writeBuffer(context, instruction, addresses[0].guest, packed); + }); + return ConstantU32(state, 0u); + } + for (std::uint32_t component = 0; component < info.componentCount; ++component) { + values[component] = EmitValueOrZeroIfCondition(state, both(state, inBounds, required[component]), [&] { + const auto bits = info.componentBits[component]; + auto raw = readBuffer(context, instruction, addresses[component].guest, info.packedBitfield ? 32u : bits); + if (bits != 32u) { + const auto signedType = IsSignedFormatComponent(info.type); + const auto type = signedType ? TypeI32(state) : u32; + const auto input = signedType ? Unary(state, spv::OpBitcast, type, raw) : raw; + raw = state.module.AllocateId(); + state.module.AddFunction(signedType ? spv::OpBitFieldSExtract : spv::OpBitFieldUExtract, type, raw, input, ConstantU32(state, info.packedBitfield ? info.componentBitOffset[component] : 0u), ConstantU32(state, bits)); + if (signedType) raw = Unary(state, spv::OpBitcast, u32, raw); + } + return memory.d16 ? EmitD16FormatComponent(state, info, component, raw) : NormalizeFormatComponent(state, info, component, raw); + }); + } + const bool integer = info.type == SpirvFormatComponentType::Uint || info.type == SpirvFormatComponentType::Sint; + const auto one = ConstantU32(state, integer ? 1u : memory.d16 ? 0x3c00u : 0x3f800000u); + std::array outputs{}; + for (std::uint32_t output = 0; output < components; ++output) { + auto value = values[0]; + for (std::uint32_t component = 1; component < info.componentCount; ++component) value = Select(state, u32, equal(state, selected[output], ConstantU32(state, component)), values[component], value); + value = Select(state, u32, equal(state, selectors[output], ConstantU32(state, 1u)), one, value); + outputs[output] = Select(state, u32, equal(state, selectors[output], ConstantU32(state, 0u)), ConstantU32(state, 0u), value); + } + return composite(state, components, outputs); +} + +std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + if (memory.typed) { + const auto info = GetFormatInfo(DecodeTBufferFormat(memory.dataFormat, memory.numberFormat)); + if (info.type == SpirvFormatComponentType::Unknown) context.Fail(instruction, "typed buffer instruction has an invalid format"); + return formatted(context, instruction, resource, info, components, store); + } + if (!store && memory.gpuDescriptor) { + auto invalid = ConstantBool(state, false); + for (std::uint32_t component = 0; component < components; ++component) { + const auto selector = field(state, resource.word3, component * 3u, 3u); + invalid = Binary(state, spv::OpLogicalOr, TypeBool(state), invalid, Binary(state, spv::OpLogicalOr, TypeBool(state), equal(state, selector, ConstantU32(state, 2u)), equal(state, selector, ConstantU32(state, 3u)))); + } + faultIf(context, instruction, invalid, resource.base, 0u, BdaAbi::FaultReason::InvalidDescriptor); + } + const auto* word = instruction.Argument(0)->Resolve()->Argument(3)->Resolve(); + if (memory.gpuDescriptor && word->HasImmediate()) { + const auto format = static_cast((word->ImmediateU32() >> 12u) & 0x7fu); + if (format == IrBufferFormat::Invalid) context.Fail(instruction, "formatted buffer instruction has an invalid constant format"); + return formatted(context, instruction, resource, GetFormatInfo(format), components, store); + } + const auto format = field(state, resource.word3, 12u, 7u); + const auto merge = state.module.AllocateId(); + const auto invalid = state.module.AllocateId(); + constexpr std::uint32_t count = static_cast(IrBufferFormat::Format32_32_32_32Float); + std::array labels{}; + std::vector branch{spv::OpSwitch, format, invalid}; + for (std::uint32_t index = 0; index < count; ++index) { + labels[index] = state.module.AllocateId(); + branch.insert(branch.end(), {index + 1u, labels[index]}); + } + const auto type = store || components == 1u ? TypeU32(state) : TypeU32Composite(state, components); + const auto result = state.module.AllocateId(); + std::vector phi{spv::OpPhi, type, result}; + state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + state.module.AddFunction(branch); + for (std::uint32_t index = 0; index < count; ++index) { + EmitLabel(state, labels[index]); + const auto value = formatted(context, instruction, resource, GetFormatInfo(static_cast(index + 1u)), components, store); + const auto exit = state.module.AllocateId(); + state.module.AddFunction(spv::OpBranch, exit); + EmitLabel(state, exit); + state.module.AddFunction(spv::OpBranch, merge); + phi.insert(phi.end(), {value, exit}); + } + EmitLabel(state, invalid); + if (memory.gpuDescriptor) RecordBdaFault(state, resource.base, ConstantU32(state, 0u), ConstantU32(state, instruction.Flags().pc), BdaAbi::FaultReason::InvalidDescriptor); + const auto zero = store || components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); + const auto invalidExit = state.module.AllocateId(); + state.module.AddFunction(spv::OpBranch, invalidExit); + EmitLabel(state, invalidExit); + state.module.AddFunction(spv::OpBranch, merge); + phi.insert(phi.end(), {zero, invalidExit}); + EmitLabel(state, merge); + state.module.AddFunction(phi); + const auto unsupported = Binary(state, spv::OpLogicalOr, TypeBool(state), equal(state, format, ConstantU32(state, 0u)), Binary(state, spv::OpUGreaterThan, TypeBool(state), format, ConstantU32(state, count))); + if (memory.gpuDescriptor) StopBdaInvocationIf(state, unsupported); + return result; +} + +} + +std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + const auto type = components == 1u ? TypeU32(state) : TypeU32Composite(state, components); + const auto zero = components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); + return EmitValueOrDefaultIfCondition(state, context.Arg(instruction, instruction.ArgumentCount() - 1u), type, zero, [&] { + const auto resource = descriptor(context, instruction); + if (memory.formatted) return EmitValueOrDefaultIfCondition(state, resource.valid, type, zero, [&] { return formattedDispatch(context, instruction, resource, components, false); }); + const auto readComponents = [&] { + std::array values{}; + for (std::uint32_t component = 0; component < components; ++component) { + const auto access = address(context, instruction, resource, component * 4u, memory.dataBits / 8u); + values[component] = EmitValueOrZeroIfCondition(state, access.inBounds, [&] { return readBuffer(context, instruction, access.guest, memory.dataBits); }); + } + return composite(state, components, values); + }; + if (components != 2u || !memory.coherent || memory.gpuDescriptor || state.storageBufferU64Variable == 0u) return readComponents(); + const auto runtime = address(context, instruction, resource, 0u, 8u); + const auto wide = directAccess(context, instruction, true); + const auto byte = EmitAddU32(state, runtime.guest, wide.byteOffset); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte, ConstantU32(state, 3u)); + const auto aligned = both(state, equal(state, baseMisalignment(context, instruction), ConstantU32(state, 0u)), equal(state, EmitAndConstant(state, byte, 7u), ConstantU32(state, 0u))); + const auto inBounds = both(state, runtime.inBounds, EmitMemoryElementInBounds(state, wide, index)); + return EmitValueIfElse(state, both(state, aligned, inBounds), type, [&] { + const auto pointer = EmitStorageBufferElementPointer(state, wide, index, TypeStorageBufferU64ElementPointer(state)); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicLoad, TypeScalarU64(state), value, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone)); + const auto high = Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), value, BdaConstant(state, 32u)); + return composite(state, 2u, {Unary(state, spv::OpUConvert, TypeU32(state), value), Unary(state, spv::OpUConvert, TypeU32(state), high), 0u, 0u}); + }, readComponents); + }); +} + +void EmitRuntimeBufferStore(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + EmitIfCondition(state, context.Arg(instruction, instruction.ArgumentCount() - 1u), [&] { + const auto resource = descriptor(context, instruction); + if (memory.formatted) { + EmitIfCondition(state, resource.valid, [&] { static_cast(formattedDispatch(context, instruction, resource, components, true)); }); + return; + } + const auto values = storeValues(context, instruction, components); + for (std::uint32_t component = 0; component < components; ++component) { + const auto access = address(context, instruction, resource, component * 4u, memory.dataBits / 8u); + EmitIfCondition(state, access.inBounds, [&] { writeBuffer(context, instruction, access.guest, values[component], memory.dataBits); }); + } + }); +} + +std::uint32_t EmitRuntimeScalarBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction) { + auto& state = context.state; + if (std::ranges::find(state.supportedCapabilities, spv::CapabilityInt64) == state.supportedCapabilities.end()) context.Fail(instruction, "scalar buffer bounds require shaderInt64"); + state.module.EmitCapability(spv::CapabilityInt64); + const auto resource = descriptor(context, instruction); + const auto u64 = TypeScalarU64(state); + const auto size = Select(state, u64, equal(state, resource.stride, ConstantU32(state, 0u)), Unary(state, spv::OpUConvert, u64, resource.records), Binary(state, spv::OpIMul, u64, Unary(state, spv::OpUConvert, u64, resource.stride), Unary(state, spv::OpUConvert, u64, resource.records))); + const auto address = EmitAddU32(state, context.Arg(instruction, 1u), ConstantU32(state, context.Memory(instruction).offset)); + const auto noCarry = Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), address, context.Arg(instruction, 1u)); + const auto byte = EmitAndConstant(state, address, ~3u); + const auto offset = Unary(state, spv::OpUConvert, u64, byte); + const auto inBounds = both(state, noCarry, both(state, resource.valid, Binary(state, spv::OpULessThanEqual, TypeBool(state), Binary(state, spv::OpIAdd, u64, offset, BdaConstant(state, 4u)), size))); + if (!context.Memory(instruction).gpuDescriptor) return EmitValueOrZeroIfCondition(state, inBounds, [&] { return readBuffer(context, instruction, byte, 32u); }); + return EmitValueOrZeroIfCondition(state, inBounds, [&] { return readBuffer(context, instruction, AddBdaAddress(context, instruction, resource.base, offset, false), 32u); }); +} + +std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t bits, std::uint32_t active, const std::function& operation) { + auto& state = context.state; + if (bits != 32u && bits != 64u) context.Fail(instruction, "unsupported runtime buffer atomic width"); + if (context.Memory(instruction).gpuDescriptor && (state.bdaAtomicPointerFunction == 0u || state.bdaNoteWriteFunction == 0u)) context.Fail(instruction, "BDA atomic functions are missing"); + const auto type = bits == 64u ? TypeU64(state) : TypeU32(state); + const auto scalar = bits == 64u ? TypeScalarU64(state) : TypeU32(state); + const auto zero = bits == 64u ? ConstantU64(state, 0u) : ConstantU32(state, 0u); + return EmitValueOrDefaultIfCondition(state, active, type, zero, [&] { + const auto resource = descriptor(context, instruction); + const auto access = address(context, instruction, resource, 0u, bits / 8u); + return EmitValueOrDefaultIfCondition(state, access.inBounds, type, zero, [&] { + if (!context.Memory(instruction).gpuDescriptor) { + const auto storage = directAccess(context, instruction, bits == 64u); + const auto byte = EmitAddU32(state, access.guest, storage.byteOffset); + const auto unaligned = nonzero(state, EmitAndConstant(state, byte, bits / 8u - 1u)); + const auto guest = Binary(state, spv::OpIAdd, TypeScalarU64(state), resource.base, Unary(state, spv::OpUConvert, TypeScalarU64(state), access.guest)); + faultIf(context, instruction, unaligned, guest, bits / 8u, BdaAbi::FaultReason::Unaligned); + const auto index = Binary(state, spv::OpShiftRightLogical, TypeU32(state), byte, ConstantU32(state, bits == 64u ? 3u : 2u)); + return EmitValueOrDefaultIfCondition(state, both(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), EmitMemoryElementInBounds(state, storage, index)), type, zero, [&] { + const auto pointer = EmitStorageBufferElementPointer(state, storage, index, bits == 64u ? TypeStorageBufferU64ElementPointer(state) : TypeStorageBufferElementPointer(state)); + return operation(pointer); + }); + } + const auto unaligned = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), access.guest, BdaConstant(state, bits / 8u - 1u)), BdaConstant(state, 0u)); + faultIf(context, instruction, unaligned, access.guest, bits / 8u, BdaAbi::FaultReason::Unaligned); + return EmitValueOrDefaultIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), type, zero, [&] { + const auto physical = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaAtomicPointerFunction, access.guest, ConstantU32(state, bits / 8u), ConstantU32(state, instruction.Flags().pc)); + const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)); + return EmitValueOrDefaultIfCondition(state, mapped, type, zero, [&] { + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, scalar), pointer, physical); + const auto result = operation(pointer); + state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, access.guest); + return result; + }); + }); + }); + }); +} + +} diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvSpecialization.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvSpecialization.cpp new file mode 100644 index 000000000..2a62cbcde --- /dev/null +++ b/core/shader/recompiler/SpirvBackend/src/SpirvSpecialization.cpp @@ -0,0 +1,626 @@ +#define SPV_ENABLE_UTILITY_CODE +#include +#undef SPV_ENABLE_UTILITY_CODE +#include "SpirvBackend/SpirvSpecialization.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { +namespace { + +using Instruction = std::vector; + +spv::Op Opcode(const Instruction& instruction) { + return instruction.empty() ? spv::OpNop : static_cast(instruction[0] & 0xffffu); +} + +std::uint32_t Result(const Instruction& instruction) { + if (instruction.empty()) return 0u; + bool result = false; + bool type = false; + spv::HasResultAndType(Opcode(instruction), &result, &type); + return result ? instruction.at(type ? 2u : 1u) : 0u; +} + +Instruction Make(spv::Op op, std::initializer_list operands) { + Instruction result{(static_cast(operands.size() + 1u) << 16u) | op}; + result.insert(result.end(), operands); + return result; +} + +bool PureInstruction(const Instruction& instruction) { + switch (Opcode(instruction)) { + case spv::OpVectorExtractDynamic: + case spv::OpVectorInsertDynamic: + case spv::OpVectorShuffle: + case spv::OpCompositeConstruct: + case spv::OpCompositeExtract: + case spv::OpCompositeInsert: + case spv::OpCopyObject: + case spv::OpTranspose: + case spv::OpConvertFToU: + case spv::OpConvertFToS: + case spv::OpConvertSToF: + case spv::OpConvertUToF: + case spv::OpUConvert: + case spv::OpSConvert: + case spv::OpFConvert: + case spv::OpQuantizeToF16: + case spv::OpConvertPtrToU: + case spv::OpSatConvertSToU: + case spv::OpSatConvertUToS: + case spv::OpConvertUToPtr: + case spv::OpPtrCastToGeneric: + case spv::OpGenericCastToPtr: + case spv::OpGenericCastToPtrExplicit: + case spv::OpBitcast: + case spv::OpSNegate: + case spv::OpFNegate: + case spv::OpIAdd: + case spv::OpFAdd: + case spv::OpISub: + case spv::OpFSub: + case spv::OpIMul: + case spv::OpFMul: + case spv::OpUDiv: + case spv::OpSDiv: + case spv::OpFDiv: + case spv::OpUMod: + case spv::OpSRem: + case spv::OpSMod: + case spv::OpFRem: + case spv::OpFMod: + case spv::OpVectorTimesScalar: + case spv::OpMatrixTimesScalar: + case spv::OpVectorTimesMatrix: + case spv::OpMatrixTimesVector: + case spv::OpMatrixTimesMatrix: + case spv::OpOuterProduct: + case spv::OpDot: + case spv::OpIAddCarry: + case spv::OpISubBorrow: + case spv::OpUMulExtended: + case spv::OpSMulExtended: + case spv::OpAny: + case spv::OpAll: + case spv::OpIsNan: + case spv::OpIsInf: + case spv::OpIsFinite: + case spv::OpIsNormal: + case spv::OpSignBitSet: + case spv::OpLessOrGreater: + case spv::OpOrdered: + case spv::OpUnordered: + case spv::OpLogicalEqual: + case spv::OpLogicalNotEqual: + case spv::OpLogicalOr: + case spv::OpLogicalAnd: + case spv::OpLogicalNot: + case spv::OpSelect: + case spv::OpIEqual: + case spv::OpINotEqual: + case spv::OpUGreaterThan: + case spv::OpSGreaterThan: + case spv::OpUGreaterThanEqual: + case spv::OpSGreaterThanEqual: + case spv::OpULessThan: + case spv::OpSLessThan: + case spv::OpULessThanEqual: + case spv::OpSLessThanEqual: + case spv::OpFOrdEqual: + case spv::OpFUnordEqual: + case spv::OpFOrdNotEqual: + case spv::OpFUnordNotEqual: + case spv::OpFOrdLessThan: + case spv::OpFUnordLessThan: + case spv::OpFOrdGreaterThan: + case spv::OpFUnordGreaterThan: + case spv::OpFOrdLessThanEqual: + case spv::OpFUnordLessThanEqual: + case spv::OpFOrdGreaterThanEqual: + case spv::OpFUnordGreaterThanEqual: + case spv::OpShiftRightLogical: + case spv::OpShiftRightArithmetic: + case spv::OpShiftLeftLogical: + case spv::OpBitwiseOr: + case spv::OpBitwiseXor: + case spv::OpBitwiseAnd: + case spv::OpNot: + case spv::OpBitFieldInsert: + case spv::OpBitFieldSExtract: + case spv::OpBitFieldUExtract: + case spv::OpBitReverse: + case spv::OpBitCount: + case spv::OpDPdx: + case spv::OpDPdy: + case spv::OpFwidth: + case spv::OpDPdxFine: + case spv::OpDPdyFine: + case spv::OpFwidthFine: + case spv::OpDPdxCoarse: + case spv::OpDPdyCoarse: + case spv::OpFwidthCoarse: + case spv::OpAccessChain: + case spv::OpInBoundsAccessChain: + case spv::OpPtrAccessChain: + case spv::OpInBoundsPtrAccessChain: + case spv::OpArrayLength: + case spv::OpPhi: + return true; + case spv::OpLoad: + return instruction.size() == 4u || (instruction.size() == 5u && instruction[4] == spv::MemoryAccessMaskNone); + default: + return false; + } +} + +template +bool VisitInputs(const Instruction& instruction, const TVisitor& visit) { + std::size_t first = 3u; + std::size_t end = instruction.size(); + switch (Opcode(instruction)) { + case spv::OpVectorShuffle: + case spv::OpCompositeInsert: + end = 5u; + break; + case spv::OpCompositeExtract: + case spv::OpGenericCastToPtrExplicit: + case spv::OpArrayLength: + end = 4u; + break; + case spv::OpLoad: + if (instruction.size() > 5u) return false; + end = 4u; + break; + case spv::OpStore: + if (instruction.size() > 4u) return false; + first = 1u; + end = 3u; + break; + case spv::OpBranch: + case spv::OpReturnValue: + first = 1u; + end = 2u; + break; + case spv::OpBranchConditional: + first = 1u; + end = 4u; + break; + case spv::OpSelectionMerge: + first = 1u; + end = 2u; + break; + case spv::OpLoopMerge: + first = 1u; + end = 3u; + break; + case spv::OpFunctionCall: + break; + case spv::OpReturn: + case spv::OpUnreachable: + case spv::OpKill: + case spv::OpLabel: + case spv::OpFunction: + case spv::OpFunctionEnd: + case spv::OpFunctionParameter: + case spv::OpLine: + case spv::OpNoLine: + return true; + default: + if (!PureInstruction(instruction)) return false; + break; + } + if (end > instruction.size()) throw std::runtime_error("truncated prepared instruction operands"); + for (auto index = first; index < end; ++index) visit(index); + return true; +} + +struct ScalarType { + std::uint32_t width; + bool boolean; +}; + +class Specialization { +public: + explicit Specialization(std::span words) { + if (words.size() < 5u || words[0] != spv::MagicNumber) throw std::runtime_error("invalid prepared SPIR-V header"); + header.assign(words.begin(), words.begin() + 5); + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + if (count == 0u || count > words.size() - cursor) throw std::runtime_error("truncated prepared SPIR-V instruction"); + instructions.emplace_back(words.begin() + cursor, words.begin() + cursor + count); + const auto& instruction = instructions.back(); + const auto op = Opcode(instruction); + bool hasResult = false; + bool hasType = false; + spv::HasResultAndType(op, &hasResult, &hasType); + if (hasResult && hasType) resultTypes.emplace(instruction.at(2), instruction.at(1)); + if (op == spv::OpTypeBool) types.emplace(instruction.at(1), ScalarType{1u, true}); + if (op == spv::OpTypeInt) types.emplace(instruction.at(1), ScalarType{instruction.at(2), false}); + if (op == spv::OpConstant && types.contains(instruction.at(1)) && types.at(instruction[1]).width <= 32u) values.emplace(instruction.at(2), instruction.at(3)); + if (op == spv::OpConstantTrue || op == spv::OpConstantFalse) values.emplace(instruction.at(2), op == spv::OpConstantTrue ? 1u : 0u); + cursor += count; + } + } + + std::vector Run() { + bool changed = true; + while (changed) { + changed = fold(); + changed |= prune(); + changed |= propagateCopies(); + } + removeDeadComputations(); + orderPhis(); + std::vector words = header; + bool inserted = false; + for (const auto& instruction : instructions) { + if (instruction.empty()) continue; + if (!inserted && Opcode(instruction) == spv::OpFunction) { + for (const auto& constant : constants) words.insert(words.end(), constant.begin(), constant.end()); + inserted = true; + } + if ((Opcode(instruction) == spv::OpName || Opcode(instruction) == spv::OpDecorate) && removed.contains(instruction.at(1))) continue; + words.insert(words.end(), instruction.begin(), instruction.end()); + } + return words; + } + +private: + std::optional value(std::uint32_t id) const { + const auto found = values.find(id); + return found != values.end() ? std::optional(found->second) : std::nullopt; + } + + std::optional evaluate(const Instruction& instruction) const { + const auto op = Opcode(instruction); + if (instruction.size() < 4u) return std::nullopt; + const auto type = types.find(instruction[1]); + if (type == types.end() || type->second.width > 32u) return std::nullopt; + const auto left = value(instruction[3]); + if (!left) return std::nullopt; + if (op == spv::OpCopyObject) return left; + if (op == spv::OpLogicalNot) return *left == 0u; + if (op == spv::OpNot) return ~*left; + if (instruction.size() < 5u) return std::nullopt; + const auto right = value(instruction[4]); + if (!right) return std::nullopt; + switch (op) { + case spv::OpIAdd: return *left + *right; + case spv::OpISub: return *left - *right; + case spv::OpIMul: return *left * *right; + case spv::OpUDiv: return *right != 0u ? std::optional(*left / *right) : std::nullopt; + case spv::OpUMod: return *right != 0u ? std::optional(*left % *right) : std::nullopt; + case spv::OpBitwiseAnd: return *left & *right; + case spv::OpBitwiseOr: return *left | *right; + case spv::OpBitwiseXor: return *left ^ *right; + case spv::OpShiftRightLogical: return *right < type->second.width ? std::optional(*left >> *right) : std::nullopt; + case spv::OpShiftLeftLogical: return *right < type->second.width ? std::optional(*left << *right) : std::nullopt; + case spv::OpIEqual: return *left == *right; + case spv::OpINotEqual: return *left != *right; + case spv::OpULessThan: return *left < *right; + case spv::OpULessThanEqual: return *left <= *right; + case spv::OpUGreaterThan: return *left > *right; + case spv::OpUGreaterThanEqual: return *left >= *right; + case spv::OpLogicalEqual: return (*left != 0u) == (*right != 0u); + case spv::OpLogicalNotEqual: return (*left != 0u) != (*right != 0u); + case spv::OpLogicalAnd: return *left != 0u && *right != 0u; + case spv::OpLogicalOr: return *left != 0u || *right != 0u; + case spv::OpBitFieldUExtract: { + if (instruction.size() != 6u) throw std::runtime_error("invalid prepared bitfield instruction"); + const auto count = value(instruction[5]); + if (!count || *right > 32u || *count > 32u - *right) return std::nullopt; + if (*count == 0u) return 0u; + const auto mask = *count == 32u ? UINT32_MAX : (1u << *count) - 1u; + return (*left >> *right) & mask; + } + default: return std::nullopt; + } + } + + bool fold() { + bool changed = false; + bool function = false; + std::map> extracts; + for (auto& instruction : instructions) { + if (instruction.empty()) continue; + auto op = Opcode(instruction); + if (op == spv::OpFunction) function = true; + if (op == spv::OpFunctionEnd) function = false; + if (!function) continue; + if (op == spv::OpSelect && instruction.size() == 6u) { + if (const auto condition = value(instruction[3])) { + instruction = Make(spv::OpCopyObject, {instruction[1], instruction[2], instruction[*condition != 0u ? 4u : 5u]}); + changed = true; + } + } + if (op == spv::OpVectorExtractDynamic && instruction.size() == 5u) { + if (const auto index = value(instruction[4])) { + instruction = Make(spv::OpCompositeExtract, {instruction[1], instruction[2], instruction[3], *index}); + changed = true; + } + } + op = Opcode(instruction); + if (op == spv::OpCompositeExtract && instruction.size() == 5u) extracts.emplace(instruction[2], std::pair{instruction[3], instruction[4]}); + if (op == spv::OpCompositeConstruct && instruction.size() == 7u) { + const auto first = extracts.find(instruction[3]); + bool shuffle = first != extracts.end(); + for (std::size_t index = 4; shuffle && index < instruction.size(); ++index) { + const auto found = extracts.find(instruction[index]); + shuffle = found != extracts.end() && found->second.first == first->second.first; + } + if (shuffle) { + const auto source = first->second.first; + bool identity = resultTypes.contains(source) && resultTypes.at(source) == instruction[1]; + Instruction replacement = Make(spv::OpVectorShuffle, {instruction[1], instruction[2], source, source}); + for (std::size_t index = 3; index < instruction.size(); ++index) { + const auto channel = extracts.at(instruction[index]).second; + replacement.push_back(channel); + identity &= channel == index - 3u; + } + replacement[0] = (static_cast(replacement.size()) << 16u) | spv::OpVectorShuffle; + instruction = identity ? Make(spv::OpCopyObject, {instruction[1], instruction[2], source}) : std::move(replacement); + changed = true; + } + } + const auto result = evaluate(instruction); + if (!result) continue; + const auto type = types.at(instruction[1]); + const auto bits = type.width == 32u ? *result : *result & ((1u << type.width) - 1u); + values[instruction[2]] = bits; + constants.push_back(type.boolean ? Make(bits != 0u ? spv::OpConstantTrue : spv::OpConstantFalse, {instruction[1], instruction[2]}) : Make(spv::OpConstant, {instruction[1], instruction[2], bits})); + instruction.clear(); + changed = true; + } + return changed; + } + + bool prune() { + std::map> blocks; + std::vector entries; + std::uint32_t block = 0; + bool entry = false; + for (std::size_t index = 0; index < instructions.size(); ++index) { + const auto& instruction = instructions[index]; + if (instruction.empty()) continue; + const auto op = Opcode(instruction); + if (op == spv::OpFunction) entry = true; + if (op == spv::OpFunctionEnd) block = 0; + if (op == spv::OpLabel) { + block = instruction.at(1); + if (entry) entries.push_back(block); + entry = false; + } + if (block != 0u) blocks[block].push_back(index); + } + bool changed = false; + const auto linearSelection = [&](std::uint32_t target, std::uint32_t merge) { + std::set visited; + while (target != merge) { + if (!visited.insert(target).second) return false; + const auto& indices = blocks.at(target); + for (const auto index : indices) { + const auto op = Opcode(instructions[index]); + if (op == spv::OpSelectionMerge || op == spv::OpLoopMerge) return false; + } + const auto& terminal = instructions[indices.back()]; + const auto op = Opcode(terminal); + if (op == spv::OpBranchConditional || op == spv::OpSwitch) return false; + if (op != spv::OpBranch) return true; + target = terminal.at(1); + } + return true; + }; + std::map> edges; + for (const auto& [label, indices] : blocks) { + auto& terminal = instructions[indices.back()]; + auto op = Opcode(terminal); + std::uint32_t target = 0; + bool loop = false; + for (const auto index : indices) loop |= Opcode(instructions[index]) == spv::OpLoopMerge; + if (!loop && op == spv::OpBranchConditional) { + if (const auto condition = value(terminal.at(1))) target = terminal.at(*condition != 0u ? 2u : 3u); + } else if (!loop && op == spv::OpSwitch) { + if (const auto selector = value(terminal.at(1))) { + target = terminal.at(2); + if ((terminal.size() - 3u) % 2u != 0u) throw std::runtime_error("invalid prepared 32-bit switch"); + for (std::size_t index = 3; index < terminal.size(); index += 2u) if (terminal[index] == *selector) target = terminal[index + 1u]; + } + } + if (target != 0u) { + bool retainMerge = false; + for (const auto index : indices) { + auto& instruction = instructions[index]; + if (Opcode(instruction) != spv::OpSelectionMerge) continue; + retainMerge = !linearSelection(target, instruction.at(1)); + if (!retainMerge) instruction.clear(); + } + const auto replacement = !retainMerge ? Make(spv::OpBranch, {target}) : op == spv::OpSwitch ? + Make(spv::OpSwitch, {terminal.at(1), target}) : Make(spv::OpBranchConditional, {terminal.at(1), target, target}); + if (terminal != replacement) { + terminal = replacement; + changed = true; + } + op = Opcode(terminal); + } + auto& successors = edges[label]; + if (op == spv::OpBranch) successors.push_back(terminal.at(1)); + else if (op == spv::OpBranchConditional) successors = {terminal.at(2), terminal.at(3)}; + else if (op == spv::OpSwitch) { + successors.push_back(terminal.at(2)); + const auto selectorType = resultTypes.find(terminal.at(1)); + const auto stride = selectorType != resultTypes.end() && types.contains(selectorType->second) && types.at(selectorType->second).width == 64u ? 3u : 2u; + for (std::size_t index = 3; index + stride <= terminal.size(); index += stride) successors.push_back(terminal[index + stride - 1u]); + } + } + std::set live; + auto pending = entries; + while (!pending.empty()) { + const auto label = pending.back(); + pending.pop_back(); + if (!blocks.contains(label)) throw std::runtime_error("prepared branch references a missing block"); + if (!live.insert(label).second) continue; + const auto& successors = edges.at(label); + pending.insert(pending.end(), successors.begin(), successors.end()); + } + for (const auto& [label, indices] : blocks) { + if (!live.contains(label)) { + if (indices.size() == 2u && Opcode(instructions[indices.back()]) == spv::OpUnreachable) continue; + for (std::size_t index = 1; index < indices.size(); ++index) { + auto& instruction = instructions[indices[index]]; + if (const auto result = Result(instruction)) removed.insert(result); + instruction.clear(); + } + instructions[indices.back()] = Make(spv::OpUnreachable, {}); + changed = true; + continue; + } + for (const auto index : indices) { + auto& instruction = instructions[index]; + if (instruction.empty() || Opcode(instruction) != spv::OpPhi) continue; + Instruction phi(instruction.begin(), instruction.begin() + 3); + for (std::size_t operand = 3; operand + 1u < instruction.size(); operand += 2u) { + const auto parent = instruction[operand + 1u]; + if (live.contains(parent) && std::ranges::find(edges.at(parent), label) != edges.at(parent).end()) phi.insert(phi.end(), {instruction[operand], parent}); + } + if (phi.size() == 3u) throw std::runtime_error("reachable prepared phi has no predecessor"); + if (phi.size() == instruction.size() && phi.size() != 5u) continue; + phi[0] = (static_cast(phi.size()) << 16u) | spv::OpPhi; + instruction = phi.size() == 5u ? Make(spv::OpCopyObject, {phi[1], phi[2], phi[3]}) : std::move(phi); + changed = true; + } + } + return changed; + } + + void orderPhis() { + std::vector ordered; + std::vector copies; + bool prefix = false; + for (auto& instruction : instructions) { + if (instruction.empty()) continue; + const auto op = Opcode(instruction); + if (op == spv::OpLabel) prefix = true; + else if (prefix && op == spv::OpCopyObject) { + copies.push_back(std::move(instruction)); + continue; + } else if (prefix && op != spv::OpPhi && op != spv::OpLine && op != spv::OpNoLine) { + for (auto& copy : copies) ordered.push_back(std::move(copy)); + copies.clear(); + prefix = false; + } + ordered.push_back(std::move(instruction)); + } + if (!copies.empty()) throw std::runtime_error("unterminated prepared phi block"); + instructions = std::move(ordered); + } + + bool propagateCopies() { + std::map copies; + for (const auto& instruction : instructions) { + if (Opcode(instruction) != spv::OpCopyObject) continue; + if (instruction.size() != 4u) throw std::runtime_error("invalid prepared copy instruction"); + copies.emplace(instruction[2], instruction[3]); + } + if (copies.empty()) return false; + for (const auto& instruction : instructions) { + if (instruction.empty()) continue; + const auto op = Opcode(instruction); + if (op == spv::OpName || op == spv::OpMemberName) continue; + if (VisitInputs(instruction, [](std::size_t) {})) continue; + bool hasResult = false; + bool hasType = false; + spv::HasResultAndType(op, &hasResult, &hasType); + const auto resultIndex = hasResult ? (hasType ? 2u : 1u) : 0u; + for (std::size_t index = 1; index < instruction.size(); ++index) if (index != resultIndex) copies.erase(instruction[index]); + } + if (copies.empty()) return false; + const auto resolve = [&](std::uint32_t id) { + std::size_t count = 0; + while (copies.contains(id)) { + if (++count > copies.size()) throw std::runtime_error("cyclic prepared copy chain"); + id = copies.at(id); + } + return id; + }; + bool function = false; + for (auto& instruction : instructions) { + if (instruction.empty()) continue; + const auto op = Opcode(instruction); + if (op == spv::OpFunction) function = true; + if (op == spv::OpFunctionEnd) function = false; + if (!function) continue; + if (op == spv::OpCopyObject && copies.contains(instruction[2])) { + removed.insert(instruction[2]); + instruction.clear(); + continue; + } + VisitInputs(instruction, [&](std::size_t index) { instruction[index] = resolve(instruction[index]); }); + } + return true; + } + + void removeDeadComputations() { + std::map definitions; + std::vector pending; + bool function = false; + for (std::size_t index = 0; index < instructions.size(); ++index) { + const auto& instruction = instructions[index]; + if (instruction.empty()) continue; + const auto op = Opcode(instruction); + if (op == spv::OpFunction) function = true; + if (op == spv::OpFunctionEnd) function = false; + if (!function) { + if (op == spv::OpDecorateId || op == spv::OpGroupDecorate || op == spv::OpGroupMemberDecorate) { + for (std::size_t operand = 1; operand < instruction.size(); ++operand) pending.push_back(instruction[operand]); + } + continue; + } + if (PureInstruction(instruction)) { + definitions.emplace(Result(instruction), index); + continue; + } + if (VisitInputs(instruction, [&](std::size_t operand) { pending.push_back(instruction[operand]); })) continue; + bool hasResult = false; + bool hasType = false; + spv::HasResultAndType(op, &hasResult, &hasType); + const auto resultIndex = hasResult ? (hasType ? 2u : 1u) : 0u; + for (std::size_t operand = 1; operand < instruction.size(); ++operand) if (operand != resultIndex) pending.push_back(instruction[operand]); + } + std::set live; + while (!pending.empty()) { + const auto id = pending.back(); + pending.pop_back(); + const auto found = definitions.find(id); + if (found == definitions.end() || !live.insert(id).second) continue; + const auto& instruction = instructions[found->second]; + if (!VisitInputs(instruction, [&](std::size_t operand) { pending.push_back(instruction[operand]); })) throw std::runtime_error("missing prepared pure instruction operands"); + } + for (const auto& [id, index] : definitions) { + if (live.contains(id)) continue; + removed.insert(id); + instructions[index].clear(); + } + } + + std::vector header; + std::vector instructions; + std::vector constants; + std::map types; + std::map values; + std::map resultTypes; + std::set removed; +}; + +} + +std::vector SpecializeSpirv(std::span words) { + return Specialization(words).Run(); +} + +} diff --git a/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp b/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp index ff4d76e02..3085542e0 100644 --- a/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp +++ b/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp @@ -5,9 +5,10 @@ #include "RdnaDecoder/RdnaProgram.hpp" #include "IntermediateRepresentation/IrBuilder.hpp" #include "IntermediateRepresentation/IrProgram.hpp" -#include "Translation/EmbeddedVertexFetch.hpp" #include "Optimization/ShaderStageInputInfo.hpp" +#include "Translation/EmbeddedVertexFetch.hpp" #include +#include namespace ShaderRecompiler { @@ -31,6 +32,7 @@ struct TranslateOptions { std::uint32_t scratchDwords = 0; std::uint64_t shaderHash = 0; bool fragmentShaderBarycentricEnabled = false; + std::optional floatMode; ShaderStageInputInfo inputInfo; const EmbeddedFetchPlan* embeddedFetch = nullptr; }; diff --git a/core/shader/recompiler/Translation/include/Translation/ShaderInputInfoBuilder.hpp b/core/shader/recompiler/Translation/include/Translation/ShaderInputInfoBuilder.hpp index 1f923d34d..8fe4a0187 100644 --- a/core/shader/recompiler/Translation/include/Translation/ShaderInputInfoBuilder.hpp +++ b/core/shader/recompiler/Translation/include/Translation/ShaderInputInfoBuilder.hpp @@ -9,7 +9,7 @@ namespace ShaderRecompiler { // hostSubgroupSize is the device subgroup width; wave64 workgroups on 32-wide subgroups run each // invocation as two guest lanes so a guest wave stays one host subgroup. -ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const GuestContext& context, std::uint32_t hostSubgroupSize, const MeshConfiguration* mesh = nullptr); +ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const GuestContext& context, std::uint32_t hostSubgroupSize, const MeshConfiguration* mesh = nullptr, const TessellationConfiguration* tessellation = nullptr); } diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index d87a7e86e..bf5a58d69 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -4,6 +4,7 @@ #include "Translation/InstructionTranslator.hpp" #include #include +#include #include namespace ShaderRecompiler { @@ -12,9 +13,10 @@ class TranslationContext { public: TranslationContext(IrProgram& program, IrBlock& block, std::uint32_t vectorLimit); + void TranslateEmbeddedFetch(const RdnaInstruction& instruction, std::uint32_t attribute, std::uint32_t components); void TranslateInstruction(const RdnaInstruction& instruction); void SetPixelInput(const ShaderPixelInputInfo* info, bool barycentricEnabled) { pixelInput = info; fragmentShaderBarycentricEnabled = barycentricEnabled; } - void TranslateEmbeddedFetch(const RdnaInstruction& instruction, std::uint32_t attribute, std::uint32_t componentCount, const ShaderBufferResource& resource); + void SetFloatMode(const std::optional& mode) { floatMode = mode; ieeeMode = mode.has_value() && mode->ieeeMode; } void AddBranchCondition(const BasicBlock& source, BlockInfo& info); void TranslateCodeTableLoad(const RdnaInstruction& instruction, const ControlFlowGraph::CodeTableLoad& table); @@ -38,6 +40,7 @@ private: RdnaOperand scalarDestinationOperand(const RdnaOperand& operand, std::uint32_t offset); RdnaOperand plainOperand(const RdnaOperand& operand); std::array ballotMask(IrU1 value); + IrU32 hostExecWord(std::uint32_t half); IrU32 readRawU32(const RdnaOperand& operand); IrU32 readScalarCode(std::uint32_t code); IrU32 applyBitSourceModifiers(const RdnaOperand& operand, IrU32 value); @@ -46,12 +49,21 @@ private: void writeRawU32(const RdnaOperand& operand, IrU32 value); IrF32 applyF32ResultModifiers(const RdnaOperand& operand, IrF32 value); IrF32 applyF16ResultModifiers(const RdnaOperand& operand, IrF32 value); + bool outputModifierApplies(std::uint32_t denormalShift) const; + bool dx10Clamp() const { return !floatMode.has_value() || floatMode->dx10Clamp; } + void rejectHalfOrDoubleOutputModifier(const RdnaOperand& operand) const; IrU32 clampF16Bits(const RdnaOperand& operand, IrU32 bits); void writeOperand(const RdnaOperand& operand, IrValue* value); IrU32 packHalf2x16(IrF32 low, IrF32 high); void write16Bits(const RdnaOperand& operand, IrU32 value); void writeF16(const RdnaOperand& operand, IrF32 value); IrU32 readU32(const RdnaOperand& operand); + IrU32 flushF32Denormal(IrU32 bits); + IrF32 flushTinyProduct(IrValue* lhs, IrValue* rhs, IrValue* product, IrValue* addend = nullptr); + IrU32 quietNan32(IrU32 bits); + IrU32 quietNan16(IrU32 bits); + IrValue* nanResultF32(std::initializer_list sources, IrValue* result, IrValue* invalidProduct = nullptr); + IrValue& invalidProductF32(IrValue* lhs, IrValue* rhs); std::array readU32Pair(const RdnaOperand& operand); IrU64 readU64(const RdnaOperand& operand); std::array readF64Bits(const RdnaOperand& operand); @@ -102,6 +114,7 @@ private: bool imageGetResinfo(const RdnaInstruction& inst); bool imageGetLod(const RdnaInstruction& inst); bool imageLoad(const RdnaInstruction& inst); + bool imageBy(const RdnaInstruction& inst); bool imageMsaaLoad(const RdnaInstruction& inst); bool imageStore(const RdnaInstruction& inst); bool imageSample(const RdnaInstruction& inst); @@ -114,8 +127,10 @@ private: bool dsRead2(const RdnaInstruction& inst); bool dsWrite(const RdnaInstruction& inst); bool dsWrite2(const RdnaInstruction& inst); + bool dsSrc2(const RdnaInstruction& inst, IrOpcode opcode); bool dsAtomic2(const RdnaInstruction& inst, IrOpcode opcode, bool returnsValue); bool dsAtomic64(const RdnaInstruction& inst, IrOpcode opcode, bool returnsValue); + bool dsCondxchg32(const RdnaInstruction& inst); bool dsAppendConsume(const RdnaInstruction& inst, IrOpcode opcode); bool dsAddtid(const RdnaInstruction& inst, bool write); bool globalAddtid(const RdnaInstruction& inst, bool write); @@ -165,6 +180,7 @@ private: bool floatUnary(const RdnaInstruction& inst, IrOpcode opcode); bool floatBinary(const RdnaInstruction& inst, IrOpcode opcode, bool reverse); bool floatTernary(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool mix); + bool ieeeMinMaxF32(const RdnaInstruction& inst, IrOpcode opcode); bool vDivScaleF32(const RdnaInstruction& inst); bool vDivFmasF32(const RdnaInstruction& inst); bool vDivFixupF32(const RdnaInstruction& inst); @@ -274,6 +290,7 @@ private: void sTtracedata(); void sInstPrefetch(); void sGetpcB64(const RdnaInstruction& inst); + void sSwappcB64(const RdnaInstruction& inst); void sCselectB32(const RdnaInstruction& inst); void scalarSelect64(const RdnaInstruction& inst, const RdnaOperand& falseSource); void movB32(const RdnaInstruction& inst, bool applyFloatModifiers); @@ -292,19 +309,27 @@ private: void vInterpP1F32(const RdnaInstruction& inst); void vInterpP2F32(const RdnaInstruction& inst); void vInterpMovF32(const RdnaInstruction& inst); + std::uint32_t interpolationModeF16(const RdnaInstruction& inst) const; + IrF32 interpolationParameterF16(const RdnaInstruction& inst, std::uint32_t mode); + void vInterpP1F16(const RdnaInstruction& inst); + void vInterpP2F16(const RdnaInstruction& inst); void eXP(const RdnaInstruction& inst); bool emitScalar(const RdnaInstruction& inst); bool emitVector(const RdnaInstruction& inst); + std::uint32_t f32DenormalFlushFor(const RdnaInstruction& inst) const; bool emitInterpolation(const RdnaInstruction& inst); bool emitMemory(const RdnaInstruction& inst); IrProgram& program; const ShaderPixelInputInfo* pixelInput = nullptr; bool fragmentShaderBarycentricEnabled = false; + std::optional floatMode; + bool ieeeMode = false; IrBuilder ir; IrBlock& block; IrU1 instructionBranchCondition; RdnaOpcode currentOpcode = RdnaOpcode::Unknown; + std::uint32_t f32DenormalFlush = 0u; std::uint32_t currentProgramCounter = 0; std::uint32_t currentVectorLimit = 1; }; diff --git a/core/shader/recompiler/Translation/src/AttributeInstructions.cpp b/core/shader/recompiler/Translation/src/AttributeInstructions.cpp index f9d247a0e..aebe411e0 100644 --- a/core/shader/recompiler/Translation/src/AttributeInstructions.cpp +++ b/core/shader/recompiler/Translation/src/AttributeInstructions.cpp @@ -33,56 +33,6 @@ ExportTargetKind exportTargetKindFromTarget(std::uint32_t target, std::uint32_t& return ExportTargetKind::Unknown; } -std::uint32_t getDstSel(std::uint32_t dstSelXYZW, std::uint32_t component) { - return (dstSelXYZW >> (component * 3u)) & 0x7u; -} - -bool isIntegerBufferFormat(IrBufferFormat format) { - switch (format) { - case IrBufferFormat::Format8UInt: - case IrBufferFormat::Format8SInt: - case IrBufferFormat::Format16UInt: - case IrBufferFormat::Format16SInt: - case IrBufferFormat::Format8_8UInt: - case IrBufferFormat::Format8_8SInt: - case IrBufferFormat::Format32UInt: - case IrBufferFormat::Format32SInt: - case IrBufferFormat::Format16_16UInt: - case IrBufferFormat::Format16_16SInt: - case IrBufferFormat::Format11_11_10UInt: - case IrBufferFormat::Format11_11_10SInt: - case IrBufferFormat::Format10_11_11UInt: - case IrBufferFormat::Format10_11_11SInt: - case IrBufferFormat::Format2_10_10_10UInt: - case IrBufferFormat::Format2_10_10_10SInt: - case IrBufferFormat::Format10_10_10_2UInt: - case IrBufferFormat::Format10_10_10_2SInt: - case IrBufferFormat::Format8_8_8_8UInt: - case IrBufferFormat::Format8_8_8_8SInt: - case IrBufferFormat::Format32_32UInt: - case IrBufferFormat::Format32_32SInt: - case IrBufferFormat::Format16_16_16_16UInt: - case IrBufferFormat::Format16_16_16_16SInt: - case IrBufferFormat::Format32_32_32UInt: - case IrBufferFormat::Format32_32_32SInt: - case IrBufferFormat::Format32_32_32_32UInt: - case IrBufferFormat::Format32_32_32_32SInt: - return true; - default: - return false; - } -} - -std::uint32_t formattedConstantBits(IrBufferFormat format, std::uint32_t selector) { - if (selector == 0u) { - return 0u; - } - if (selector == 1u) { - return isIntegerBufferFormat(format) ? 1u : std::bit_cast(1.0f); - } - throw std::runtime_error("reserved buffer destination selector"); -} - } void TranslateAttributeInstruction(IrBuilder& builder, const RdnaInstruction& instruction, const TranslateOptions& options) { @@ -143,6 +93,40 @@ void TranslationContext::vInterpMovF32(const RdnaInstruction& inst) { writeOperand(inst.destination, &value); } +IrF32 TranslationContext::interpolationParameterF16(const RdnaInstruction& inst, std::uint32_t mode) { + if (!fragmentShaderBarycentricEnabled) { + throw std::runtime_error("16-bit interpolation of pixel input " + std::to_string(inst.source1.value) + " requires fragmentShaderBarycentric"); + } + if (pixelInput == nullptr || !pixelInput->InputIsFp16(inst.source1.value)) { + throw std::runtime_error("pixel input " + std::to_string(inst.source1.value) + " is read with 16-bit interpolation without FP16_INTERP_MODE"); + } + return IrF32(ir.Emit(IrOpcode::GetInterpolationParameterF16, IrType::F32, {&ir.Constant(inst.source1.value), &ir.Constant(inst.source2.value), &ir.Constant(mode), &ir.Constant(inst.source1.opSel ? 1u : 0u)})); +} + +std::uint32_t TranslationContext::interpolationModeF16(const RdnaInstruction& inst) const { + if (!floatMode.has_value()) return 0u; + const auto mode = floatMode->floatMode; + const auto denorm32 = (mode >> 4u) & 3u; + const auto denorm16 = (mode >> 6u) & 3u; + if ((mode & 0xfu) != 0u || (denorm32 != 0u && denorm32 != 3u) || (denorm16 != 0u && denorm16 != 3u)) { + throw std::runtime_error("16-bit interpolation at pc " + std::to_string(inst.programCounter) + " in FLOAT_MODE " + std::to_string(mode) + " is not measured"); + } + return (floatMode->ieeeMode ? InterpolationQuiet : 0u) | (denorm32 == 0u ? InterpolationFlush32 : 0u) | (denorm16 == 0u ? InterpolationFlush16 : 0u); +} + +void TranslationContext::vInterpP1F16(const RdnaInstruction& inst) { + const auto mode = interpolationModeF16(inst); + const IrF32 delta = interpolationParameterF16(inst, 0u); + const IrF32 origin = inst.op == RdnaOpcode::VInterpP1lvF16 ? readF16AsF32(inst.source3) : interpolationParameterF16(inst, 2u); + writeOperand(inst.destination, &ir.Emit(IrOpcode::FPInterpolateF32, IrType::F32, {&delta.Value(), readOperand(inst.source0, IrType::F32), &origin.Value(), &ir.Constant(mode)})); +} + +void TranslationContext::vInterpP2F16(const RdnaInstruction& inst) { + const auto mode = interpolationModeF16(inst); + const IrF32 delta = interpolationParameterF16(inst, 1u); + writeF16(inst.destination, IrF32(ir.Emit(IrOpcode::FPInterpolateF16, IrType::F32, {&delta.Value(), readOperand(inst.source0, IrType::F32), readOperand(inst.source3, IrType::F32), &ir.Constant(mode)}))); +} + void TranslationContext::eXP(const RdnaInstruction& inst) { std::uint32_t index = 0u; if (exportTargetKindFromTarget(inst.exportTarget, index) == ExportTargetKind::Unknown) { @@ -159,7 +143,8 @@ void TranslationContext::eXP(const RdnaInstruction& inst) { } bool TranslationContext::emitInterpolation(const RdnaInstruction& inst) { - if ((inst.op == RdnaOpcode::VInterpP1F32 || inst.op == RdnaOpcode::VInterpP2F32) && pixelInput != nullptr && pixelInput->InputIsCustom(inst.source1.value)) { + const bool interpolates = inst.op == RdnaOpcode::VInterpP1F32 || inst.op == RdnaOpcode::VInterpP2F32 || inst.op == RdnaOpcode::VInterpP1llF16 || inst.op == RdnaOpcode::VInterpP1lvF16 || inst.op == RdnaOpcode::VInterpP2F16; + if (interpolates && pixelInput != nullptr && pixelInput->InputIsCustom(inst.source1.value)) { throw std::runtime_error("pixel input " + std::to_string(inst.source1.value) + " passes its vertices through unchanged but is read with v_interp_p1/p2"); } switch (inst.op) { @@ -172,28 +157,24 @@ bool TranslationContext::emitInterpolation(const RdnaInstruction& inst) { case RdnaOpcode::VInterpMovF32: vInterpMovF32(inst); return true; + case RdnaOpcode::VInterpP1llF16: + case RdnaOpcode::VInterpP1lvF16: + vInterpP1F16(inst); + return true; + case RdnaOpcode::VInterpP2F16: + vInterpP2F16(inst); + return true; default: return false; } } -void TranslationContext::TranslateEmbeddedFetch(const RdnaInstruction& instruction, std::uint32_t attribute, std::uint32_t componentCount, const ShaderBufferResource& resource) { - const auto format = static_cast((resource.fields[3] >> 12u) & 0x7Fu); - const std::uint32_t dstSel = resource.fields[3] & 0xFFFu; - for (std::uint32_t component = 0u; component < componentCount; ++component) { - const std::uint32_t selector = instruction.formatted && !instruction.typed ? getDstSel(dstSel, component) : component + 4u; - IrValue* value = nullptr; - if (selector <= 1u) { - value = &ir.Constant(formattedConstantBits(format, selector)); - } else if (selector >= 4u && selector <= 7u) { - const std::uint32_t memoryComponent = selector - 4u; - value = &ir.Emit(IrOpcode::GetAttribute, IrType::U32, {&ir.Constant(attribute), &ir.Constant(memoryComponent)}); - std::uint8_t& required = program.Info().vertexFetchComponents[attribute]; - required = static_cast(std::max(required, memoryComponent + 1u)); - } else { - throw std::runtime_error("invalid embedded fetch component selector"); - } - writeOperand(offsetOperand(instruction.destination, component), value); +void TranslationContext::TranslateEmbeddedFetch(const RdnaInstruction& instruction, std::uint32_t attribute, std::uint32_t components) { + if (!instruction.formatted || instruction.typed || components == 0u || components > 4u) throw std::runtime_error("invalid prepared vertex fetch"); + for (std::uint32_t component = 0; component < components; ++component) { + auto& value = ir.Emit(IrOpcode::GetAttribute, IrType::U32, {&ir.Constant(attribute), &ir.Constant(component)}); + value.SetFlags(1u); + writeOperand(offsetOperand(instruction.destination, component), &value); } } diff --git a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp index 2a47caf74..798eaf34c 100644 --- a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp @@ -31,11 +31,13 @@ void TranslationContext::sSaveexec(const RdnaInstruction& inst, IrOpcode operati else if (operation == IrOpcode::LogicalOr) operation = IrOpcode::BitwiseOr32; else if (operation == IrOpcode::LogicalXor) operation = IrOpcode::BitwiseXor32; const bool writeFirst = SaveexecWritesDestinationFirst() && !writeResult; + const bool execSource = sourceAt(inst, 0u).kind == RdnaOperandKind::ExecLo; if (write64) { const std::array oldExec{IrU32(ir.GetExecLo()), IrU32(ir.GetExecHi())}; - if (writeFirst) writeU32Pair(inst.destination, oldExec); - const std::array source = readU32Pair(sourceAt(inst, 0u)); - if (!writeFirst && !writeResult) writeU32Pair(inst.destination, oldExec); + const std::array savedExec{hostExecWord(0u), hostExecWord(1u)}; + if (writeFirst) writeU32Pair(inst.destination, savedExec); + const std::array source = execSource ? oldExec : readU32Pair(sourceAt(inst, 0u)); + if (!writeFirst && !writeResult) writeU32Pair(inst.destination, savedExec); IrValue& lowExecOperand = negateExec ? ir.BitwiseNot(oldExec[0].Value()) : oldExec[0].Value(); IrValue& lowSourceOperand = negateSource ? ir.BitwiseNot(source[0].Value()) : source[0].Value(); IrValue& highExecOperand = negateExec ? ir.BitwiseNot(oldExec[1].Value()) : oldExec[1].Value(); @@ -46,22 +48,23 @@ void TranslationContext::sSaveexec(const RdnaInstruction& inst, IrOpcode operati const IrU32 newExecHi(negateResult ? ir.BitwiseNot(highCombined) : highCombined); ir.SetExecLo(newExecLo.Value()); ir.SetExecHi(newExecHi.Value()); - if (writeResult) writeU32Pair(inst.destination, {newExecLo, newExecHi}); + if (writeResult) writeU32Pair(inst.destination, {hostExecWord(0u), hostExecWord(1u)}); const IrU1 nonZero(ir.LogicalOr(ir.INotEqual(newExecLo.Value(), ir.Constant(0u)), ir.INotEqual(newExecHi.Value(), ir.Constant(0u)))); ir.SetScc(nonZero.Value()); ir.SetExec(threadBit({newExecLo, newExecHi}).Value()); return; } const IrU32 oldExec(ir.GetExecLo()); - if (writeFirst) writeRawU32(inst.destination, oldExec); - const IrU32 source = readU32(sourceAt(inst, 0u)); - if (!writeFirst && !writeResult) writeRawU32(inst.destination, oldExec); + const IrU32 savedExec = hostExecWord(0u); + if (writeFirst) writeRawU32(inst.destination, savedExec); + const IrU32 source = execSource ? oldExec : readU32(sourceAt(inst, 0u)); + if (!writeFirst && !writeResult) writeRawU32(inst.destination, savedExec); IrValue& execOperand = negateExec ? ir.BitwiseNot(oldExec.Value()) : oldExec.Value(); IrValue& sourceOperand = negateSource ? ir.BitwiseNot(source.Value()) : source.Value(); IrValue& combined = ir.Emit(operation, IrType::U32, {&execOperand, &sourceOperand}); const IrU32 newExec(negateResult ? ir.BitwiseNot(combined) : combined); ir.SetExecLo(newExec.Value()); - if (writeResult) writeRawU32(inst.destination, newExec); + if (writeResult) writeRawU32(inst.destination, hostExecWord(0u)); const IrU1 nonZero(ir.INotEqual(newExec.Value(), ir.Constant(0u))); ir.SetScc(nonZero.Value()); ir.SetExec(threadBit({newExec, IrU32(ir.GetExecHi())}).Value()); @@ -219,6 +222,12 @@ void TranslationContext::sGetpcB64(const RdnaInstruction& inst) { writeU32Pair(inst.destination, extractU64(pc)); } +void TranslationContext::sSwappcB64(const RdnaInstruction& inst) { + IrValue& base = ir.Emit(IrOpcode::GetShaderBase, IrType::U64, {}); + const IrU64 link(ir.Emit(IrOpcode::IAdd64, IrType::U64, {&base, &ir.ConstantU64(static_cast(currentProgramCounter) + 4u)})); + writeU32Pair(inst.destination, extractU64(link)); +} + void TranslationContext::sCselectB32(const RdnaInstruction& inst) { const IrU32 trueValue = readU32(sourceAt(inst, 0u)); const IrU32 falseValue = readU32(sourceAt(inst, 1u)); diff --git a/core/shader/recompiler/Translation/src/ConvertInstructions.cpp b/core/shader/recompiler/Translation/src/ConvertInstructions.cpp index 91978c218..b0a367c0d 100644 --- a/core/shader/recompiler/Translation/src/ConvertInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ConvertInstructions.cpp @@ -163,8 +163,8 @@ void TranslationContext::vCvtPkrtzF16F32(const RdnaInstruction& inst) { } void TranslationContext::vCvtPknormF32(const RdnaInstruction& inst, bool signedValue) { - const IrU32 low = normF32(readU32(sourceAt(inst, 0u)), signedValue); - const IrU32 high = normF32(readU32(sourceAt(inst, 1u)), signedValue); + const IrU32 low = normF32(flushF32Denormal(readU32(sourceAt(inst, 0u))), signedValue); + const IrU32 high = normF32(flushF32Denormal(readU32(sourceAt(inst, 1u))), signedValue); const IrU32 result(ir.BitwiseOr(low.Value(), ir.ShiftLeftLogical(high.Value(), ir.Constant(16u)))); writeOperand(inst.destination, &result.Value()); } diff --git a/core/shader/recompiler/Translation/src/DispatchInstructions.cpp b/core/shader/recompiler/Translation/src/DispatchInstructions.cpp index e2d3e4322..e2c9f9f7b 100644 --- a/core/shader/recompiler/Translation/src/DispatchInstructions.cpp +++ b/core/shader/recompiler/Translation/src/DispatchInstructions.cpp @@ -24,6 +24,7 @@ void TranslationContext::TranslateInstruction(const RdnaInstruction& decoded) { if (instruction.op == RdnaOpcode::Unsupported) { throw std::runtime_error(instruction.unsupportedReason.empty() ? "unsupported decoded instruction at pc " + std::to_string(instruction.programCounter) : std::string(instruction.unsupportedReason)); } + f32DenormalFlush = f32DenormalFlushFor(instruction); bool translated = false; switch (instruction.family) { case RdnaInstructionFamily::SOP1: @@ -38,7 +39,7 @@ void TranslationContext::TranslateInstruction(const RdnaInstruction& decoded) { case RdnaInstructionFamily::VOP3: case RdnaInstructionFamily::VOP3P: case RdnaInstructionFamily::VOPC: - translated = emitVector(instruction); + translated = emitVector(instruction) || emitInterpolation(instruction); break; case RdnaInstructionFamily::SMEM: case RdnaInstructionFamily::MUBUF: diff --git a/core/shader/recompiler/Translation/src/Float64Instructions.cpp b/core/shader/recompiler/Translation/src/Float64Instructions.cpp index 0e031dc92..47e19cf03 100644 --- a/core/shader/recompiler/Translation/src/Float64Instructions.cpp +++ b/core/shader/recompiler/Translation/src/Float64Instructions.cpp @@ -26,6 +26,7 @@ bool TranslationContext::float64Operation(const RdnaInstruction& inst, IrOpcode writeF64Result(inst.destination, *result); return true; } + rejectHalfOrDoubleOutputModifier(inst.destination); RdnaOperand destination = inst.destination; destination.omod = 0u; writeOperand(destination, result); @@ -33,10 +34,12 @@ bool TranslationContext::float64Operation(const RdnaInstruction& inst, IrOpcode } void TranslationContext::writeF64Result(const RdnaOperand& operand, IrValue& value) { + rejectHalfOrDoubleOutputModifier(operand); RdnaOperand destination = operand; destination.omod = 0u; IrValue* result = &value; if (destination.clamp) { + if (!dx10Clamp()) throw std::runtime_error("clamp on an f64 result with DX10_CLAMP=0 is not implemented"); result = &ir.Emit(IrOpcode::FPSaturate64, IrType::U64, {result}); } writeOperand(destination, result); diff --git a/core/shader/recompiler/Translation/src/FloatInstructions.cpp b/core/shader/recompiler/Translation/src/FloatInstructions.cpp index 7893e151c..502e601a3 100644 --- a/core/shader/recompiler/Translation/src/FloatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/FloatInstructions.cpp @@ -3,7 +3,6 @@ #include #include #include -#include namespace ShaderRecompiler { @@ -12,6 +11,9 @@ void TranslateFloatInstruction(IrBuilder& builder, const RdnaInstruction& instru } bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool quietSnan) { + if (floatMode.has_value() && (floatMode->floatMode & 0xccu) != 0xc0u) { + throw std::runtime_error("packed f16 at pc " + std::to_string(inst.programCounter) + " in FLOAT_MODE " + std::to_string(floatMode->floatMode) + " (f16 denormals not kept or rounding not to nearest even) is not implemented"); + } const auto translateLane = [&](bool high) -> IrF32 { const IrF32 lhs = readF16LaneAsF32(sourceAt(inst, 0u), high, true); const IrF32 rhs = readF16LaneAsF32(sourceAt(inst, 1u), high, true); @@ -26,22 +28,55 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value()}))); }; IrU32 result = packHalf2x16(translateLane(false), translateLane(true)); - if (quietSnan) { - const auto quietSnanLane = [&](const RdnaOperand& operand, bool high) { - const IrU32 bits = readU16LaneAsU32(operand, high, false); - const IrU32 exponent(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7c00u))); - const IrU32 payload(ir.BitwiseAnd(bits.Value(), ir.Constant(0x01ffu))); - const IrU1 snan(ir.LogicalAnd(ir.IEqual(exponent.Value(), ir.Constant(0x7c00u)), ir.INotEqual(payload.Value(), ir.Constant(0u)))); - const IrU32 quiet(ir.BitwiseOr(bits.Value(), ir.Constant(0x0200u))); - return std::pair(snan, quiet); + if (!inst.destination.clamp || !dx10Clamp()) { + const auto laneBits = [&](const RdnaOperand& operand, bool high) { + const IrU32 source = readF16SourceBits(operand); + const bool selectHigh = high ? operand.opSelHi : operand.opSel; + IrU32 bits(ir.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&source.Value(), &ir.Constant(selectHigh ? 16u : 0u), &ir.Constant(16u)})); + if (operand.absolute) bits = IrU32(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu))); + if (high ? operand.negateHi : operand.negate) bits = IrU32(ir.BitwiseXor(bits.Value(), ir.Constant(0x8000u))); + return bits; }; - const auto overrideLane = [&](bool high) { - const auto [lhsSnan, lhsQuiet] = quietSnanLane(sourceAt(inst, 0u), high); - const auto [rhsSnan, rhsQuiet] = quietSnanLane(sourceAt(inst, 1u), high); - const IrU32 normal(high ? ir.ShiftRightLogical(result.Value(), ir.Constant(16u)) : ir.BitwiseAnd(result.Value(), ir.Constant(0xffffu))); - return IrU32(ir.Select(lhsSnan.Value(), lhsQuiet.Value(), ir.Select(rhsSnan.Value(), rhsQuiet.Value(), normal.Value()))); + const auto isNan = [&](const IrU32& bits) { + return IrU1(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u))); }; - result = packU16Lanes(overrideLane(false), overrideLane(true)); + const auto isSnan = [&](const IrU32& bits) { + return IrU1(ir.LogicalAnd(isNan(bits).Value(), ir.IEqual(ir.BitwiseAnd(bits.Value(), ir.Constant(0x0200u)), ir.Constant(0u)))); + }; + const auto quiet = [&](const IrU32& bits) { return ieeeMode ? IrU32(ir.BitwiseOr(bits.Value(), ir.Constant(0x0200u))) : bits; }; + const auto magnitude = [&](const IrU32& bits) { return IrU32(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu))); }; + const auto invalidProduct = [&](const IrU32& lhs, const IrU32& rhs) { + const auto infZero = [&](const IrU32& inf, const IrU32& zero) { + return IrU1(ir.LogicalAnd(ir.IEqual(magnitude(inf).Value(), ir.Constant(0x7c00u)), ir.IEqual(magnitude(zero).Value(), ir.Constant(0u)))); + }; + return IrU1(ir.LogicalOr(infZero(lhs, rhs).Value(), infZero(rhs, lhs).Value())); + }; + const auto nanLane = [&](bool high) { + const std::uint32_t count = accumulator ? 3u : inst.sourceCount; + std::array bits{}; + for (std::uint32_t index = 0u; index < count; ++index) { + bits[index] = laneBits(accumulator && index == 2u ? accumulatorOperand(inst) : sourceAt(inst, index), high); + } + IrU32 lane(high ? ir.ShiftRightLogical(result.Value(), ir.Constant(16u)) : ir.BitwiseAnd(result.Value(), ir.Constant(0xffffu))); + if (quietSnan) { + lane = IrU32(ir.Select(isNan(bits[0]).Value(), bits[1].Value(), lane.Value())); + lane = IrU32(ir.Select(isNan(bits[1]).Value(), bits[0].Value(), lane.Value())); + if (!ieeeMode) return lane; + for (std::uint32_t index = count; index-- > 0u;) { + lane = IrU32(ir.Select(isSnan(bits[index]).Value(), quiet(bits[index]).Value(), lane.Value())); + } + return lane; + } + lane = IrU32(ir.Select(isNan(lane).Value(), ir.Constant(0xfe00u), lane.Value())); + for (std::uint32_t index = count; index-- > 0u;) { + IrU1 selected = isNan(bits[index]); + if (index == 2u) selected = IrU1(ir.LogicalAnd(selected.Value(), ir.LogicalNot(invalidProduct(bits[0], bits[1]).Value()))); + lane = IrU32(ir.Select(selected.Value(), quiet(bits[index]).Value(), lane.Value())); + } + return lane; + }; + const auto clampedLane = [&](bool high) { return inst.destination.clamp ? clampF16Bits(inst.destination, nanLane(high)) : nanLane(high); }; + result = packU16Lanes(clampedLane(false), clampedLane(true)); } writeRawU32(inst.destination, result); return true; @@ -57,8 +92,8 @@ bool TranslationContext::float16Unary(const RdnaInstruction& inst, IrOpcode opco } const IrU32 half = readF16Bits(operand); const IrU1 nan(ir.UGreaterThan(ir.BitwiseAnd(half.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u))); - const IrU32 special(ir.Select(nan.Value(), ir.BitwiseOr(half.Value(), ir.Constant(0x0200u)), ir.Constant(0xfe00u))); - const IrU32 value(ir.Select(ir.LogicalOr(nan.Value(), invalid.Value()), inst.destination.clamp ? ir.Constant(0u) : special.Value(), bits.Value())); + const IrU32 special(ir.Select(nan.Value(), quietNan16(half).Value(), ir.Constant(0xfe00u))); + const IrU32 value(ir.Select(ir.LogicalOr(nan.Value(), invalid.Value()), inst.destination.clamp && dx10Clamp() ? ir.Constant(0u) : special.Value(), bits.Value())); write16Bits(inst.destination, value); return true; } @@ -159,8 +194,8 @@ bool TranslationContext::vDivFixupF16(const RdnaInstruction& inst) { const IrU1 bothZero(ir.LogicalAnd(isZero(1u).Value(), isZero(2u).Value())); const IrU1 bothInf(ir.LogicalAnd(isInf(1u).Value(), isInf(2u).Value())); result = IrU32(ir.Select(ir.LogicalOr(bothZero.Value(), bothInf.Value()), ir.Constant(0xfe00u), result.Value())); - result = IrU32(ir.Select(isNan(1u).Value(), ir.BitwiseOr(bits[1].Value(), ir.Constant(0x0200u)), result.Value())); - result = IrU32(ir.Select(isNan(2u).Value(), ir.BitwiseOr(bits[2].Value(), ir.Constant(0x0200u)), result.Value())); + result = IrU32(ir.Select(isNan(1u).Value(), quietNan16(bits[1]).Value(), result.Value())); + result = IrU32(ir.Select(isNan(2u).Value(), quietNan16(bits[2]).Value(), result.Value())); write16Bits(inst.destination, clampF16Bits(inst.destination, result)); return true; } @@ -250,7 +285,7 @@ bool TranslationContext::vLdexpF16(const RdnaInstruction& inst) { const IrF16 scaled(ir.Emit(IrOpcode::ConvertF16F32, IrType::F16, {&ir.Emit(IrOpcode::FPMul32, IrType::F32, {&value.Value(), &power})})); const IrU32 result(ir.Emit(IrOpcode::ConvertU32U16, IrType::U32, {&ir.Emit(IrOpcode::BitCastU16F16, IrType::U16, {&scaled.Value()})})); const IrU1 nan(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u))); - write16Bits(inst.destination, clampF16Bits(inst.destination, IrU32(ir.Select(nan.Value(), ir.BitwiseOr(bits.Value(), ir.Constant(0x200u)), result.Value())))); + write16Bits(inst.destination, clampF16Bits(inst.destination, IrU32(ir.Select(nan.Value(), quietNan16(bits).Value(), result.Value())))); return true; } @@ -269,7 +304,7 @@ bool TranslationContext::vFrexpF16(const RdnaInstruction& inst, bool exponent) { const IrF16 half(ir.Emit(IrOpcode::ConvertF16F32, IrType::F16, {&ir.BitCastF32(mantissa.Value())})); const IrU32 converted(ir.Emit(IrOpcode::ConvertU32U16, IrType::U32, {&ir.Emit(IrOpcode::BitCastU16F16, IrType::U16, {&half.Value()})})); const IrU1 nan(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u))); - const IrU32 kept(ir.Select(nan.Value(), ir.BitwiseOr(bits.Value(), ir.Constant(0x200u)), bits.Value())); + const IrU32 kept(ir.Select(nan.Value(), quietNan16(bits).Value(), bits.Value())); write16Bits(inst.destination, clampF16Bits(inst.destination, IrU32(ir.Select(special.Value(), kept.Value(), converted.Value())))); return true; } @@ -303,7 +338,7 @@ bool TranslationContext::vMulLegacyF32(const RdnaInstruction& inst, bool accumul IrValue* rhs = readOperand(sourceAt(inst, 1u), IrType::F32); const auto isZero = [&](IrValue* value) { return IrU1(ir.IEqual(ir.BitwiseAnd(ir.BitCastU32(*value), ir.Constant(0x7fffffffu)), ir.Constant(0u))); }; const IrU1 zero(ir.LogicalOr(isZero(lhs).Value(), isZero(rhs).Value())); - IrValue* result = &ir.Emit(IrOpcode::SelectF32, IrType::F32, {&zero.Value(), &ir.ConstantF32(0.0f), &ir.Emit(IrOpcode::FPMul32, IrType::F32, {lhs, rhs})}); + IrValue* result = &ir.Emit(IrOpcode::SelectF32, IrType::F32, {&zero.Value(), &ir.ConstantF32(0.0f), nanResultF32({lhs, rhs}, &flushTinyProduct(lhs, rhs, &ir.Emit(IrOpcode::FPMul32, IrType::F32, {lhs, rhs})).Value())}); if (accumulate) { IrValue* addend = readOperand(accumulatorOperand(inst), IrType::F32); result = &ir.Emit(IrOpcode::FPAdd32, IrType::F32, {result, addend}); @@ -324,7 +359,7 @@ bool TranslationContext::vMullitF32(const RdnaInstruction& inst) { IrU1 lowest(ir.LogicalOr(rhsNegativeMax.Value(), ir.LogicalOr(isNan(limit).Value(), limitNotPositive.Value()))); lowest = IrU1(ir.LogicalOr(lowest.Value(), isNan(rhs).Value())); const IrU32 product(ir.BitCastU32(ir.Emit(IrOpcode::FPMul32, IrType::F32, {&ir.BitCastF32(lhs.Value()), &ir.BitCastF32(rhs.Value())}))); - IrU32 result(ir.Select(isNan(lhs).Value(), ir.BitwiseOr(lhs.Value(), ir.Constant(0x00400000u)), product.Value())); + IrU32 result(ir.Select(isNan(lhs).Value(), quietNan32(lhs).Value(), product.Value())); result = IrU32(ir.Select(ir.LogicalOr(isZero(lhs).Value(), isZero(rhs).Value()), ir.Constant(0u), result.Value())); result = IrU32(ir.Select(lowest.Value(), ir.Constant(0xff7fffffu), result.Value())); writeOperand(inst.destination, &result.Value()); @@ -450,18 +485,18 @@ bool TranslationContext::vDivFixupF32(const RdnaInstruction& inst) { result = IrU32(ir.Select(ir.LogicalOr(isZero(denominator).Value(), isInf(numerator).Value()), infinity.Value(), result.Value())); const IrU1 invalid(ir.LogicalOr(ir.LogicalAnd(isZero(denominator).Value(), isZero(numerator).Value()), ir.LogicalAnd(isInf(denominator).Value(), isInf(numerator).Value()))); result = IrU32(ir.Select(invalid.Value(), ir.Constant(0xffc00000u), result.Value())); - result = IrU32(ir.Select(isNan(denominator).Value(), ir.BitwiseOr(denominator.Value(), ir.Constant(0x400000u)), result.Value())); - result = IrU32(ir.Select(isNan(numerator).Value(), ir.BitwiseOr(numerator.Value(), ir.Constant(0x400000u)), result.Value())); + result = IrU32(ir.Select(isNan(denominator).Value(), quietNan32(denominator).Value(), result.Value())); + result = IrU32(ir.Select(isNan(numerator).Value(), quietNan32(numerator).Value(), result.Value())); writeOperand(inst.destination, &ir.BitCastF32(result.Value())); return true; } bool TranslationContext::floatUnary(const RdnaInstruction& inst, IrOpcode opcode) { - const IrU32 bits = readU32(sourceAt(inst, 0u)); + const IrU32 bits = flushF32Denormal(readU32(sourceAt(inst, 0u))); const IrF32 argument(ir.BitCastF32(bits.Value())); const auto [result, invalid] = unaryFloatSpecials(opcode, argument, IrF32(ir.Emit(opcode, IrType::F32, {&argument.Value()}))); const IrU1 nan(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffffffu)), ir.Constant(0x7f800000u))); - const IrU32 special(ir.Select(nan.Value(), ir.BitwiseOr(bits.Value(), ir.Constant(0x00400000u)), ir.Constant(0xffc00000u))); + const IrU32 special(ir.Select(nan.Value(), quietNan32(bits).Value(), ir.Constant(0xffc00000u))); const IrF32 value(ir.Select(ir.LogicalOr(nan.Value(), invalid.Value()), ir.BitCastF32(special.Value()), result.Value())); writeOperand(inst.destination, &value.Value()); return true; @@ -473,8 +508,14 @@ bool TranslationContext::floatBinary(const RdnaInstruction& inst, IrOpcode opcod const RdnaOperand& operand = sourceAt(inst, reverse ? 1u - index : index); args[index] = readOperand(operand, IrOpcodeArgumentType(opcode, index)); } - IrValue& result = ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1]}); - writeOperand(inst.destination, &result); + IrValue* result = &ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1]}); + if (opcode == IrOpcode::FPMul32) result = &flushTinyProduct(args[0], args[1], result).Value(); + if (opcode == IrOpcode::FPAdd32 || opcode == IrOpcode::FPMul32) result = nanResultF32({args[0], args[1]}, result); + if (opcode == IrOpcode::FPSub32) { + IrValue* negated = &ir.BitCastF32(ir.BitwiseXor(ir.BitCastU32(*args[1]), ir.Constant(0x80000000u))); + result = reverse ? nanResultF32({negated, args[0]}, result) : nanResultF32({args[0], negated}, result); + } + writeOperand(inst.destination, result); return true; } @@ -485,8 +526,42 @@ bool TranslationContext::floatTernary(const RdnaInstruction& inst, IrOpcode opco const IrType type = IrOpcodeArgumentType(opcode, index); args[index] = type == IrType::F32 && mix ? &readMixF32(operand).Value() : readOperand(operand, type); } - IrValue& result = ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1], args[2]}); - writeOperand(inst.destination, &result); + IrValue* result = &ir.Emit(opcode, IrOpcodeType(opcode), {args[0], args[1], args[2]}); + if (opcode == IrOpcode::FPFma32) result = &flushTinyProduct(args[0], args[1], result, args[2]).Value(); + const bool f32Sources = !mix || !(sourceAt(inst, 0u).opSelHi || sourceAt(inst, 1u).opSelHi || (!accumulator && sourceAt(inst, 2u).opSelHi)); + if (opcode == IrOpcode::FPFma32 && f32Sources) result = nanResultF32({args[0], args[1], args[2]}, result, &invalidProductF32(args[0], args[1])); + writeOperand(inst.destination, result); + return true; +} + +bool TranslationContext::ieeeMinMaxF32(const RdnaInstruction& inst, IrOpcode opcode) { + const bool ternary = opcode == IrOpcode::FPMinTri32 || opcode == IrOpcode::FPMaxTri32 || opcode == IrOpcode::FPMedTri32; + std::array args{}; + for (std::uint32_t index = 0u; index < (ternary ? 3u : 2u); ++index) args[index] = readOperand(sourceAt(inst, index), IrType::F32); + const auto signaling = [&](IrValue* value) { + IrValue& bits = ir.BitCastU32(*value); + return &ir.LogicalAnd(ir.IEqual(ir.BitwiseAnd(bits, ir.Constant(0x7fc00000u)), ir.Constant(0x7f800000u)), ir.INotEqual(ir.BitwiseAnd(bits, ir.Constant(0x003fffffu)), ir.Constant(0u))); + }; + const auto quiet = [&](IrValue* value) { return &ir.BitCastF32(ir.BitwiseOr(ir.BitCastU32(*value), ir.Constant(0x00400000u))); }; + const auto pair = [&](IrOpcode pairOpcode, IrValue* lhs, IrValue* rhs) { + IrValue& plain = ir.Emit(pairOpcode, IrType::F32, {lhs, rhs}); + IrValue& rhsChecked = ir.Emit(IrOpcode::SelectF32, IrType::F32, {signaling(rhs), quiet(rhs), &plain}); + return &ir.Emit(IrOpcode::SelectF32, IrType::F32, {signaling(lhs), quiet(lhs), &rhsChecked}); + }; + IrValue* result = nullptr; + if (opcode == IrOpcode::FPMin32 || opcode == IrOpcode::FPMax32) { + result = pair(opcode, args[0], args[1]); + } else if (opcode == IrOpcode::FPMinTri32 || opcode == IrOpcode::FPMaxTri32) { + const auto pairOpcode = opcode == IrOpcode::FPMinTri32 ? IrOpcode::FPMin32 : IrOpcode::FPMax32; + result = pair(pairOpcode, pair(pairOpcode, args[0], args[1]), args[2]); + } else { + const auto isNan = [&](IrValue* value) { return &ir.Emit(IrOpcode::FPIsNan32, IrType::U1, {value}); }; + IrValue& anyNan = ir.LogicalOr(ir.LogicalOr(*isNan(args[0]), *isNan(args[1])), *isNan(args[2])); + IrValue* minimum = pair(IrOpcode::FPMin32, pair(IrOpcode::FPMin32, args[0], args[1]), args[2]); + IrValue& median = ir.Emit(IrOpcode::FPMedTri32, IrType::F32, {args[0], args[1], args[2]}); + result = &ir.Emit(IrOpcode::SelectF32, IrType::F32, {&anyNan, minimum, &median}); + } + writeOperand(inst.destination, result); return true; } @@ -497,12 +572,14 @@ bool TranslationContext::vFmaLegacyF32(const RdnaInstruction& inst) { const auto isZero = [&](IrValue* value) { return IrU1(ir.IEqual(ir.BitwiseAnd(ir.BitCastU32(*value), ir.Constant(0x7fffffffu)), ir.Constant(0u))); }; const IrU1 zero(ir.LogicalOr(isZero(lhs).Value(), isZero(rhs).Value())); const auto factor = [&](IrValue* value) { return &ir.Emit(IrOpcode::SelectF32, IrType::F32, {&zero.Value(), &ir.ConstantF32(0.0f), value}); }; - writeOperand(inst.destination, &ir.Emit(IrOpcode::FPFma32, IrType::F32, {factor(lhs), factor(rhs), addend})); + IrValue* lhsFactor = factor(lhs); + IrValue* rhsFactor = factor(rhs); + writeOperand(inst.destination, nanResultF32({lhsFactor, rhsFactor, addend}, &ir.Emit(IrOpcode::FPFma32, IrType::F32, {lhsFactor, rhsFactor, addend}), &invalidProductF32(lhsFactor, rhsFactor))); return true; } bool TranslationContext::vFrexpMantF32(const RdnaInstruction& inst) { - const IrU32 bits = readU32(sourceAt(inst, 0u)); + const IrU32 bits = flushF32Denormal(readU32(sourceAt(inst, 0u))); const IrU32 exponent(ir.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&bits.Value(), &ir.Constant(23u), &ir.Constant(8u)})); const IrU32 mantissa(ir.BitwiseAnd(bits.Value(), ir.Constant(0x007fffffu))); const IrU32 sign(ir.BitwiseAnd(bits.Value(), ir.Constant(0x80000000u))); @@ -517,7 +594,7 @@ bool TranslationContext::vFrexpMantF32(const RdnaInstruction& inst) { const IrU32 zeroOrSubnormal(ir.Select(zero.Value(), bits.Value(), subnormal.Value())); const IrU32 finite(ir.Select(exponentNonZero.Value(), normal.Value(), zeroOrSubnormal.Value())); const IrU1 exponentAllOnes(ir.IEqual(exponent.Value(), ir.Constant(0xffu))); - const IrU32 special(ir.Select(zero.Value(), bits.Value(), ir.BitwiseOr(bits.Value(), ir.Constant(0x00400000u)))); + const IrU32 special(ir.Select(zero.Value(), bits.Value(), quietNan32(bits).Value())); const IrU32 result(ir.Select(exponentAllOnes.Value(), special.Value(), finite.Value())); writeOperand(inst.destination, &ir.BitCastF32(result.Value())); return true; @@ -534,7 +611,8 @@ bool TranslationContext::vLdexpF32(const RdnaInstruction& inst) { const IrU32 overflow(ir.BitwiseOr(sign.Value(), ir.Constant(0x7f800000u))); IrU32 result(ir.Select(ir.Emit(IrOpcode::SGreaterThanEqual32, IrType::U1, {&scaled.Value(), &ir.Constant(255u)}), overflow.Value(), normal.Value())); result = IrU32(ir.Select(ir.Emit(IrOpcode::SLessThanEqual32, IrType::U1, {&scaled.Value(), &ir.Constant(0u)}), sign.Value(), result.Value())); - result = IrU32(ir.Select(ir.IEqual(exponent.Value(), ir.Constant(0xffu)), bits.Value(), result.Value())); + const IrU1 nan(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffffffu)), ir.Constant(0x7f800000u))); + result = IrU32(ir.Select(ir.IEqual(exponent.Value(), ir.Constant(0xffu)), ir.Select(nan.Value(), quietNan32(bits).Value(), bits.Value()), result.Value())); result = IrU32(ir.Select(ir.IEqual(exponent.Value(), ir.Constant(0u)), sign.Value(), result.Value())); writeOperand(inst.destination, &ir.BitCastF32(result.Value())); return true; diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index b3b6c383d..2f5a7d6c5 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -38,15 +38,13 @@ IrShaderStage toIrShaderStage(ShaderStageKind stage) { } void validateTranslateOptions(const TranslateOptions& options) { + if (options.embeddedFetch != nullptr && options.stage != ShaderStageKind::Vertex && options.stage != ShaderStageKind::Local) throw std::runtime_error("embedded vertex fetch requires a vertex or local shader"); if (options.userDataBaseRegister >= NumScalarRegs || options.userDataCount > NumScalarRegs - options.userDataBaseRegister) { throw std::runtime_error("shader user data exceeds the scalar register bank"); } if (options.waveSize != 32u && options.waveSize != 64u) { throw std::runtime_error("shader translation requires wave32 or wave64, got " + std::to_string(options.waveSize)); } - if (options.embeddedFetch != nullptr && options.stage != ShaderStageKind::Vertex && options.stage != ShaderStageKind::Local) { - throw std::runtime_error("embedded vertex fetch requires a vertex or local shader"); - } switch (options.stage) { case ShaderStageKind::Vertex: case ShaderStageKind::Local: @@ -92,51 +90,26 @@ bool isCodeTableLoad(const ControlFlowGraph& cfg, std::uint32_t programCounter) return std::find(cfg.codeTableLoadProgramCounters.begin(), cfg.codeTableLoadProgramCounters.end(), programCounter) != cfg.codeTableLoadProgramCounters.end(); } -const EmbeddedFetchLoad* findEmbeddedFetchLoad(const EmbeddedFetchPlan* plan, std::uint32_t programCounter) { - if (plan == nullptr) { - return nullptr; +void translateExternalFetch(TranslationContext& context, const RdnaInstruction& call, const ShaderVertexInputInfo* input) { + if (input == nullptr || input->resourcesNum <= 0) { + throw std::runtime_error("s_swappc_b64 fetch-shader call at program counter " + std::to_string(call.programCounter) + " has no parsed fetch-shader plan to inline"); } - const auto found = std::find_if(plan->loads.begin(), plan->loads.end(), [programCounter](const EmbeddedFetchLoad& load) { - return load.programCounter == programCounter; - }); - return found != plan->loads.end() ? &*found : nullptr; -} - -int resolveEmbeddedFetchResource(const ShaderVertexInputInfo& input, const EmbeddedFetchLoad& load) { - if (load.attributeId >= 0 && load.attributeId < input.resourcesNum && input.resourcesDst[load.attributeId].attrId == load.attributeId) { - return load.attributeId; - } - for (int index = 0; index < input.resourcesNum; index++) { - const auto& destination = input.resourcesDst[index]; - if (destination.attrId == load.attributeId && load.componentCount <= static_cast(std::max(destination.registersNum, 1))) { - return index; + for (int index = 0; index < input->resourcesNum && index < ShaderVertexInputInfo::MaxResources; ++index) { + const auto& destination = input->resourcesDst[index]; + if (destination.attrId < 0 || destination.registersNum <= 0) { + continue; } - } - for (int index = 0; index < input.resourcesNum; index++) { - if (input.resourcesDst[index].attrId == load.attributeId) { - return index; + if (destination.registersNum > 4 || destination.registerStart < 0) { + throw std::runtime_error("external fetch-shader attribute " + std::to_string(destination.attrId) + " writes " + std::to_string(destination.registersNum) + + " registers starting at " + std::to_string(destination.registerStart) + "; the inline model supports one to four"); } - } - return -1; -} - -bool isBufferDwordLoad(RdnaOpcode opcode) { - switch (opcode) { - case RdnaOpcode::BufferLoadFormatX: - case RdnaOpcode::BufferLoadFormatXy: - case RdnaOpcode::BufferLoadFormatXyz: - case RdnaOpcode::BufferLoadFormatXyzw: - case RdnaOpcode::BufferLoadDword: - case RdnaOpcode::BufferLoadDwordx2: - case RdnaOpcode::BufferLoadDwordx3: - case RdnaOpcode::BufferLoadDwordx4: - case RdnaOpcode::TbufferLoadFormatX: - case RdnaOpcode::TbufferLoadFormatXy: - case RdnaOpcode::TbufferLoadFormatXyz: - case RdnaOpcode::TbufferLoadFormatXyzw: - return true; - default: - return false; + RdnaInstruction fetch{}; + fetch.op = RdnaOpcode::BufferLoadFormatXyzw; + fetch.programCounter = call.programCounter; + fetch.destination.kind = RdnaOperandKind::VectorRegister; + fetch.destination.reg = static_cast(destination.registerStart); + fetch.formatted = true; + context.TranslateEmbeddedFetch(fetch, static_cast(index), static_cast(destination.registersNum)); } } @@ -365,6 +338,9 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO entryIr.SetVectorReg(static_cast(vgpr(PixelInput::Ancillary)), builtin(StageInputKind::PackedAncillary)); } } else if (options.stage == ShaderStageKind::Vertex) { + if (options.userDataBaseRegister >= 8u) { + entryIr.SetScalarReg(static_cast(3), entryIr.Constant(options.waveSize | (options.waveSize << 8u))); + } entryIr.SetVectorReg(static_cast(5), builtin(StageInputKind::VertexIndex)); entryIr.SetVectorReg(static_cast(8), builtin(StageInputKind::InstanceIndex)); } @@ -457,6 +433,7 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con const auto typedIndex = blockIndices.at(cfgBlock.id); TranslationContext context(program, *blocks[typedIndex], vectorLimit); context.SetPixelInput(options.inputInfo.pixel, options.fragmentShaderBarycentricEnabled); + context.SetFloatMode(options.floatMode); for (std::uint32_t index = cfgBlock.instructionBegin; index < cfgBlock.instructionEnd; index++) { const auto& instruction = decoded.instructions[index]; if (isCodeTableLoad(cfg, instruction.programCounter)) { @@ -465,19 +442,22 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con context.TranslateCodeTableLoad(instruction, *table); continue; } - const auto* embedded = findEmbeddedFetchLoad(options.embeddedFetch, instruction.programCounter); - if (embedded != nullptr && isBufferDwordLoad(instruction.op) && instruction.dataDwordCount == embedded->componentCount && instruction.destination.kind == RdnaOperandKind::VectorRegister) { - if (options.inputInfo.vertex == nullptr) { - throw std::runtime_error("embedded vertex fetch requires vertex input metadata"); + if (options.embeddedFetch != nullptr) { + const auto load = std::ranges::find(options.embeddedFetch->loads, instruction.programCounter, &EmbeddedFetchLoad::programCounter); + if (load != options.embeddedFetch->loads.end()) { + if (options.inputInfo.vertex == nullptr || instruction.destination.kind != RdnaOperandKind::VectorRegister || instruction.dataDwordCount != load->componentCount) throw std::runtime_error("prepared vertex fetch does not match its instruction"); + const auto& vertex = *options.inputInfo.vertex; + std::uint32_t attribute = 0; + while (attribute < static_cast(vertex.resourcesNum) && vertex.resourcesDst[attribute].attrId != load->attributeId) ++attribute; + if (attribute == static_cast(vertex.resourcesNum)) throw std::runtime_error("prepared vertex fetch has no matching semantic"); + context.TranslateEmbeddedFetch(instruction, attribute, load->componentCount); + continue; } - const auto resource = resolveEmbeddedFetchResource(*options.inputInfo.vertex, *embedded); - if (resource < 0 || resource >= options.inputInfo.vertex->resourcesNum) { - throw std::runtime_error("embedded vertex fetch at program counter " + std::to_string(instruction.programCounter) + " has no resource for attribute " + std::to_string(embedded->attributeId)); - } - context.TranslateEmbeddedFetch(instruction, static_cast(resource), embedded->componentCount, options.inputInfo.vertex->resources[resource]); - continue; } context.TranslateInstruction(instruction); + if (cfg.hasFetchCall && instruction.programCounter == cfg.fetchCallProgramCounter) { + translateExternalFetch(context, instruction, options.inputInfo.vertex); + } } context.AddBranchCondition(cfgBlock, blockInfos[typedIndex]); } diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp index 6be88ccce..4b5b9a0fa 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp @@ -35,7 +35,7 @@ MemoryInfo imageMemoryInfoFromInstruction(const RdnaInstruction& inst) { memory.imageDimension = inst.imageDimension; memory.imageAddressComponents = inst.imageAddressComponents; memory.imageHasMip = inst.op == RdnaOpcode::ImageLoadMip || inst.op == RdnaOpcode::ImageStoreMip || inst.op == RdnaOpcode::ImageLoadMipPck || inst.op == RdnaOpcode::ImageLoadMipPckSgn || inst.op == RdnaOpcode::ImageStoreMipPck; - memory.imagePacked = inst.op == RdnaOpcode::ImageLoadPck || inst.op == RdnaOpcode::ImageLoadPckSgn || inst.op == RdnaOpcode::ImageLoadMipPck || inst.op == RdnaOpcode::ImageLoadMipPckSgn || inst.op == RdnaOpcode::ImageStorePck || inst.op == RdnaOpcode::ImageStoreMipPck; + memory.imagePacked = inst.op == RdnaOpcode::ImageLoadPck || inst.op == RdnaOpcode::ImageLoadPckSgn || inst.op == RdnaOpcode::ImageLoadMipPck || inst.op == RdnaOpcode::ImageLoadMipPckSgn || inst.op == RdnaOpcode::ImageStorePck || inst.op == RdnaOpcode::ImageStoreMipPck || inst.op == RdnaOpcode::ImageGather4hPck; memory.dataSigned = inst.op == RdnaOpcode::ImageLoadPckSgn || inst.op == RdnaOpcode::ImageLoadMipPckSgn; memory.imageR128 = inst.imageR128; return memory; @@ -138,6 +138,24 @@ bool TranslationContext::imageGetLod(const RdnaInstruction& inst) { return true; } +bool TranslationContext::imageBy(const RdnaInstruction& inst) { + const auto id = inst.imageOpcodeId; + const bool packed = id >= 0x70u; + if (packed ? id >= 0x76u : (id & 0x10u) != 0u) { + throw std::runtime_error(packed ? "MIMG PCK2/PCK4 stores are not implemented" : "MIMG BY2/BY4 stores are not implemented"); + } + MemoryInfo memory = imageMemoryInfoFromInstruction(inst); + memory.imageHasMip = packed ? id == 0x73u || id == 0x74u : (id & 8u) != 0u; + memory.imagePacked = packed; + memory.imageByElements = packed ? (id == 0x71u || id == 0x74u ? 4u : 2u) : (id & 1u) != 0u ? 4u : 2u; + IrValue* resource = getImageResource(memory); + IrValue* address = makeImageAddress(inst, inst.source0); + IrValue& exec = ir.GetExec(); + IrValue& result = ir.Emit(IrOpcode::ImageRead, IrOpcodeType(IrOpcode::ImageRead), {resource, address, &exec}, addMemoryInfo(memory, inst.programCounter)); + writeImageComponents(inst.destination, &result, memory, 4u); + return true; +} + bool TranslationContext::imageLoad(const RdnaInstruction& inst) { const MemoryInfo memory = imageMemoryInfoFromInstruction(inst); IrValue* resource = getImageResource(memory); @@ -199,6 +217,9 @@ bool TranslationContext::imageSample(const RdnaInstruction& inst) { } bool TranslationContext::imageGather(const RdnaInstruction& inst) { + if (inst.op == RdnaOpcode::ImageGather4hPck && inst.dataBits != 32u) { + throw std::runtime_error("packed horizontal gather with D16 data is not measured"); + } const MemoryInfo memory = imageMemoryInfoFromInstruction(inst); IrValue* resource = getImageResource(memory); IrValue* sampler = getSamplerResource(memory); diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp index 593cf1c43..8e4e86c87 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp @@ -5,6 +5,22 @@ namespace ShaderRecompiler { +namespace { + +const char* globalWaveSyncName(RdnaOpcode opcode) { + switch (opcode) { + case RdnaOpcode::DsGwsInit: return "ds_gws_init"; + case RdnaOpcode::DsGwsSemaV: return "ds_gws_sema_v"; + case RdnaOpcode::DsGwsSemaBr: return "ds_gws_sema_br"; + case RdnaOpcode::DsGwsSemaP: return "ds_gws_sema_p"; + case RdnaOpcode::DsGwsSemaReleaseAll: return "ds_gws_sema_release_all"; + case RdnaOpcode::DsGwsBarrier: return "ds_gws_barrier"; + default: throw std::logic_error("opcode is not a global wave sync instruction"); + } +} + +} + void TranslateMemoryInstruction(IrBuilder& builder, const RdnaInstruction& instruction) { throw std::runtime_error("TranslateMemoryInstruction not implemented"); } @@ -323,6 +339,36 @@ bool TranslationContext::emitMemory(const RdnaInstruction& inst) { return dsAtomic(inst, IrOpcode::SharedAtomicXor32, false); case RdnaOpcode::DsXorRtnB32: return dsAtomic(inst, IrOpcode::SharedAtomicXor32, true); + case RdnaOpcode::DsAddSrc2U32: + return dsSrc2(inst, IrOpcode::SharedAtomicIAdd32); + case RdnaOpcode::DsSubSrc2U32: + return dsSrc2(inst, IrOpcode::SharedAtomicISub32); + case RdnaOpcode::DsRsubSrc2U32: + return dsSrc2(inst, IrOpcode::SharedAtomicRsub32); + case RdnaOpcode::DsIncSrc2U32: + return dsSrc2(inst, IrOpcode::SharedAtomicInc32); + case RdnaOpcode::DsDecSrc2U32: + return dsSrc2(inst, IrOpcode::SharedAtomicDec32); + case RdnaOpcode::DsMinSrc2I32: + return dsSrc2(inst, IrOpcode::SharedAtomicSMin32); + case RdnaOpcode::DsMaxSrc2I32: + return dsSrc2(inst, IrOpcode::SharedAtomicSMax32); + case RdnaOpcode::DsMinSrc2U32: + return dsSrc2(inst, IrOpcode::SharedAtomicUMin32); + case RdnaOpcode::DsMaxSrc2U32: + return dsSrc2(inst, IrOpcode::SharedAtomicUMax32); + case RdnaOpcode::DsAndSrc2B32: + return dsSrc2(inst, IrOpcode::SharedAtomicAnd32); + case RdnaOpcode::DsOrSrc2B32: + return dsSrc2(inst, IrOpcode::SharedAtomicOr32); + case RdnaOpcode::DsXorSrc2B32: + return dsSrc2(inst, IrOpcode::SharedAtomicXor32); + case RdnaOpcode::DsWriteSrc2B32: + return dsSrc2(inst, IrOpcode::WriteSharedU32); + case RdnaOpcode::DsMinSrc2F32: + case RdnaOpcode::DsMaxSrc2F32: + case RdnaOpcode::DsAddSrc2F32: + throw std::runtime_error("f32 DS src2 operations depend on the f32 denormal mode, which is not modelled for LDS float atomics"); case RdnaOpcode::DsWrxchgRtnB32: return dsAtomic(inst, IrOpcode::SharedAtomicSwap32, true); @@ -430,6 +476,8 @@ bool TranslationContext::emitMemory(const RdnaInstruction& inst) { return dsAtomic64(inst, IrOpcode::SharedAtomicFMax64, true); case RdnaOpcode::DsWrxchgRtnB64: return dsAtomic64(inst, IrOpcode::SharedAtomicSwap64, true); + case RdnaOpcode::DsCondxchg32RtnB64: + return dsCondxchg32(inst); case RdnaOpcode::DsNop: emitControlNop(); return true; @@ -443,6 +491,15 @@ bool TranslationContext::emitMemory(const RdnaInstruction& inst) { return dsAppendConsume(inst, IrOpcode::DataConsume); case RdnaOpcode::DsAppend: return dsAppendConsume(inst, IrOpcode::DataAppend); + case RdnaOpcode::DsGwsInit: + case RdnaOpcode::DsGwsSemaV: + case RdnaOpcode::DsGwsSemaBr: + case RdnaOpcode::DsGwsSemaP: + case RdnaOpcode::DsGwsSemaReleaseAll: + case RdnaOpcode::DsGwsBarrier: + throw std::runtime_error(std::string(globalWaveSyncName(inst.op)) + " at pc " + std::to_string(inst.programCounter) + ": global wave sync barriers and semaphores shared by waves of different workgroups are not modeled"); + case RdnaOpcode::DsOrderedCount: + throw std::runtime_error("ds_ordered_count at pc " + std::to_string(inst.programCounter) + ": GDS counters updated in wave launch order are not modeled"); case RdnaOpcode::DsWriteAddtidB32: return dsAddtid(inst, true); case RdnaOpcode::DsReadAddtidB32: @@ -547,6 +604,7 @@ bool TranslationContext::emitMemory(const RdnaInstruction& inst) { case RdnaOpcode::ImageGather4CO: case RdnaOpcode::ImageGather4CLzO: case RdnaOpcode::ImageGather4h: + case RdnaOpcode::ImageGather4hPck: case RdnaOpcode::ImageGather4: case RdnaOpcode::ImageGather4B: case RdnaOpcode::ImageGather4BCl: @@ -598,6 +656,23 @@ bool TranslationContext::emitMemory(const RdnaInstruction& inst) { return imageAtomic(inst, IrOpcode::ImageAtomicFMin32); case RdnaOpcode::ImageAtomicFmax: return imageAtomic(inst, IrOpcode::ImageAtomicFMax32); + case RdnaOpcode::ImageLoadBy2: + case RdnaOpcode::ImageLoadBy4: + case RdnaOpcode::ImageLoadMipBy2: + case RdnaOpcode::ImageLoadMipBy4: + case RdnaOpcode::ImageStoreBy2: + case RdnaOpcode::ImageStoreBy4: + case RdnaOpcode::ImageStoreMipBy2: + case RdnaOpcode::ImageStoreMipBy4: + case RdnaOpcode::ImageLoadPck2: + case RdnaOpcode::ImageLoadPck4: + case RdnaOpcode::ImageLoadMipPck2: + case RdnaOpcode::ImageLoadMipPck4: + case RdnaOpcode::ImageStorePck2: + case RdnaOpcode::ImageStorePck4: + case RdnaOpcode::ImageStoreMipPck2: + case RdnaOpcode::ImageStoreMipPck4: + return imageBy(inst); case RdnaOpcode::ImageLoad: case RdnaOpcode::ImageLoadMip: case RdnaOpcode::ImageLoadPck: diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp index 47327d850..be58eaf47 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp @@ -194,6 +194,23 @@ bool TranslationContext::dsWrite2(const RdnaInstruction& inst) { return true; } +bool TranslationContext::dsSrc2(const RdnaInstruction& inst, IrOpcode opcode) { + if (inst.gds) { + throw std::runtime_error("GDS src2 operations are not supported"); + } + const MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); + const IrU32 address = readU32(inst.source0); + MemoryInfo source = memory; + source.offset = memory.secondaryOffset; + source.secondaryOffset = 0u; + IrValue* value = loadSharedU32(1u, address, source, inst.programCounter); + MemoryInfo target = memory; + target.secondaryOffset = 0u; + IrValue& active = ir.GetExec(); + (void)ir.Emit(opcode, IrOpcodeType(opcode), {&address.Value(), value, &active}, addMemoryInfo(target, inst.programCounter)); + return true; +} + bool TranslationContext::dsAtomic2(const RdnaInstruction& inst, IrOpcode opcode, bool returnsValue) { const MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); const IrU32 address = readU32(inst.source0); @@ -224,6 +241,29 @@ bool TranslationContext::dsAtomic64(const RdnaInstruction& inst, IrOpcode opcode return true; } +bool TranslationContext::dsCondxchg32(const RdnaInstruction& inst) { + if (inst.gds) { + throw std::runtime_error("DS conditional exchange GDS mode is not supported"); + } + MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); + memory.dataDwords = 1u; + memory.componentCount = 1u; + const auto data = readU32Pair(inst.source1); + const IrU32 address = readU32(inst.source0); + IrValue& active = ir.GetExec(); + std::array old{}; + for (std::uint32_t index = 0u; index < 2u; ++index) { + IrValue& enabled = ir.UGreaterThan(data[index].Value(), ir.Constant(0x7fffffffu)); + IrValue& mask = ir.Select(enabled, ir.Constant(0xffffffffu), ir.Constant(0u)); + IrValue& value = ir.Select(enabled, ir.BitwiseAnd(data[index].Value(), ir.Constant(0x7fffffffu)), ir.Constant(0u)); + memory.offset = inst.memoryOffset + index * 4u; + old[index] = &ir.Emit(IrOpcode::SharedAtomicMskor32, IrType::U32, {&address.Value(), &mask, &value, &active}, addMemoryInfo(memory, inst.programCounter)); + } + writeOperand(inst.destination, old[0]); + writeOperand(offsetOperand(inst.destination, 1u), old[1]); + return true; +} + bool TranslationContext::dsAppendConsume(const RdnaInstruction& inst, IrOpcode opcode) { const MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); const IrU32 address = readU32(makeM0Operand()); diff --git a/core/shader/recompiler/Translation/src/ScalarInstructions.cpp b/core/shader/recompiler/Translation/src/ScalarInstructions.cpp index 505d91545..7136560e7 100644 --- a/core/shader/recompiler/Translation/src/ScalarInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ScalarInstructions.cpp @@ -29,6 +29,10 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { return true; case RdnaOpcode::SSetpcB64: return true; + case RdnaOpcode::SSwappcB64: + case RdnaOpcode::SCallB64: + sSwappcB64(inst); + return true; case RdnaOpcode::SSubvectorLoopBegin: sSubvectorLoop(inst, true); return true; @@ -417,6 +421,18 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { return true; case RdnaOpcode::SDenormMode: throw std::runtime_error("s_denorm_mode " + std::to_string(inst.source0.value) + " at pc " + std::to_string(inst.programCounter) + ": the recompiler does not model denormal modes"); + case RdnaOpcode::SSethalt: + if ((inst.source0.value & 1u) != 0u) { + throw std::runtime_error("s_sethalt " + std::to_string(inst.source0.value) + " at pc " + std::to_string(inst.programCounter) + " halts the wave until a debugger resumes it"); + } + emitControlNop(); + return true; + case RdnaOpcode::SSendmsghalt: + throw std::runtime_error("s_sendmsghalt " + std::to_string(inst.source0.value) + " at pc " + std::to_string(inst.programCounter) + " halts the wave until a debugger resumes it"); + case RdnaOpcode::SCodeEnd: + throw std::runtime_error("s_code_end at pc " + std::to_string(inst.programCounter) + " is reached: it marks the end of the code and raises an illegal instruction exception"); + case RdnaOpcode::SRfeB64: + throw std::runtime_error("s_rfe_b64 at pc " + std::to_string(inst.programCounter) + " returns from a trap handler, and recompiled shaders run without one"); case RdnaOpcode::SWaitcntDepctr: case RdnaOpcode::SWaitIdle: emitWaitcnt(); diff --git a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp index 81327c49f..c75a92eec 100644 --- a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp +++ b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp @@ -63,7 +63,7 @@ void _detectVertexBuffers(ShaderVertexInputInfo& info) { } -ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const GuestContext& context, std::uint32_t hostSubgroupSize, const MeshConfiguration* mesh) { +ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const GuestContext& context, std::uint32_t hostSubgroupSize, const MeshConfiguration* mesh, const TessellationConfiguration* tessellation) { switch (stage) { case ShaderStageKind::Compute: { if (!context.compute.has_value()) { @@ -83,6 +83,7 @@ ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const Gues computeStorage.groupId[1] = compute.groupIdEnable[1]; computeStorage.groupId[2] = compute.groupIdEnable[2]; computeStorage.tgSizeEn = compute.tgSizeEnable; + computeStorage.scratchSizeDwords = compute.scratchDwords; computeStorage.threadIdsNum = static_cast(compute.threadIdComponentCount); computeStorage.partialGroups = compute.PartialGroups(); // Workgroup ids (and the thread-group size word) follow the user SGPRs. @@ -122,7 +123,6 @@ ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const Gues place(PixelInput::Ancillary, pixel.ancillary); for (std::uint32_t i = 0; i < 8; ++i) { pixelStorage.targetOutputMode[i] = pixel.targetOutputMode[i]; - pixelStorage.targetExportMapping[i].packed = pixel.targetExportMapping[i]; } pixelStorage.psPosX = pixel.posX; pixelStorage.psPosY = pixel.posY; @@ -138,6 +138,7 @@ ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const Gues pixelStorage.psEarlyZ = pixel.earlyZ; pixelStorage.psExecuteOnNoop = pixel.executeOnNoop; pixelStorage.psConservativeZExport = pixel.conservativeZExport; + pixelStorage.psOrderedPixelShader = pixel.orderedPixelShader; ShaderStageInputInfo result; result.pixel = &pixelStorage; return result; @@ -161,14 +162,26 @@ ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const Gues vertexStorage.fetchAttribReg = static_cast(vertex.fetchAttribReg); vertexStorage.fetchBufferReg = static_cast(vertex.fetchBufferReg); vertexStorage.resourcesNum = static_cast(vertex.resourcesNum); - for (std::uint32_t i = 0; i < vertex.resourcesNum; ++i) { + for (std::uint32_t i = 0; i < static_cast(vertexStorage.resourcesNum); ++i) { vertexStorage.resources[i].fields = vertex.resources[i].fields; + if (vertex.fetchEmbedded) vertexStorage.resources[i].fields = {0u, 0u, 0u, (static_cast(IrBufferFormat::Format32_32_32_32Float) << 12u) | ShaderImageIdentitySwizzle}; vertexStorage.resourcesDst[i].registerStart = vertex.resourcesDst[i].registerStart; vertexStorage.resourcesDst[i].registersNum = vertex.resourcesDst[i].registersNum; vertexStorage.resourcesDst[i].attrId = vertex.resourcesDst[i].attrId; - vertexStorage.resourcesDst[i].fetchIndex = vertex.resourcesDst[i].fetchIndex; + vertexStorage.resourcesDst[i].fetchIndex = vertex.fetchEmbedded ? 0u : vertex.resourcesDst[i].fetchIndex; } _detectVertexBuffers(vertexStorage); + if (stage == ShaderStageKind::Local || stage == ShaderStageKind::TessellationControl || stage == ShaderStageKind::TessellationEvaluation) { + if (tessellation == nullptr) throw std::runtime_error("ShaderInputInfoBuilder: tessellation configuration is missing"); + if (tessellation->inputControlPoints == 0u || tessellation->inputControlPoints > 32u || tessellation->outputControlPoints == 0u || tessellation->outputControlPoints > 32u) throw std::runtime_error("ShaderInputInfoBuilder: invalid tessellation control-point counts"); + if (tessellation->domain != 1u || tessellation->partitioning != 2u || tessellation->outputTopology != 2u) throw std::runtime_error("ShaderInputInfoBuilder: unsupported tessellation configuration"); + auto& target = vertexStorage.tess; + target.inputControlPoints = tessellation->inputControlPoints; + target.outputControlPoints = tessellation->outputControlPoints; + target.domain = tessellation->domain; + target.partitioning = tessellation->partitioning; + target.outputTopology = tessellation->outputTopology; + } if (stage == ShaderStageKind::Mesh) { if (mesh == nullptr) throw std::runtime_error("ShaderInputInfoBuilder: a mesh-stage program has no mesh configuration"); auto& target = vertexStorage.mesh; diff --git a/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp b/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp index 26e4a0c88..66b55249d 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp @@ -10,6 +10,14 @@ std::array TranslationContext::ballotMask(IrU1 value) { return {IrU32(ir.CompositeExtract(mask, 0u)), program.WaveSize() == 64u ? IrU32(ir.CompositeExtract(mask, 1u)) : IrU32(ir.Constant(0u))}; } +IrU32 TranslationContext::hostExecWord(std::uint32_t half) { + const IrU32 word(half == 0u ? ir.GetExecLo() : ir.GetExecHi()); + if (half != 0u && program.WaveSize() != 64u) { + return word; + } + return IrU32(ir.BitwiseAnd(word.Value(), ballotMask(IrU1(ir.ConstantBool(true)))[half].Value())); +} + IrU1 TranslationContext::threadBit(const std::array& mask) { const IrU32 lane(ir.Emit(IrOpcode::LaneId, IrType::U32, {})); const IrU32 word = program.WaveSize() == 64u ? IrU32(ir.Select(ir.ULessThan(lane.Value(), ir.Constant(32u)), mask[0].Value(), mask[1].Value())) : mask[0]; diff --git a/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp b/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp index d0aafbe91..3136b2f11 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp @@ -125,6 +125,28 @@ RdnaOperand TranslationContext::plainOperand(const RdnaOperand& operand) { return result; } +IrU32 TranslationContext::flushF32Denormal(IrU32 bits) { + if ((f32DenormalFlush & 1u) == 0u) return bits; + const IrU1 denormal(ir.IEqual(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7f800000u)), ir.Constant(0u))); + return IrU32(ir.Select(denormal.Value(), ir.BitwiseAnd(bits.Value(), ir.Constant(0x80000000u)), bits.Value())); +} + +IrF32 TranslationContext::flushTinyProduct(IrValue* lhs, IrValue* rhs, IrValue* product, IrValue* addend) { + if ((f32DenormalFlush & 2u) == 0u) return IrF32(*product); + IrValue& lhsBits = ir.BitCastU32(*lhs); + IrValue& rhsBits = ir.BitCastU32(*rhs); + const auto exponent = [&](IrValue& bits) -> IrValue& { return ir.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&bits, &ir.Constant(23u), &ir.Constant(8u)}); }; + const auto significand = [&](IrValue& bits) -> IrValue& { return ir.BitwiseOr(ir.BitwiseAnd(bits, ir.Constant(0x007fffffu)), ir.Constant(0x00800000u)); }; + IrValue& lhsExponent = exponent(lhsBits); + IrValue& rhsExponent = exponent(rhsBits); + const auto finiteNonZero = [&](IrValue& value) -> IrValue& { return ir.LogicalAnd(ir.INotEqual(value, ir.Constant(0u)), ir.INotEqual(value, ir.Constant(0xffu))); }; + IrValue& carry = ir.Select(ir.UGreaterThan(ir.Emit(IrOpcode::UMulHi, IrType::U32, {&significand(lhsBits), &significand(rhsBits)}), ir.Constant(0x7fffu)), ir.Constant(1u), ir.Constant(0u)); + IrValue* tiny = &ir.LogicalAnd(ir.LogicalAnd(finiteNonZero(lhsExponent), finiteNonZero(rhsExponent)), ir.ULessThan(ir.IAdd(ir.IAdd(lhsExponent, rhsExponent), carry), ir.Constant(128u))); + if (addend != nullptr) tiny = &ir.LogicalAnd(*tiny, ir.IEqual(ir.BitwiseAnd(ir.BitCastU32(*addend), ir.Constant(0x7fffffffu)), ir.Constant(0u))); + IrValue& sign = ir.BitwiseAnd(ir.BitwiseXor(lhsBits, rhsBits), ir.Constant(0x80000000u)); + return IrF32(ir.Emit(IrOpcode::SelectF32, IrType::F32, {tiny, &ir.BitCastF32(sign), product})); +} + IrValue* TranslationContext::readOperand(const RdnaOperand& operand, IrType type) { if (type == IrType::U16) { return &ir.Emit(IrOpcode::ConvertU16U32, IrType::U16, {&applyBitSourceModifiers(operand, readRawU32(operand)).Value()}); @@ -158,7 +180,7 @@ IrValue* TranslationContext::readOperand(const RdnaOperand& operand, IrType type bits = IrU32(ir.BitwiseXor(bits.Value(), ir.Constant(0x80000000u))); } if (TypesOverlap(type, IrType::F32) && !TypesOverlap(type, IrType::U32)) { - return &ir.BitCastF32(bits.Value()); + return &ir.BitCastF32(flushF32Denormal(bits).Value()); } if (!TypesOverlap(type, IrType::U32)) { throw std::runtime_error("TranslationContext::readOperand requested unsupported operand type"); @@ -172,6 +194,7 @@ void TranslationContext::writeOperand(const RdnaOperand& operand, IrValue* value } IrType type = value->Type(); if (type == IrType::F32) { + if ((f32DenormalFlush & 2u) != 0u) value = &ir.BitCastF32(flushF32Denormal(IrU32(ir.BitCastU32(*value))).Value()); value = &applyF32ResultModifiers(operand, IrF32(*value)).Value(); type = IrType::F32; } @@ -248,8 +271,48 @@ IrU32 TranslationContext::applyBitSourceModifiers(const RdnaOperand& operand, Ir return value; } +bool TranslationContext::outputModifierApplies(std::uint32_t denormalShift) const { + const std::uint32_t mode = floatMode.has_value() ? floatMode->floatMode : 0xc0u; + return !ieeeMode && ((mode >> denormalShift) & 2u) == 0u; +} + +void TranslationContext::rejectHalfOrDoubleOutputModifier(const RdnaOperand& operand) const { + if (operand.omod != 0u && outputModifierApplies(6u)) { + throw std::runtime_error("output modifier on an f16 or f64 result with f16/f64 output denormals flushed is not implemented"); + } +} + +IrU32 TranslationContext::quietNan32(IrU32 bits) { + return ieeeMode ? IrU32(ir.BitwiseOr(bits.Value(), ir.Constant(0x00400000u))) : bits; +} + +IrValue* TranslationContext::nanResultF32(std::initializer_list sources, IrValue* result, IrValue* invalidProduct) { + const auto isNan = [&](IrValue& bits) -> IrValue& { return ir.UGreaterThan(ir.BitwiseAnd(bits, ir.Constant(0x7fffffffu)), ir.Constant(0x7f800000u)); }; + IrValue* bits = &ir.BitCastU32(*result); + bits = &ir.Select(isNan(*bits), ir.Constant(0xffc00000u), *bits); + for (auto source = sources.end(); source != sources.begin();) { + --source; + IrValue& sourceBits = ir.BitCastU32(**source); + IrValue* selected = &isNan(sourceBits); + if (invalidProduct != nullptr && source + 1 == sources.end() && sources.size() == 3u) selected = &ir.LogicalAnd(*selected, ir.LogicalNot(*invalidProduct)); + bits = &ir.Select(*selected, quietNan32(IrU32(sourceBits)).Value(), *bits); + } + return &ir.BitCastF32(*bits); +} + +IrValue& TranslationContext::invalidProductF32(IrValue* lhs, IrValue* rhs) { + IrValue& lhsMagnitude = ir.BitwiseAnd(ir.BitCastU32(*lhs), ir.Constant(0x7fffffffu)); + IrValue& rhsMagnitude = ir.BitwiseAnd(ir.BitCastU32(*rhs), ir.Constant(0x7fffffffu)); + const auto infZero = [&](IrValue& inf, IrValue& zero) -> IrValue& { return ir.LogicalAnd(ir.IEqual(inf, ir.Constant(0x7f800000u)), ir.IEqual(zero, ir.Constant(0u))); }; + return ir.LogicalOr(infZero(lhsMagnitude, rhsMagnitude), infZero(rhsMagnitude, lhsMagnitude)); +} + +IrU32 TranslationContext::quietNan16(IrU32 bits) { + return ieeeMode ? IrU32(ir.BitwiseOr(bits.Value(), ir.Constant(0x0200u))) : bits; +} + IrF32 TranslationContext::applyF32ResultModifiers(const RdnaOperand& operand, IrF32 value) { - if (operand.omod != 0u) { + if (operand.omod != 0u && outputModifierApplies(4u)) { IrValue& bits = ir.BitCastU32(value.Value()); IrValue& magnitude = ir.BitwiseAnd(bits, ir.Constant(0x7fffffffu)); IrValue& sign = ir.BitwiseAnd(bits, ir.Constant(0x80000000u)); @@ -265,29 +328,34 @@ IrF32 TranslationContext::applyF32ResultModifiers(const RdnaOperand& operand, Ir value = IrF32(ir.BitCastF32(*result)); } if (operand.clamp) { - value = IrF32(ir.Emit(IrOpcode::FPSaturate32, IrType::F32, {&value.Value()})); + const IrF32 saturated(ir.Emit(IrOpcode::FPSaturate32, IrType::F32, {&value.Value()})); + value = dx10Clamp() ? saturated : selectF32(IrU1(ir.Emit(IrOpcode::FPIsNan32, IrType::U1, {&value.Value()})), value, saturated); } return value; } IrF32 TranslationContext::applyF16ResultModifiers(const RdnaOperand& operand, IrF32 value) { + rejectHalfOrDoubleOutputModifier(operand); if (!operand.clamp) { return value; } const IrF32 zero(ir.ConstantF32(0.0f)); const IrU1 positive(ir.Emit(IrOpcode::FPOrdGreaterThan32, IrType::U1, {&value.Value(), &zero.Value()})); const IrF32 limited(ir.Emit(IrOpcode::FPMin32, IrType::F32, {&value.Value(), &ir.ConstantF32(1.0f)})); - return selectF32(positive, limited, zero); + const IrF32 clamped = selectF32(positive, limited, zero); + return dx10Clamp() ? clamped : selectF32(IrU1(ir.Emit(IrOpcode::FPIsNan32, IrType::U1, {&value.Value()})), value, clamped); } IrU32 TranslationContext::clampF16Bits(const RdnaOperand& operand, IrU32 bits) { + rejectHalfOrDoubleOutputModifier(operand); if (!operand.clamp) { return bits; } const IrU32 magnitude(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu))); const IrU1 zero(ir.LogicalOr(ir.UGreaterThan(bits.Value(), ir.Constant(0x7fffu)), ir.UGreaterThan(magnitude.Value(), ir.Constant(0x7c00u)))); const IrU32 limited(ir.Select(ir.UGreaterThan(magnitude.Value(), ir.Constant(0x3c00u)), ir.Constant(0x3c00u), bits.Value())); - return IrU32(ir.Select(zero.Value(), ir.Constant(0u), limited.Value())); + const IrU32 clamped(ir.Select(zero.Value(), ir.Constant(0u), limited.Value())); + return dx10Clamp() ? clamped : IrU32(ir.Select(ir.UGreaterThan(magnitude.Value(), ir.Constant(0x7c00u)), bits.Value(), clamped.Value())); } IrU32 TranslationContext::readScalarCode(std::uint32_t code) { @@ -319,15 +387,15 @@ IrU32 TranslationContext::readRawU32(const RdnaOperand& operand) { case RdnaOperandKind::VccLo: return IrU32(ir.GetVccLo()); case RdnaOperandKind::VccHi: return IrU32(ir.GetVccHi()); case RdnaOperandKind::M0: return IrU32(ir.GetM0()); - case RdnaOperandKind::ExecLo: return IrU32(ir.GetExecLo()); - case RdnaOperandKind::ExecHi: return IrU32(ir.GetExecHi()); + case RdnaOperandKind::ExecLo: return hostExecWord(0u); + case RdnaOperandKind::ExecHi: return hostExecWord(1u); case RdnaOperandKind::Scc: return IrU32(ir.Select(ir.GetScc(), ir.Constant(1u), ir.Constant(0u))); case RdnaOperandKind::VccZ: case RdnaOperandKind::ExecZ: { const bool vcc = operand.kind == RdnaOperandKind::VccZ; - IrU32 mask(vcc ? ir.GetVccLo() : ir.GetExecLo()); + IrU32 mask(vcc ? IrU32(ir.GetVccLo()) : hostExecWord(0u)); if (program.WaveSize() == 64u) { - mask = IrU32(ir.BitwiseOr(mask.Value(), vcc ? ir.GetVccHi() : ir.GetExecHi())); + mask = IrU32(ir.BitwiseOr(mask.Value(), vcc ? ir.GetVccHi() : hostExecWord(1u).Value())); } const IrU32 zero(ir.Constant(0u)); return IrU32(ir.Select(ir.IEqual(mask.Value(), zero.Value()), ir.Constant(1u), zero.Value())); diff --git a/core/shader/recompiler/Translation/src/VectorInstructions.cpp b/core/shader/recompiler/Translation/src/VectorInstructions.cpp index 1a60e2525..9b5fd4580 100644 --- a/core/shader/recompiler/Translation/src/VectorInstructions.cpp +++ b/core/shader/recompiler/Translation/src/VectorInstructions.cpp @@ -14,12 +14,89 @@ bool roundsProductSeparately(const RdnaInstruction& inst) { if (inst.op == RdnaOpcode::VMadF32) { return true; } + if (inst.family == RdnaInstructionFamily::VOP3P) { + return false; + } const std::uint32_t vop2 = inst.family == RdnaInstructionFamily::VOP3 ? inst.opcodeId - 0x100u : inst.opcodeId; return vop2 == 0x1fu || vop2 == 0x20u || vop2 == 0x21u; } } +std::uint32_t TranslationContext::f32DenormalFlushFor(const RdnaInstruction& inst) const { + std::uint32_t flush = 0u; + switch (inst.op) { + case RdnaOpcode::VAddF32: + case RdnaOpcode::VSubF32: + case RdnaOpcode::VSubrevF32: + case RdnaOpcode::VMulF32: + case RdnaOpcode::VMulLegacyF32: + case RdnaOpcode::VMinF32: + case RdnaOpcode::VMaxF32: + case RdnaOpcode::VMin3F32: + case RdnaOpcode::VMax3F32: + case RdnaOpcode::VMed3F32: + case RdnaOpcode::VFractF32: + case RdnaOpcode::VFloorF32: + case RdnaOpcode::VCeilF32: + case RdnaOpcode::VFrexpMantF32: + case RdnaOpcode::VSinF32: + case RdnaOpcode::VCosF32: + flush = 3u; + break; + case RdnaOpcode::VMacF32: + case RdnaOpcode::VMadmkF32: + case RdnaOpcode::VMadakF32: + case RdnaOpcode::VFmaF32: + flush = roundsProductSeparately(inst) ? 0u : 3u; + break; + case RdnaOpcode::VCmpEqF32: + case RdnaOpcode::VCmpGeF32: + case RdnaOpcode::VCmpGtF32: + case RdnaOpcode::VCmpLeF32: + case RdnaOpcode::VCmpLgF32: + case RdnaOpcode::VCmpLtF32: + case RdnaOpcode::VCmpNeqF32: + case RdnaOpcode::VCmpNgeF32: + case RdnaOpcode::VCmpNgtF32: + case RdnaOpcode::VCmpNleF32: + case RdnaOpcode::VCmpNlgF32: + case RdnaOpcode::VCmpNltF32: + case RdnaOpcode::VCmpxEqF32: + case RdnaOpcode::VCmpxGeF32: + case RdnaOpcode::VCmpxGtF32: + case RdnaOpcode::VCmpxLeF32: + case RdnaOpcode::VCmpxLgF32: + case RdnaOpcode::VCmpxLtF32: + case RdnaOpcode::VCmpxNeqF32: + case RdnaOpcode::VCmpxNgeF32: + case RdnaOpcode::VCmpxNgtF32: + case RdnaOpcode::VCmpxNleF32: + case RdnaOpcode::VCmpxNlgF32: + case RdnaOpcode::VCmpxNltF32: + case RdnaOpcode::VCvtI32F32: + case RdnaOpcode::VCvtU32F32: + case RdnaOpcode::VCvtRpiI32F32: + case RdnaOpcode::VCvtFlrI32F32: + case RdnaOpcode::VFrexpExpI32F32: + case RdnaOpcode::VCvtF64F32: + case RdnaOpcode::VCvtPknormI16F32: + case RdnaOpcode::VCvtPknormU16F32: + case RdnaOpcode::VCvtPkU8F32: + case RdnaOpcode::VCvtPkrtzF16F32: + case RdnaOpcode::VCvtF16F32: + flush = 1u; + break; + default: + return 0u; + } + const std::uint32_t denormals = floatMode.has_value() ? (floatMode->floatMode >> 4u) & 3u : 0u; + if (denormals == 1u || denormals == 2u) { + throw std::runtime_error("f32 denormal mode " + std::to_string(denormals) + " at pc " + std::to_string(inst.programCounter) + " is not implemented"); + } + return denormals == 0u ? flush : 0u; +} + bool TranslationContext::emitVector(const RdnaInstruction& inst) { switch (inst.op) { case RdnaOpcode::VNop: @@ -933,9 +1010,9 @@ bool TranslationContext::emitVector(const RdnaInstruction& inst) { case RdnaOpcode::VMulF32: return floatBinary(inst, IrOpcode::FPMul32, false); case RdnaOpcode::VMinF32: - return floatBinary(inst, IrOpcode::FPMin32, false); + return ieeeMode ? ieeeMinMaxF32(inst, IrOpcode::FPMin32) : floatBinary(inst, IrOpcode::FPMin32, false); case RdnaOpcode::VMaxF32: - return floatBinary(inst, IrOpcode::FPMax32, false); + return ieeeMode ? ieeeMinMaxF32(inst, IrOpcode::FPMax32) : floatBinary(inst, IrOpcode::FPMax32, false); case RdnaOpcode::VDivScaleF32: return vDivScaleF32(inst); case RdnaOpcode::VDivFmasF32: @@ -960,11 +1037,11 @@ bool TranslationContext::emitVector(const RdnaInstruction& inst) { case RdnaOpcode::VMadLegacyF32: return vFmaLegacyF32(inst); case RdnaOpcode::VMin3F32: - return floatTernary(inst, IrOpcode::FPMinTri32, false, false); + return ieeeMode ? ieeeMinMaxF32(inst, IrOpcode::FPMinTri32) : floatTernary(inst, IrOpcode::FPMinTri32, false, false); case RdnaOpcode::VMax3F32: - return floatTernary(inst, IrOpcode::FPMaxTri32, false, false); + return ieeeMode ? ieeeMinMaxF32(inst, IrOpcode::FPMaxTri32) : floatTernary(inst, IrOpcode::FPMaxTri32, false, false); case RdnaOpcode::VMed3F32: - return floatTernary(inst, IrOpcode::FPMedTri32, false, false); + return ieeeMode ? ieeeMinMaxF32(inst, IrOpcode::FPMedTri32) : floatTernary(inst, IrOpcode::FPMedTri32, false, false); case RdnaOpcode::VDot2cF32F16: return vDot2cF32F16(inst); case RdnaOpcode::VDot2F32F16: diff --git a/core/shader/recompiler/VertexInputSpecialization.hpp b/core/shader/recompiler/VertexInputSpecialization.hpp new file mode 100644 index 000000000..2b2722eed --- /dev/null +++ b/core/shader/recompiler/VertexInputSpecialization.hpp @@ -0,0 +1,131 @@ +#ifndef CORE_SHADER_RECOMPILER_VERTEXINPUTSPECIALIZATION_HPP +#define CORE_SHADER_RECOMPILER_VERTEXINPUTSPECIALIZATION_HPP + +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { + +inline SharedSpirv SpecializeVertexInputTypes(const CompiledShaderArtifact& artifact, std::span classes) { + if (artifact.vertexInputPatches.empty()) return artifact.spirv; + auto words = artifact.spirv.Words(); + for (const auto& patch : artifact.vertexInputPatches) { + if (patch.location >= classes.size() || classes[patch.location] >= patch.values.size() || patch.word >= words.size()) throw std::runtime_error("invalid prepared vertex type patch"); + words[patch.word] = patch.values[classes[patch.location]]; + } + return SharedSpirv(std::move(words)); +} + +inline void PrepareVertexInputSpecialization(CompiledShaderArtifact& artifact) { + const auto& source = artifact.spirv.Words(); + if (source.size() < 5u || source[0] != spv::MagicNumber) throw std::runtime_error("invalid vertex specialization module"); + std::map locations; + std::map, std::uint32_t> types; + std::vector> declarations; + std::vector> variables; + std::map pointers; + std::map loads; + std::map> variableTypes; + std::map components; + for (const auto& input : artifact.vertexInputs) { + if (input.location >= ShaderVertexStageInfo::MaxResources || input.components == 0u || input.components > 4u || !components.emplace(input.location, input.components).second) throw std::runtime_error("invalid vertex specialization input"); + } + auto bound = source[3]; + const auto declare = [&](std::vector key) { + const auto found = types.find(key); + if (found != types.end()) return found->second; + if (bound == std::numeric_limits::max()) throw std::runtime_error("vertex specialization module ID overflow"); + const auto id = bound++; + std::vector words{(static_cast(key.size() + 1u) << 16u) | key.front(), id}; + words.insert(words.end(), key.begin() + 1, key.end()); + types.emplace(std::move(key), id); + declarations.push_back(std::move(words)); + return id; + }; + for (std::size_t cursor = 5; cursor < source.size();) { + const auto count = source[cursor] >> 16u; + const auto op = static_cast(source[cursor] & 0xffffu); + if (count == 0u || count > source.size() - cursor) throw std::runtime_error("truncated vertex specialization instruction"); + if (op == spv::OpDecorate && count == 4u && source[cursor + 2u] == spv::DecorationLocation) locations.emplace(source[cursor + 1u], source[cursor + 3u]); + if (op == spv::OpTypeInt || op == spv::OpTypeFloat || op == spv::OpTypeVector || op == spv::OpTypePointer) { + std::vector key{static_cast(op)}; + key.insert(key.end(), source.begin() + cursor + 2u, source.begin() + cursor + count); + types.emplace(std::move(key), source[cursor + 1u]); + } + cursor += count; + } + const std::array scalarTypes{declare({spv::OpTypeFloat, 32u}), declare({spv::OpTypeInt, 32u, 1u}), declare({spv::OpTypeInt, 32u, 0u})}; + std::array scalarPointers{}; + for (std::size_t kind = 0; kind < 3u; ++kind) scalarPointers[kind] = declare({spv::OpTypePointer, spv::StorageClassInput, scalarTypes[kind]}); + for (std::size_t cursor = 5; cursor < source.size();) { + const auto count = source[cursor] >> 16u; + const auto op = static_cast(source[cursor] & 0xffffu); + if (op == spv::OpVariable && count == 4u && source[cursor + 3u] == spv::StorageClassInput) { + const auto location = locations.find(source[cursor + 2u]); + if (location != locations.end() && components.contains(location->second)) { + const auto id = source[cursor + 2u]; + pointers.emplace(id, location->second); + auto& choices = variableTypes[id]; + for (std::size_t kind = 0; kind < 3u; ++kind) { + const auto count = components.at(location->second); + const auto type = count == 1u ? scalarTypes[kind] : declare({spv::OpTypeVector, scalarTypes[kind], count}); + choices[kind] = declare({spv::OpTypePointer, spv::StorageClassInput, type}); + } + variables.emplace_back(source.begin() + cursor, source.begin() + cursor + count); + } + } + cursor += count; + } + if (variables.size() != artifact.vertexInputs.size()) throw std::runtime_error("vertex specialization inputs disagree with metadata"); + std::vector result(source.begin(), source.begin() + 5); + result[3] = bound; + bool inserted = false; + const auto patch = [&](std::uint32_t location, std::size_t word, const std::array& values) { + if (word > std::numeric_limits::max()) throw std::runtime_error("vertex specialization offset overflow"); + artifact.vertexInputPatches.push_back({location, static_cast(word), values}); + }; + artifact.vertexInputPatches.clear(); + for (std::size_t cursor = 5; cursor < source.size();) { + const auto count = source[cursor] >> 16u; + const auto op = static_cast(source[cursor] & 0xffffu); + if (op == spv::OpFunction && !inserted) { + for (const auto& declaration : declarations) result.insert(result.end(), declaration.begin(), declaration.end()); + for (const auto& variable : variables) { + patch(pointers.at(variable[2]), result.size() + 1u, variableTypes.at(variable[2])); + result.insert(result.end(), variable.begin(), variable.end()); + } + inserted = true; + } + if (op == spv::OpVariable && variableTypes.contains(source[cursor + 2u])) { + cursor += count; + continue; + } + if (op == spv::OpAccessChain && count == 5u && pointers.contains(source[cursor + 3u])) { + const auto location = pointers.at(source[cursor + 3u]); + pointers.emplace(source[cursor + 2u], location); + patch(location, result.size() + 1u, scalarPointers); + } else if (op == spv::OpLoad && count == 4u && pointers.contains(source[cursor + 3u])) { + const auto location = pointers.at(source[cursor + 3u]); + loads.emplace(source[cursor + 2u], location); + patch(location, result.size() + 1u, scalarTypes); + } else if (op == spv::OpBitcast && count == 4u && loads.contains(source[cursor + 3u])) { + patch(loads.at(source[cursor + 3u]), result.size(), {(4u << 16u) | spv::OpBitcast, (4u << 16u) | spv::OpBitcast, (4u << 16u) | spv::OpCopyObject}); + } + result.insert(result.end(), source.begin() + cursor, source.begin() + cursor + count); + cursor += count; + } + if (!inserted) throw std::runtime_error("vertex specialization module has no function"); + artifact.spirv = std::move(result); +} + +} + +#endif diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index 847b63cf7..2b5eea0f9 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -1,5 +1,23 @@ # Build +## Relinker only + +The relinker can be built without initializing submodules or configuring SDL, Vulkan, FFmpeg, FreeType or the shader recompiler. It requires CMake, a C++20 compiler and a build tool. Python 3 enables the Python regression tests. + +```sh +cmake -S . -B build-relinker -G Ninja -DCMAKE_BUILD_TYPE=Release -DANYPS5_RELINKER_ONLY=ON -DBUILD_TESTING=ON +cmake --build build-relinker --parallel +ctest --test-dir build-relinker --output-on-failure +``` + +This mode builds the conversion tool and its tests on Linux, Windows and macOS, including Apple Silicon. macOS uses AppleClang from the Xcode command-line tools; Windows uses the MinGW-w64 toolchain below. The executable is `build-relinker/core/relinker/relinker` (`relinker.exe` on Windows with Ninja). + +The output remains x86-64 Linux ELF or Windows PE. Converted games need system libraries built for the target OS and a compatible x86-64 host. This mode does not build those libraries or provide macOS game execution. Tests inspect both output formats; execution checks run only on their compatible hosts. + +Use a separate build directory for the full build. + +## Full build + ```sh git submodule update --init --recursive ``` @@ -7,9 +25,9 @@ git submodule update --init --recursive ## Requirements - x86-64, Git, CMake 3.22.1 or newer, Ninja, C++20. -- Linux: GCC, G++, binutils. +- Linux: GCC, G++, binutils. SDL's X11 backend requires X11 and Xext development headers (`libx11-dev` and `libxext-dev` on Debian/Ubuntu). - Windows: only MinGW-w64 GCC 15.2.0 (WinLibs `x86_64-ucrt-posix-seh`, release `15.2.0posix-14.0.0-ucrt-r7`) is currently supported. Add its `mingw64/bin` directory to `PATH` before configuring. -- FFmpeg binaries are downloaded during configuration unless `FFMPEG_PREBUILT_DIR` is set. +- FFmpeg binaries are downloaded during configuration unless `FFMPEG_PREBUILT_DIR` is set. With the WinLibs CMake, the download fails with status 60 (`SSL peer certificate or SSH remote key was not OK`) unless `SSL_CERT_FILE` names a CA bundle, for example `C:\Program Files\Git\mingw64\etc\ssl\certs\ca-bundle.crt` from Git for Windows, as in CI. ## Commands @@ -19,6 +37,11 @@ cmake --build build --parallel cmake --build build --target libs --parallel ``` +`libs` is a custom target: every library under [core/libs/prx](../../core/libs/prx) is built with +`EXCLUDE_FROM_ALL`, so the first command alone does not produce them. Titles load the patched `.prx` +files from `build/core/libs/libs`, which only that second step refreshes. Running a title after a +library change without it therefore tests the previous binaries and can show no effect at all. + [Relinker usage and runtime layout](../user/USAGE.md). ## CMake flags @@ -28,6 +51,7 @@ Project switches accept `ON` or `OFF`: | Flag | Default | Effect | |----------------------------------|---------|--------------------------------------------------| | `-DBUILD_TESTING=ON` | `OFF` | Build and register tests. | +| `-DANYPS5_RELINKER_ONLY=ON` | `OFF` | Build only the relinker and its tests, without third-party dependencies. | | `-DANYPS5_ENABLE_SPIRV_TOOLS=ON` | `OFF` | Enable SPIR-V validation and optimization. | | `-DAPS5_ENABLE_TIMING_LOG=ON` | `OFF` | Compile frame timing logging. | | `-DAPS5_AGC_CREATE_LOG=OFF` | `ON` | Disable successful `sceAgcCreateShader` logging. | diff --git a/docs/dev/HW_ORACLE.md b/docs/dev/HW_ORACLE.md index 917efdc65..464033fa2 100644 --- a/docs/dev/HW_ORACLE.md +++ b/docs/dev/HW_ORACLE.md @@ -30,8 +30,20 @@ from the GPU (`HW_ORACLE_TARGET` overrides it). - Bytes passed as `extra` follow the rows of each dispatch, at `s[4:5] + 16 * rows`: buffer contents, texels, etc. Build buffer and image descriptors in SGPRs from that address. -The FLOAT_MODE of the dispatch is set with `ieee` (0/1), `denorm32` and `denorm16` (0-3, 3 = denormals kept). Float -atomics need `coarse`: on fine-grained system memory they do nothing. +Every run must explicitly set all floating-point controls, including runs of integer instructions. The CLI flags +use hyphens; Python keywords use underscores. Record these settings alongside the GPU and measured results. + +| Python keyword | Values | +| --- | --- | +| `ieee` | 0 or 1: IEEE mode disabled or enabled | +| `dx10_clamp` | 0 or 1: DX10 clamp disabled or enabled | +| `denorm32`, `denorm16` | 0: flush input/output; 1: preserve input, flush output; 2: flush input, preserve output; 3: preserve both | +| `round32`, `round16` | 0: nearest even; 1: toward +infinity; 2: toward -infinity; 3: toward zero | +| `fp16_overflow` | 0: overflow to infinity; 1: clamp computed overflow to the largest finite value (infinite inputs and division by zero still produce infinity) | + +`denorm16` and `round16` control both f16 and f64. These values follow the +[LLVM AMDGPU kernel descriptor documentation](https://llvm.org/docs/AMDGPUUsage.html#amdhsa-kernel-descriptor). +Float atomics need `coarse`: on fine-grained system memory they do nothing. ## Use @@ -40,7 +52,9 @@ cat > body.s <<'EOF' v_add_f16 v10, v4, v5 EOF printf '0x3c00 0x3c00 0 0\n0x7e01 0x3c00 0 0\n' > rows.txt -python3 tools/hw-oracle/hw_oracle.py body.s rows.txt --outs 1 --ieee 0 +python3 tools/hw-oracle/hw_oracle.py body.s rows.txt --outs 1 \ + --ieee 0 --dx10-clamp 1 --denorm32 0 --denorm16 3 \ + --round32 0 --round16 0 --fp16-overflow 0 ``` prints one line per row with the output dwords in hex. From Python: @@ -51,7 +65,8 @@ sys.path.insert(0, "tools/hw-oracle") from hw_oracle import run rows = [(a, b, 0, 0) for a in values for b in values] -results = run(body, rows, wave64=True, denorm32=0) +results = run(body, rows, wave64=True, ieee=0, dx10_clamp=1, + denorm32=0, denorm16=3, round32=0, round16=0, fp16_overflow=0) ``` Usual workflow: run the instruction on rows that separate the candidate models (edge values, NaN payloads, rounding @@ -62,3 +77,9 @@ ties), write a model that matches every row, implement it, and put measured rows Only compute kernels. Pixel-shader ops that read their inputs from LDS also work: `v_interp_*` reads the attribute parameters from LDS at `M0`, so writing them there by hand is enough. Behaviour that depends on the graphics pipeline (parameter cache, exports, rasterization, `SPI_*` state) can't be measured with it. + +Argument handling can be tested without a GPU: + +```sh +python3 tools/hw-oracle/test_hw_oracle.py +``` diff --git a/docs/dev/PROGRESS.md b/docs/dev/PROGRESS.md new file mode 100644 index 000000000..36d9a1e08 --- /dev/null +++ b/docs/dev/PROGRESS.md @@ -0,0 +1,26 @@ +# Progress reporting + +[`tools/progress.py`](../../tools/progress.py) generates `progress.json`, HTML, SVG maps and badges. The [Progress workflow](../../.github/workflows/progress.yml) publishes them after pushes to `main`. The [Progress report workflow](../../.github/workflows/progress-report.yml) compares the base and head trees of pull requests. + +## Library counts + +The scanner reads `APS5_VABI` function definitions under `core/libs/prx//`, excluding `tests` subdirectories and their descendants. Test callbacks do not contribute to library counts. A definition is pending when its body calls `NotImplemented_nid_no_patch`, directly or through a recognized local stub wrapper. Other definitions are counted as implemented. These counts describe declared functions, not every firmware export, verified ABI compatibility or playable games. Silent behavior and unverified assumptions remain in [Technical debt](TechnicalDebt.md). + +A library with no scanned definitions can include a sibling library's source builder. The scanner recognizes literal `include(${CMAKE_CURRENT_SOURCE_DIR}/..//.cmake)` references, including quoted paths. It does not evaluate arbitrary CMake expressions. + +For these wrappers, the JSON group adds `shared_sources` with the source library's directory name. Its own counts and name lists remain empty. The HTML table links to the source library instead of showing `0/0`; the source library's map tooltip lists its wrappers. Global totals count the definitions at the source library once. + +## Comparisons + +Comparisons identify functions by library and name. When a head group becomes a shared-source wrapper, its previous definitions are compared under the source library's identity. Removing a duplicate implementation is reported as sharing sources, without reporting the retained functions as removed. + +Real removals and transitions back to stubs remain visible. Percentages retain the original snapshot totals, so eliminating duplicate accounting can change a percentage without changing available functionality. Older JSON snapshots without `shared_sources` are accepted. + +## Verification + +```sh +python3 tools/tests/test_progress.py +python3 tools/progress.py /tmp/anyps5-progress +``` + +The regressions check that adding test callbacks leaves counts and comparisons unchanged, including when a callback shares a pending function's name or a wrapper uses shared sources. They also create a source library and a copied wrapper, move the wrapper to a shared builder, check output labels and totals, then verify real regressions and removals are still reported. With `BUILD_TESTING=ON` and Python available, CTest registers them as `progress_report`. diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 05c9990bb..f93041a74 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -11,48 +11,68 @@ ### Silent stubs Throughout the project, every function at every stage either **does exactly what it's supposed to or throws an exception**. Everywhere... except: -- [libSceSaveDataDialog.native](../../core/libs/prx/libSceSaveDataDialog.native/Export.cpp) +- [libSceSaveDataDialog.native](../../core/libs/prx/libSceSaveDataDialog.native/Export.cpp) shows no UI and ignores progress values. Progress-bar mode (5) stays running until the caller closes it, matching Skyrim PPSA03747's save sequence; other modes still finish immediately and return button 1 and the first nonempty directory name. Console dialog timing and selection behavior remain unverified. - [libSceMsgDialog.native](../../core/libs/prx/libSceMsgDialog.native/Export.cpp) - dialogs finish at `sceMsgDialogOpen` without UI and answer with button 1, so `sceMsgDialogClose` returns `NOT_RUNNING`; its result before `sceMsgDialogInitialize` follows libSceMsgDialog - [libSceCommonDialog](../../core/libs/prx/libSceCommonDialog/Export.cpp) - [libSceErrorDialog](../../core/libs/prx/libSceErrorDialog/Export.cpp) - the error dialog runs the state machine but shows nothing; the error code passed to `sceErrorDialogOpen` is only logged - [libSceLoginDialog](../../core/libs/prx/libSceLoginDialog/Export.cpp) - the login dialog runs the common-dialog state machine but shows nothing and reports the user as having cancelled - [libSceHmd](../../core/libs/prx/libSceHmd/Export.cpp) implements only the disconnected-headset path: initialization succeeds, device queries report `NotDetected`, and opening a device returns `DeviceDisconnected`. Headset support, tracking, rendering and additional HMD exports are not implemented; SDK-level ABI compatibility and in-game behaviour remain unverified. - [libSceHmd2](../../core/libs/prx/libSceHmd2/Export.cpp) has no PS VR2 runtime: `sceHmd2Initialize` fails, and every other export it has (and every [libSceVrTracker2](../../core/libs/prx/libSceVrTracker2/Export.cpp) export) throws. PPSA01317 skips its whole VR set-up when initialization fails +- [libSceMbus](../../core/libs/prx/libSceMbus/Export.cpp) implements only the no-device path: initialization and termination succeed, no device events are reported and device queries return no entries - [libSceAudioPropagation](../../core/libs/prx/libSceAudioPropagation/Export.cpp) implements only the no-acoustic-propagation path: the system keeps a 16-byte header in the title's CPU memory (no GPU memory), rooms, portals, sources and materials are tracked handles, attributes are validated and have no effect, no rays are requested and sources report no audio paths. Ray results, path calculation and audio paths are accepted only when empty and throw otherwise; `sceAudioPropagationSourceRender` writes silence (zeroes) to each source's output buffer for the given size, without a dry signal. Objects still alive when their system is destroyed become unusable, and their later destroy or unregister is a no-op: PPSA21567 releases its reference-counted room, portal and source owners just before `sceAudioPropagationSystemDestroy`, so the order for every object is not proven. Occlusion, reflections and reverb from level geometry are absent; in-game behaviour remains unverified. - [libSceNpCommerce](../../core/libs/prx/libSceNpCommerce/Export.cpp) - the PS Store icon show/hide calls and `sceNpCommerceSetPsStoreIconLayout` do nothing -- [libSceSystemService](../../core/libs/prx/libSceSystemService/Export.cpp) - `sceSystemServicePowerTick`, `sceSystemServiceReportAbnormalTermination`, `sceSystemServiceDisableMusicPlayer` and `sceSystemServiceReenableMusicPlayer` return success and do nothing: the host has no auto power-off timer to reset, no system crash reporter to notify and no system music player to pause or resume +- [libSceUsbd](../../core/libs/prx/libSceUsbd/Export.cpp) implements only the no-device path: `sceUsbdInit` and `sceUsbdExit` succeed without a host USB context, the device list is always empty and event handling waits for its timeout; calls that need a device throw +- [libSceSystemService](../../core/libs/prx/libSceSystemService/Export.cpp) - `sceSystemServicePowerTick`, `sceSystemServiceReportAbnormalTermination`, `sceSystemServiceDisableMusicPlayer`, `sceSystemServiceReenableMusicPlayer`, `sceSystemServiceDisableMediaPlay` and `sceSystemServiceReenableMediaPlay` return success and do nothing: the host has no auto power-off timer to reset, no system crash reporter to notify, no system music player to pause or resume and no system media playback to disable. PPSA01521 calls `sceSystemServiceDisableMediaPlay` during start-up, and its wrappers for both pass no arguments +- [libSceBluetoothHid](../../core/libs/prx/libSceBluetoothHid/Export.cpp) implements only the no-device path: initialization and callback and device registration succeed, the callback is never called because no device connects, and calls that need a connected device throw - [sceVideoOutOpen](../../core/libs/prx/libSceVideoOut/src/Output.cpp) validates the priority and CPU affinity that the open param requests for the VideoOut service thread but does not apply them: the port's present and vblank threads are host threads, and guest priorities and affinities do not reach host scheduling - Without `VK_KHR_fragment_shader_barycentric` the shader recompiler [does not load the barycentric VGPRs](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp): [`v_interp_p1_f32` is dropped and `v_interp_p2_f32` reads the attribute Vulkan interpolated](../../core/shader/recompiler/Translation/src/AttributeInstructions.cpp), so pixel shaders that use the barycentrics for anything other than `v_interp` get wrong values. - [libSceAudio3d](../../core/libs/prx/libSceAudio3d/Export.cpp) - the port produces no sound: objects can be reserved but the library has no bed exports or `sceAudio3dObjectSetAttributes` to give them audio, so `sceAudio3dPortPush` only paces the queue at `granularity` samples per frame at 48 kHz. `sceAudio3dPortSetAttribute` accepts the late reverb level and the downmix spread attributes and ignores them +- [libSceTextToSpeech2](../../core/libs/prx/libSceTextToSpeech2/Export.cpp) has no speech engine: `sceTextToSpeech2Initialize` fails and the other exports throw. PPSA01317 skips its narration set-up when initialization fails - [libSceAvPlayer](../../core/libs/prx/libSceAvPlayer/Export.cpp): `sceAvPlayerSetLogCallback` accepts a callback that is never called, as the player produces no log messages, and `sceAvPlayerSetAvailableBandwidth` has no effect, as it governs HLS sources, which `sceAvPlayerAddSource` does not implement. - [libSceHttp2](../../core/libs/prx/libSceHttp2/Export.cpp) - no request reaches the network, so `sceHttp2SetMinSslVersion` has no TLS connection to restrict and returns success for any `id`, like `sceHttp2SslEnableOption` (prosper returns the invalid-id error for an unknown one) - [libSceNpTrophy2](../../core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp): `sceNpTrophy2RegisterUnlockCallback` accepts a callback that is never called, as no trophy is ever unlocked, and `sceNpTrophy2UnregisterUnlockCallback` only returns success. - [sceAvPlayerSetTrickSpeed](../../core/libs/prx/libSceAvPlayer/src/Source.cpp) (libSceAvPlayer) with a negative speed runs the clock backwards but delivers no frames; when a forward speed is set again, playback resumes from the rewound time. - [ulobjmgr](../../core/libs/prx/ulobjmgr/Export.cpp) registers no object: `_sceUlobjmgrRegisterObject` always hands out id 0 and `_sceUlobjmgrUnregisterObject` releases nothing, as shadPS4 does -- [libSceHttp](../../core/libs/prx/libSceHttp/Export.cpp) - no request reaches the network, so `sceHttpSetResponseHeaderMaxSize` has no response header to limit and `sceHttpRedirectCacheFlush` no redirect to forget; `sceHttpsUnloadCert` returns success like `sceHttpsLoadCert`, which keeps no certificate +- [libSceHttp](../../core/libs/prx/libSceHttp/Export.cpp) - no request reaches the network, so `sceHttpSetResponseHeaderMaxSize` has no response header to limit and `sceHttpRedirectCacheFlush` no redirect to forget; `sceHttpsUnloadCert` returns success like `sceHttpsLoadCert`, which keeps no certificate; `sceHttpSetRedirectCallback` and `sceHttpSetCookieRecvCallback` accept callbacks that are never called, `sceHttpsSetSslVersion` has no TLS connection to configure and `sceHttpsGetSslError` reports error 0 with detail 0; `sceHttpSetAuthInfoCallback` accepts a callback that is never called, as no server asks for authentication +- [sceFontGlyphDefineAttribute](../../core/libs/prx/libSceFont/src/Render.cpp) (libSceFont) checks the glyph and ignores the attribute and its value, whose meaning is unknown. PPSA01325 sets attribute 0x11 to 0 on a glyph from `sceFontGenerateCharGlyph` that it deletes unused, and draws the character with `sceFontRenderCharGlyphImageHorizontal` ### Unknown function info +- [sceNpEntitlementAccessGetEntitlementKey](../../core/libs/prx/libSceNpEntitlementAccess/Export.cpp) - the three-argument signature, 16-byte key and errors `0x817D0002`/`0x817D0007` follow [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/main/src/libs/libNet.cpp); PPSA01521 passes a service label, entitlement label and output pointer. Only the no-key path is implemented: local entitlement labels carry no keys, and the output is untouched on failure. Console error precedence and key provisioning are unverified +- [sceNpEntitlementAccess{Request,Poll}{Unified,Service}EntitlementInfoList](../../core/libs/prx/libSceNpEntitlementAccess/Export.cpp) - signatures unknown; no reference implements them, so they are declared without parameters. Like the consume requests in the same library, they report the signed-out NP state with `SCE_NP_ERROR_SIGNED_OUT` (`0x80550006`) and write nothing. PPSA15657 calls the unified request during start-up and continues past the error. Whether the console fails at the request or at the poll when signed out is unverified +- [sceRudpGetStatus](../../core/libs/prx/libSceRudp/Export.cpp) - PPSA01521 passes an output pointer and byte count `0xF8`; the status layout, size rules and error behavior are unknown. The export throws without writing output, including after initialization. [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/rudp/rudp.cpp) declares it without arguments and returns success without writing status +- [sceHttpSetCookieEnabled](../../core/libs/prx/libSceHttp/Export.cpp) - the two-argument signature and 0/1 enable values follow [VitaSDK](https://github.com/vitasdk/vita-headers/blob/master/include/psp2/net/http.h) and [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.cpp); PPSA01521 passes 0 to disable cookies. Disabling succeeds because the offline HTTP backend has no cookie engine; enabling throws. Invalid enable values use the same error as `sceHttpSetInflateGZIPEnabled`. Like the existing offline HTTP setters, the handle is not validated; requests still fail with the network error. Console behavior is unverified + - [sceHmd2Initialize](../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2) - the error the console returns when the PS VR2 runtime is unavailable is unknown; it returns libSceHmd's `UNSUPPORTED_FEATURE` (0x81110016). PPSA01317 passes a pointer to 16 zeroed bytes and only tests the result for non-zero - PPSA01317 imports declared without parameters, signatures unknown: the remaining [libSceHmd2](../../core/libs/prx/libSceHmd2/Export.cpp) and [libSceVrTracker2](../../core/libs/prx/libSceVrTracker2/Export.cpp) exports. Names from the aerolib NID list +- [libSceMbus](../../core/libs/prx/libSceMbus/Export.cpp) (`libSceDeviceService`) - signatures taken from PPSA01317, which calls `sceDeviceServiceInitialize(3, 0)`, polls `sceDeviceServiceGetEventState(1)` and queries again when bit 0 or 1 is set, and calls `sceDeviceServiceQueryDeviceInfo_(0x7001, 0, 0, infos, 1, &count, &unknown, 0x70)`, treating a negative result as an error and `count` as the number of entries. The host has no devices: the event state is 0 and a query returns 0 with `count` 0 +- [NGS2 ATRAC9 waveform loops](../../core/libs/prx/libSceNgs2.native/src/Atrac9.cpp) accepts one integer forward `smpl` loop. Inclusive RIFF endpoints and encoder-delay adjustment follow [vgmstream riff.c](https://github.com/vgmstream/vgmstream/blob/7dc938fa2f210943b37c7b6511852b516ef432ab/src/meta/riff.c). PPSA03747 supplies a 60-byte sampler chunk with one loop, `samplerData` 24, start 256, inclusive end 299224 and delay 256 for 298969 audible samples. The chunk size and loop count bound the records; `samplerData` is not interpreted. Output positions are assumed to use audible samples and an exclusive end. Blocks split into an optional intro, a repeated range and an optional tail; zero play count means infinite repeats, positive play count includes the first pass. ATRAC9 ranges include a preceding codec frame for decoder history. The output block layout, repeat-count mapping and error codes are unverified on hardware. Multiple, reverse, alternating and fractional loops throw. +- [NGS2 submixer UserFx](../../core/libs/prx/libSceNgs2.native/src/Custom.cpp) (control `0x20000004`) invokes the title's callback after input mixing on every playing grain, including silence. The 40-byte control and 48-byte process context follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/945dbc3cc3eee80ac3e053b438502ed936fa6bb2/src/core/libraries/ngs2/ngs2_submixer.h). PPSA03747 installs a peak-meter callback; its code confirms the channel pointers, user data 0, channel count, grain size and sample-rate offsets and returns 0. Other user data words are preserved. Flags are assumed to be 1 after registration/restart and 0 thereafter, following UserFx2's first-process convention; this callback does not inspect them. Null handlers disable the effect, setup clears it, and nonzero callback results throw. These lifecycle and error semantics are unverified on console hardware. - PPSA01341 imports declared without parameters, signatures unknown: [sceAgcSetSemaphoreMemory](../../core/libs/prx/libSceAgc/Unimplemented.cpp), [sceAgcDriverRegisterMultipleResources](../../core/libs/prx/libSceAgcDriver/Unimplemented.cpp). Names from the shadPS4 aerolib NID list - [sceAppContentDownloadDataGetAvailableSpaceKb](../../core/libs/prx/libSceAppContent/Export.cpp) (libSceAppContent) - a title whose `param.json` declares `downloadDataSize` 0 gets 0 KB and OK; the console's behaviour for a title without a download data area is unverified (it may return an error instead). KytyPS5 does not special-case it, and shadPS4 stubs 1048576 KB - [7CxI50-xlCk, pMxXhNozUX8](../../core/libs/prx/libSceNpPartner001/Export.cpp) (libSceNpPartner001) - unknown names and signatures, imported by PPSA23566; declared without parameters +- [libSceBluetoothHid](../../core/libs/prx/libSceBluetoothHid/Export.cpp) (libSceBluetoothHid) - signatures taken from PPSA01317: `sceBluetoothHidInit()` and `sceBluetoothHidUnregisterCallback()` without arguments, `sceBluetoothHidRegisterCallback(callback, 0, param)` with a 16-byte param, and `sceBluetoothHidRegisterDevice` / `sceBluetoothHidUnregisterDevice(vendorId, productId)`; the report and device name exports are declared without parameters - [sceAgcWaitRegMemPatchMask](../../core/libs/prx/libSceAgc/Patch/src/WaitRegMem.cpp) (libSceAgc) - follows `sceAgcWaitRegMemPatchReference`: the mask is taken as a 32-bit value and written to the low mask word of a 32- or 64-bit wait, as [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) does; [sharpemu](https://github.com/sharpemu/sharpemu/blob/main/src/SharpEmu.Libs/Agc/AgcExports.PacketPatching.cs) writes both words of a 64-bit wait's mask. A mask above 32 bits throws -- [0GAw7SmkwII, 1ic5q-kdOsc, 5z2gBlqxJ+0, Kyy1baXgaVU, ZLL31lzzxr4, gMduXCLYrNg, lrJwpLjKXRc, m9JLPc3wOQw, o+NM86gEwFE](../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml_mfsr2) - unknown names and signatures, imported by PPSA23566; declared without parameters +- [scePsmlMfsr2CreateContext, scePsmlMfsr2CreateSharedResources, scePsmlMfsr2GetContextInitRequirement, scePsmlMfsr2GetDispatchPackets, scePsmlMfsr2GetDispatchPacketsSizeInDwords, scePsmlMfsr2GetSharedResourcesInitRequirement, scePsmlMfsr2Init, scePsmlMfsr2ReleaseContext, scePsmlMfsr2ReleaseSharedResources](../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml_mfsr2) - unknown signatures, imported by PPSA23566; declared without parameters - [sceAgcDcbSetCfRegisterDirect, sceAgcDcbSetCfRegisterRangeDirect](../../core/libs/prx/libSceAgc/DcbState/src/Registers.cpp) (libSceAgc) - signatures follow [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) and [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Agc/SceAgc.cs), which agree; the SET_CONFIG_REG (0x68) packet is OpenAGC's. The driver does not execute SET_CONFIG_REG, so a submitted packet throws -- PPSA23566 imports declared without parameters, signatures unknown: [sceAudioOut2UserGetSupportedAttributes](../../core/libs/prx/libSceAudioOut/src/AudioOut2System.cpp), [sceConvertKeycodeGetCharacterFromKeyboardData](../../core/libs/prx/libSceConvertKeycode/Export.cpp), [sceHttpSetCookieRecvCallback, sceHttpSetRedirectCallback, sceHttpsGetSslError, sceHttpsSetSslVersion](../../core/libs/prx/libSceHttp/Export.cpp), [sce::Json::MemAllocator::notifyError, sce::Json::Value::clear](../../core/libs/prx/libSceJson2/Export.cpp), [sceNetResolverAbort](../../core/libs/prx/libSceNet/Export.cpp), [sceNpCommerceDialogOpen2](../../core/libs/prx/libSceNpCommerce/Export.cpp), [sceNpEntitlementAccessGetPftFlag](../../core/libs/prx/libSceNpEntitlementAccess/Export.cpp), [sceProprietaryVoiceChatHelper{Initialize,Terminate,SetVoiceChatState,GetVoiceChatUsageState}](../../core/libs/prx/libSceProprietaryVoiceChatHelperPrx/Export.cpp), [sceVoice{GetResourceInfo,ResetPort,EnableChat,DisableChat}](../../core/libs/prx/libSceVoice/Export.cpp). Names from the shadPS4 aerolib NID list +- [scePadOpenExt](../../core/libs/prx/libScePad/Export.cpp) (libScePad) - only special ports (type 2) are handled, and they return `DEVICE_NOT_CONNECTED` (0x80920007) because the host has no special controller; whether the console opens a handle and reports the device as disconnected later is unknown. PPSA01317 passes a 16-byte param that names the device by vendor and product id (`b7 0e 04 0e ...` for 0x0eb7:0x0e04) and skips the user on any negative result. Other port types throw +- PPSA23566 imports declared without parameters, signatures unknown: [sceAudioOut2UserGetSupportedAttributes](../../core/libs/prx/libSceAudioOut/src/AudioOut2System.cpp), [sceConvertKeycodeGetCharacterFromKeyboardData](../../core/libs/prx/libSceConvertKeycode/Export.cpp), [sce::Json::MemAllocator::notifyError](../../core/libs/prx/libSceJson2/Export.cpp), [sceNetResolverAbort](../../core/libs/prx/libSceNet/Export.cpp), [sceNpCommerceDialogOpen2](../../core/libs/prx/libSceNpCommerce/Export.cpp), [sceNpEntitlementAccessGetPftFlag](../../core/libs/prx/libSceNpEntitlementAccess/Export.cpp), [sceProprietaryVoiceChatHelper{Initialize,Terminate,SetVoiceChatState,GetVoiceChatUsageState}](../../core/libs/prx/libSceProprietaryVoiceChatHelperPrx/Export.cpp), [sceVoice{GetResourceInfo,ResetPort,EnableChat,DisableChat}](../../core/libs/prx/libSceVoice/Export.cpp). Names from the shadPS4 aerolib NID list +- PPSA01325 imports declared without parameters, signature unknown: [sceRudpActivate](../../core/libs/prx/libSceRudp/Export.cpp). Name from the shadPS4 aerolib NID list +- [sceVoiceChatRequestCreateVoiceChatGroup, sceVoiceChatRequestJoinVoiceChatGroup, sceVoiceChatRequestLeaveVoiceChatGroup, sceVoiceChatRequestDeleteVoiceChatGroup](../../core/libs/prx/libSceVoiceChat/Export.cpp) (libSceVoiceChat) - assumed to take a request id and two pointers like the other `sceVoiceChatRequest*` calls; offline, every argument is ignored and `SIGNED_OUT` is returned, as for those - [sceAvPlayerAddSource](../../core/libs/prx/libSceAvPlayer/src/Source.cpp) (libSceAvPlayer) - requests up to 4 additional decode-ahead video framebuffers from the memory replacement beyond `num_output_video_framebuffers` when memory is available; the console's internal buffering behaviour is unknown +- [sceTextToSpeech2Initialize](../../core/libs/prx/libSceTextToSpeech2/Export.cpp) (libSceTextToSpeech2) - the console's error for an unavailable speech engine is unknown; it returns the kernel's `EOPNOTSUPP` (0x8002002D). PPSA01317 passes a param that starts with 0x2000000 and only tests the result for non-zero; the other exports it imports are declared without parameters - [sceAgcDcbClearState](../../core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp) (libSceAgc) - the (buffer, command) signature follows [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Agc/SceAgc.cs) and [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c), and the 2-dword CLEAR_STATE (0x12) packet is OpenAGC's; OpenAGC masks the command to 4 bits, here a command above 0xf throws, as the driver rejects that payload +- [Command buffer cursors](../../core/libs/prx/libSceAgc/Command/src/Packet.cpp) (libSceAgc) - a null `cursor_down` is taken as a buffer without downward allocations, whose limit is `top`. PPSA16064 builds one-packet buffers of 8 dwords that way; the console's handling is unconfirmed - [sceVideoOutRegisterBuffers2](../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - buffer option 8 (STRICT_COLORIMETRY) is presented as option 0, with the stored values unchanged; what the console changes for it is unknown. Only options 0 and 8 are accepted - [sceAgcGetRegisterDefaultsInternal](../../core/libs/prx/libSceAgc/Misc/src/RegisterDefaults.cpp) (libSceAgc) - returns the version 0 internal table, as `sceAgcGetRegisterDefaults` returns the version 0 public one; [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) returns the table of the version passed to `sceAgcInit` instead, and null before it - [sceVideoOutRegisterBuffers2](../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - DCC buffers (category 1): `dcc_control` is taken as the CB_DCC_CONTROL block layout, only bits 0x10026c are accepted; `dcc_cb_register_clear_color` as the register-clear texel, only 32-bit values are presented - [sceVideoOutIsOutputSupported](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - output mode encoding unknown: every mode except the default reports not supported (0), and `sceVideoOutConfigureOutput` returns `VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE` for it, instead of returning `VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE` for values the console does not define. PPSA12544 probes `0xd000000a` and selects 7680 pixels when it is supported, 3840 otherwise - [sceVideoOutOpen](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - the open param's first word is unknown (PPSA21564 passes 16; it is not the byte size, since the affinity mask is at offset 16); only 16 is accepted. Whether the param continues past offset 24 is also unknown - [s_setkill](../../core/shader/recompiler/RdnaDecoder/src/RdnaScalarOpDecoder.cpp) (core/shader/recompiler) - decoded as `s_endpgm` for any immediate: measured on RDNA2 hardware (gfx1035), a store before it is visible and nothing after it runs ([#771](https://github.com/boykopovar/AnyPS5/pull/771)); whether the KILL status bit it sets is observable differently from `s_endpgm` is unverified -- [sceRtcParseDateTime](../../core/libs/prx/libSceRtc/Export.cpp) (libSceRtc) - accepted format assumed to be the ISO 8601 subset `YYYY-MM-DDTHH:MM:SS[.ffffff]` (T/t/space separator, optional fractional seconds) with the sibling RFC 3339 zone suffix (`Z` or `±HH:MM`), plus the RFC 2822 shape `Ddd, DD Mon YYYY HH:MM:SS[ GMT| ±HHMM]` and the asctime shape `Ddd Mon DD HH:MM:SS YYYY` (day zero- or space-padded) that shadPS4 parses at fixed positions; the weekday name is not checked against the date, a `±HHMM` offset is subtracted to get UTC like the RFC 3339 path (shadPS4 adds it), a string without a zone is taken as UTC, and anything outside these shapes throws instead of returning a parse error, unverified against hardware +- [sceRtcParseDateTime](../../core/libs/prx/libSceRtc/Export.cpp) (libSceRtc) - the console's zone name table is only known from the names measured on a PS5 (firmware 13.60); any other name is assumed to take the military letter offset of its first character, as every measured name outside the table does (`HST`, `MSK`, `XYZ`), so a name the console's table holds but nobody measured gets the wrong offset - [sceAudio3dPortOpen](../../core/libs/prx/libSceAudio3d/Export.cpp) (libSceAudio3d) - buffer modes 0 and 1 (also selected by the 0x10 and 0x18 parameter sizes) and 3 beds depend on an SDK version check in the module ([prosper](https://github.com/mattias800/prosper)) and throw. A repeated `sceAudio3dInitialize` (`NOT_READY` in shadPS4, 0 in prosper) and `sceAudio3dPortAdvance` on a full queue (`NOT_READY` in shadPS4, unhandled in KytyPS5) throw too - [sceAudio3dObjectReserve](../../core/libs/prx/libSceAudio3d/Export.cpp) (libSceAudio3d) - object ids, the `max_objects` limit and the checks of `sceAudio3dObjectReserve` and `sceAudio3dObjectUnreserve` follow [shadPS4](https://github.com/shadps4-emu/shadPS4) (PS4), the only reference with a body: ids count up from 1 per port, skipping 0, 0xFFFFFFFF and ids in use, and a failed reserve writes 0xFFFFFFFF +- [s_sethalt](../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) (core/shader/recompiler) - with SIMM16[0] clear it is a no-op: the RDNA 2 ISA guide describes that value as resuming a halted wave, and a wave that executes it is not halted; measured on a gfx1035 (RDNA 2) wave: `s_sethalt 0` followed by a VALU op completes and gives the normal result. Only SIMM16[0] is read; with it set the instruction throws - [AMPR WriteAddressFromCounterPairOnCompletion](../../core/libs/prx/libSceAmpr/Export.cpp) (libSceAmpr) - the pair of an even counter n is assumed to give counter n in the low 32 bits and n + 1 in the high 32 bits. Wait compare values, wait flush and counter index ranges follow the [ampr_emu](https://github.com/drakmor/ampr_emu) reimplementation; the flush is accepted and has no effect, as commands run in order - [AMPR command sizes](../../core/libs/prx/libSceAmpr/Export.cpp) (libSceAmpr) - commands use the emulator's own encoding with an 8-byte header, so `Nop(n)` takes 8 + 4n bytes where the console record is n dwords including its header (`NopWithData(n)`: (n + 1) dwords). The measure functions agree with the appended sizes, but a title that pads to an exact offset or the end of the buffer with `Nop(remaining / 4)` gets `SCE_AMPR_ERROR_BUFFER_FULL` or a shifted offset. The at-start flag of the `_04_00` writes is ignored; commands run in order, each after the previous one completes - [AMPR counter access](../../core/libs/prx/libkernel/Apr/src/Apr.cpp) (libSceAmpr) - signatures and checks of `WaitOnCounter_04_00`, `WriteCounter_04_00`, `ConstructNop` and `ConstructMarker` taken from the [ampr_emu](https://github.com/drakmor/ampr_emu) reimplementation, which hands counter commands to the console and does not show their effect. The counters are taken as 32 bits each: the 8-byte access of counter n spans n (low) and n + 1 (high), the 2- and 1-byte accesses select a half or a byte of the counter by its offset, waits compare at the access width (signed and wrapped compares included), the AND mask applies to both sides and the atomic writes wrap within the access. The `ConstructNop` type is not recorded @@ -65,11 +85,13 @@ Throughout the project, every function at every stage either **does exactly what - [AMPR markers](../../core/libs/prx/libSceAmpr/Export.cpp) (libSceAmpr) - signatures taken from the [ampr_emu](https://github.com/drakmor/ampr_emu) reimplementation, not confirmed on a title: `sceAmprCommandBufferSetMarkerWithColor` takes the color by pointer, the push and measure variants by value. Colors are not recorded - [sceFontGetRenderScaledKerning](../../core/libs/prx/libSceFont/src/Render.cpp) (libSceFont) - arguments, the cleared output and the bound renderer check taken from `cellFontGetRenderScaledKerning` in [RPCS3](https://github.com/RPCS3/rpcs3/blob/master/rpcs3/Emu/Cell/Modules/cellFont.cpp) (PS3 predecessor of the library), not confirmed on a title. The kerning is the `sceFontGetKerning` one at the render scale - [sceFontGetPixelResolution](../../core/libs/prx/libSceFont/src/Library.cpp) (libSceFont) - signature taken from the IDA type database of [Orbital](https://github.com/AlexAltea/orbital/blob/master/tools/ida/db_types.json), not confirmed on a title. The value is the one the library's driver reports (64 for FreeType); the argument checks and the cleared output follow `sceFontGetLibrary` +- [sceFontDettachDeviceCacheBuffer](../../core/libs/prx/libSceFont/src/Library.cpp) (libSceFont) - signature taken from the IDA type database of [Orbital](https://github.com/AlexAltea/orbital/blob/master/tools/ida/db_types.json), not confirmed on a title. It detaches the cache buffer attached by `sceFontAttachDeviceCacheBuffer`, returns the buffer and its size, and reports `SCE_FONT_ERROR_NOT_ATTACHED_CACHE_BUFFER` when no cache buffer is attached - [sceFontGetFontGlyphsCount, sceFontGetCharGlyphCode](../../core/libs/prx/libSceFont/src/Open.cpp) (libSceFont) - signatures taken from the IDA type database of [Orbital](https://github.com/AlexAltea/orbital/blob/master/tools/ida/db_types.json) and matching the [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Font/SceFont.Engine.cs) bindings, not confirmed on a title. The count is the face's glyph count and the glyph code its glyph index, as the library's driver reports them; a code with no glyph returns `NO_SUPPORT_GLYPH` and code 0 `NO_SUPPORT_CODE`, as `sceFontRenderCharGlyphImage` does. The argument checks and the cleared output follow `sceFontGetPixelResolution` - [sceFontGetFontResolution](../../core/libs/prx/libSceFont/src/Open.cpp) (libSceFont) - signature taken from the IDA type database of [Orbital](https://github.com/AlexAltea/orbital/blob/master/tools/ida/db_types.json), not confirmed on a title; the outputs are inferred from the names. The resolution is the face's units per EM and the scale pixel the driver's face scale (units per EM / 64 for FreeType), the pixel scale a font has when opened. Either output may be null, as in `sceFontGetResolutionDpi` - [sceFontCreateWritingLine, sceFontWritingLineWritesOrder](../../core/libs/prx/libSceFont/src/WritingLine.cpp) (libSceFont) - signatures and the `SceFontWritingLineStep`/`SceFontCreateWritingLineDetail` layouts (detail id `0x0FD5`) taken from [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Font/SceFontStructs.cs), not confirmed on a title; the meaning of the 64-bit writing attribute is unknown - [sceFontWritingLineGetOrderingSpace](../../core/libs/prx/libSceFont/src/WritingLine.cpp) (libSceFont) - signature taken from the IDA type database of [Orbital](https://github.com/AlexAltea/orbital/blob/master/tools/ida/db_types.json) and the [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Font/SceFont.Engine.cs) bindings, not confirmed on a title. The modelled line applies no spacing, so the head, inline, tail and advance spaces are 0; what spacing the console applies is unknown. A null output throws - [sceSaveDataTransferringMountPs4, sceSaveDataDirNameSearchPs4](../../core/libs/prx/libSceSaveData.native/Export.cpp) (libSceSaveData.native) - arguments assumed to be those of `sceSaveDataTransferringMount` and `sceSaveDataDirNameSearch`: KytyPS5 binds both NIDs to the PS5 functions, and [prosper](https://github.com/mattias800/prosper) captured the same result layout on a title for the search. There is no PS4 save area, so the mount returns `NOT_FOUND` and the search reports zero hits +- [sceCoredumpWriteUserString](../../core/libs/prx/libkernel/System/src/Coredump.cpp) and [sceSaveDataConvert, sceSaveDataGetConvertProgress](../../core/libs/prx/libSceSaveData.native/Export.cpp) - imported by PPSA16064, not called during its start-up and added without an implementation; unknown signatures - [sce::Json::Parser::parse](../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown - [sceUserServiceGetNpAccountId](../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - the account id for a user with no linked PSN account is assumed to be 0 with success; libSceNpManager's `sceNpGetAccountIdA` answers the same question with `SCE_NP_ERROR_SIGNED_OUT` - [sceUserServiceGetUserColor](../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - the default profile colour is assumed to be blue (0) @@ -80,30 +102,32 @@ Throughout the project, every function at every stage either **does exactly what - [User data registers](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Diagnostics.cpp) (AGC driver) - a `SPI_SHADER_USER_DATA_*` or `COMPUTE_USER_DATA_*` register the title never wrote reads as 0 when a stage's RSRC2 count covers it, as Kyty (zeroed `UserSgprInfo`) and SharpEmu (missing registers read 0) do, and as AMD PAL and Mesa initialize them; the console's value is not confirmed - [SET_PREDICATION](../../core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp) (AGC driver) - boolean predication follows the AMD PM4 encoding [Mesa](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/common/sid.h) uses: operation 3 reads 64 bits and 4 reads 32 bits, and bit 8 runs predicated packets when the value is non-zero (clear: when it is zero); the bit 12 hint is ignored. The address must have its low 4 bits clear, as `sceAgcDcbSetPredication` writes it. Query predication (Z-pass, primitive count), a predicated COND_EXEC, predicated command buffer chains and flips in predicated command buffers throw - [sceAgcGetGsOversubscription](../../core/libs/prx/libSceAgc/Shader/src/GsOversubscription.cpp) (libSceAgc) - signature and occupancy formula (base and expanded vertex/export capacities, budget shift, `GE_PC_ALLOC` / `SPI_SHADER_PGM_RSRC4_GS` encoding) follow KytyPS5 only, not confirmed on hardware or a title; a negative or NaN factor that gives a negative target, a missing GS context register or a zero `GE_MAX_OUTPUT_PER_SUBGROUP` throws +- [Vertex entry merged wave info](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) - `s3` of every vertex-path shader is set to `waveSize | waveSize << 8` (vertex and primitive count of a full wave, wave index 0). Layout taken from the prologue Mesa's ACO emits for NGG vertex shaders (`s_bfe_u64 exec` from `s3`), not measured on hardware. The driver gives every graphics program user data from `s8`, so the condition does not tell NGG from legacy vertex shaders: they just never read `s3`. A draw whose last wave is partial still reports a full wave. - [sceAgcCbCondWrite](../../core/libs/prx/libSceAgc/Cb/src/Branch.cpp) (libSceAgc) - argument order and the 9-dword COND_WRITE layout follow the clean-room [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) and [sharpemu](https://github.com/sharpemu/sharpemu/blob/main/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs), which agree on them; SharpProspero lists another order. The write space is accepted only as 1 (memory), the one value both encode the same way, and other values throw. - [sceAgcGetIsTrinityMode](../../core/libs/prx/libSceAgc/Misc/src/Platform.cpp) (libSceAgc) - signature from the only PPSA26344 call (a pointer to a one-byte flag, result ignored); a null pointer throws because its error code is unknown - [zARR5aCmkoY](../../core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp) (libSceAgc) - unknown name, signature -- [qj7QZpgr9Uw](../../core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp) (libSceAgc) - unknown name -- [sceAgcDcbContextStateOpGetSize](../../core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp) (libSceAgc) - sizes from KytyPS5 only (5, 27, 27 and 32 dwords), equal to what `qj7QZpgr9Uw` writes; KytyPS5 returns 0 for an operation above 3, here it throws +- [sceAgcDcbContextStateOpGetSize](../../core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp) (libSceAgc) - sizes from KytyPS5 only (5, 27, 27 and 32 dwords), equal to what `sceAgcDcbContextStateOp_0100` writes; KytyPS5 returns 0 for an operation above 3, here it throws +- [sceAgcBranchPatchSetThenTarget_0300, sceAgcBranchPatchSetElseTarget_0300](../../core/libs/prx/libSceAgc/Unimplemented.cpp) (libSceAgc) - names from their NIDs (`7Wa3aeJgeVU`, `rP5xLdOf26k`), signatures unknown +- [sceAgcUpdateInterpolantMapping_0100](../../core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp) (libSceAgc) - name from its NID (`k0E7vkgqAuE`); the `(regs, vs, ps)` arguments are the declared ones, behaviour unknown - [sceAgcWriteDataPatchSetDst, sceAgcWriteDataPatchSetCachePolicy and their Async forms](../../core/libs/prx/libSceAgc/Patch/src/WaitRegMem.cpp) (libSceAgc) - the signatures `(packet, uint8 value)` are inferred from the other patch functions, as no public declaration was found; the value is taken in the encoding of the `dst` and `cachePolicy` arguments of `sceAgcDcbWriteData` (graphics) and `sceAgcAcbWriteData` (Async, compute), and the patched packet is the one those calls would write. `sceAgcAsyncWriteDataPatchSetAddressOrOffset` is assumed to patch the address like `sceAgcWriteDataPatchSetAddressOrOffset`, as the compute WRITE_DATA packet keeps it in the same dwords -- [fd5Bp5tGTgo](../../core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp) (libSceAgc) - unknown name -- [dolOmWH+huQ](../../core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp) (libSceAgc) - unknown name - [dbOlWdppb4o](../../core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp) (libSceAgc) - unknown name; same arguments as `sceAgcCreateInterpolantMapping`; the `SPI_PS_INPUT_CNTL` bits for `is_f16` 2 inputs follow KytyPS5, not confirmed on hardware -- [V++UgBtQhn0](../../core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp) (libSceAgc) - unknown name -- [sceAgcGetDataPacketPayloadRange](../../core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp) (libSceAgc) - signature, the `{base, size in bytes}` output and the type 0 range (header + 1, one dword longer than the type 1 payload) from KytyPS5 only, matching `V++UgBtQhn0`; prosper returns header + 2 for both types from that address function +- [sceAgcGetDataPacketPayloadRange](../../core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp) (libSceAgc) - signature, the `{base, size in bytes}` output and the type 0 range (header + 1, one dword longer than the type 1 payload) from KytyPS5 only, matching `sceAgcGetDataPacketPayloadAddress_0090`; prosper returns header + 2 for both types from that address function - [sceAgcAsyncCondExecPatchSetCommandAddress, sceAgcAsyncCondExecPatchSetEnd](../../core/libs/prx/libSceAgc/Patch/src/CondExec.cpp) (libSceAgc) - no reference has a body for them; they are assumed to patch the COND_EXEC packet like `sceAgcCondExecPatchSetCommandAddress` and `sceAgcCondExecPatchSetEnd`, since `sceAgcAcbCondExec` writes the same packet as `sceAgcDcbCondExec`, as `sceAgcAsyncRewindPatchSetRewindState` does for REWIND -- [gQkqkLttcpw](../../core/libs/prx/libSceAgc/Acb/src/Control.cpp) (libSceAgc) - unknown name, signature +- [sceAgcAcbAtomicGds_0900](../../core/libs/prx/libSceAgc/Acb/src/Control.cpp) (libSceAgc) - name from its NID (`gQkqkLttcpw`), signature unknown - [sceAgcDcbPrimeUtcl2](../../core/libs/prx/libSceAgc/DcbState/src/Display.cpp) (libSceAgc) - assumed to write the packet `sceAgcAcbPrimeUtcl2` writes: PRIME_UTCL2 has the same 5 dwords on the graphics and compute rings. Not confirmed on a title. Both write a NOP of that size, as there is no TLB to prime - [sceAgcDcbQueueEndOfShaderActionGetSize](../../core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp) (libSceAgc) - the builder is inline in the SDK and not exported; the size is assumed to be that of the RELEASE_MEM packet `sceAgcCbReleaseMem` writes (32 bytes), the same as `sceAgcAcbQueueEndOfShaderActionGetSize`, as in [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) and [REmu](https://github.com/xnetcat/REmu/blob/main/crates/remu-runtime/src/agc_dcb.rs) - [sceKernelInternalMemoryGetModuleSegmentInfo](../../core/libs/prx/libkernel/Module/src/Module.cpp) (libkernel) - unknown signature +- [sceKernelGetModuleInfoForUnwind](../../core/libs/prx/libkernel/Module/src/Module.cpp) (libkernel) - `eh_frame_size` is assumed to cover the records up to, not including, the zero terminator. LLVM libunwind's `CFI_Parser::findFDE` stops at the terminator or the length, whichever comes first, so it finds the same FDEs either way; what the console reports is not known. - [sceKernelGetModuleInfoFromAddr](../../core/libs/prx/libkernel/Module/src/ModuleInfo.cpp) (libkernel) - the 0x1A8-byte info layout follows the [shadPS4](https://github.com/shadps4-emu/shadPS4) reimplementation; a title's `libc.prx` reads only `id` (PPSA14632). The meaning of `flags` is unknown; only 2 is accepted, as is only the full `st_size`. The id is the `sceKernelLoadStartModule` handle of an image opened through the loader and otherwise a stable id from the same range. The name is the host file name without `.guest.prx`. An image with more than 4 load segments (the converted executable has 7) reports its first 4. `tls_offset` is 0 and `ref_count` is 1, as the host loader exposes neither. Not implemented on Windows - [sceKernelReleaseFlexibleMemory](../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - unmaps the range like sceKernelMunmap, as fpPS4 and prosper do; whether the console keeps the virtual range reserved after releasing its pages is unverified - [Memory protection bits 0x100 and 0x200](../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - taken as audio coprocessor (ACP) read and write, as in the [KytyPS5](https://github.com/KytyPS5/KytyPS5) and [etaHEN](https://github.com/etaHEN/etaHEN) headers; the coprocessor work runs on the host CPU, so they grant host read and read/write access like the GPU bits - [sceKernelSyncOnAddressWait](../../core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp) (libkernel) - the only known caller passes a null timeout and a name string as the fourth argument; the timeout is assumed to point to microseconds like the other kernel waits, and the name is ignored - [sceKernelSyncOnAddressWait8 / sceKernelSyncOnAddressWait16 / sceKernelSyncOnAddressWait32 / sceKernelSyncOnAddressWait64](../../core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp) (libkernel) - libc's `__std_atomic_wait_direct_8/16/32/64` pass a microsecond timeout or null and ignore the result; the by-value comparand, return values and the 2- and 8-byte alignment of Wait16 and Wait64 are assumed +- [scePthreadCondWait / scePthreadCondTimedwait](../../core/libs/prx/libkernel/Pthread/src/Cond.cpp) (libkernel) - a recursive mutex locked several times is released completely for the wait and relocked to the same depth afterwards, as FreeBSD libthr does (`_mutex_cv_unlock`, `_mutex_cv_lock`); unverified on hardware - [sceKernelAioSubmitReadCommandsMultiple / sceKernelAioSubmitWriteCommandsMultiple / sceKernelAioWaitRequests / sceKernelAioCancelRequest / sceKernelAioCancelRequests / sceKernelAioDeleteRequests](../../core/libs/prx/libkernel/Aio/src/Aio.cpp) (libkernel) - behaviour from shadPS4 (one id per request for the Multiple submits, cancel turns any request aborted and id 0 reports processing, wait mode 2 returns once one request completed); the 128 requests or ids per batch limit is from SharpProspero, with an unknown error, so larger batches throw; wait modes other than 1 (and) and 2 (or) throw; null pointers, negative counts and invalid ids return the codes the single-request siblings use, checked before any state is written - [sceKernelGetAvailableCpumask](../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) - returns the default thread affinity, 0x1FFF (CPUs 0 to 12), as KytyPS5 does. Titles pin threads within it (PPSA02664 passes 0x1FFB, Hades II PPSA36082 0x3 and 0x3F); whether CPU 13 is available to a title is not confirmed -- [sceKernelMapNamedFlexibleMemoryInternal](../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - flag 0x8000 unknown; only the sceKernelMapNamedFlexibleMemory flags are accepted +- [sceKernelGetCurrentCpu](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - guest affinities are not applied to host threads, so the host CPU index is mapped into the calling thread's affinity clipped to `sceKernelGetAvailableCpumask` (the available mask when they do not overlap): host CPU n reports the (n modulo the CPU count)-th CPU of that mask, so a thread pinned to one CPU reports that CPU, as a FreeBSD `cpuset` guarantees on the console. Which CPU the console reports for a thread allowed on several is not known; shadPS4 returns 0 +- [sceKernelMapNamedFlexibleMemoryInternal](../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - flag 0x8000 is accepted and ignored; its meaning is not established. KytyPS5 (`GUEST_MAP_UNKNOWN_8000`) and sharpemu accept it without reading it. Every known caller is the SDK 9 `libc.prx` heap (Astro Bot PPSA21564, PPSA22520), which passes it with read/write protection and uses the mapping as an older `libc.prx` uses `sceKernelMapNamedFlexibleMemory` with flags 0 - [sceKernelMtypeprotect](../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - sets the memory type of the direct mappings in the page-rounded range and of the direct memory behind them, as shadPS4 and prosper do (KytyPS5 ignores the type); the type is not validated, flexible pages in the range only get the protection, and a later map of the retyped direct memory reports the type it was allocated with. `sceKernelBatchMap2` operation 4 takes the same path with the entry type - [Guest arena](../../core/libs/prx/libc/src/GuestArena.cpp) (libc) - the application map area is assumed to end at 0xFC_0000_0000, as `SCE_KERNEL_APP_MAP_AREA_END_ADDR` does on the PS4. On Windows, host allocations made in the arena before libc loads stay, and a fixed guest mapping over one throws - [sceLibcInternalBacktraceForGame](../../core/libs/prx/libc/src/HeapDiagnostics.cpp) (libSceLibcInternal, implemented in libc) - unknown signature @@ -118,29 +142,35 @@ Throughout the project, every function at every stage either **does exactly what - [sceSslGetSerialNumber](../../core/libs/prx/libSceSsl/Export.cpp) (libSceSsl) - unknown signature - [X+4jdIS75P0](../../core/libs/prx/libSceAudioIn/Export.cpp) (libSceAudioIn) - unknown name, signature - [sceAudioOut2Set3DLatency](../../core/libs/prx/libSceAudioOut/src/AudioOut2System.cpp) (libSceAudioOut) - assumed to take `(user_id, latency)`: KytyPS5 reads `(user_id, output, latency_us)`, but the known calls set just edi and esi; only `(0xFF, 2)` (PPSA26344) and `(0xFF, 1)` (PPSA14632) are accepted -- [sceAudioOut2MasteringInit](../../core/libs/prx/libSceAudioOut/src/AudioOut2Mastering.cpp) (libSceAudioOut) - flag values unknown; only 0 is accepted +- [sceAudioOut2MasteringInit](../../core/libs/prx/libSceAudioOut/src/AudioOut2Mastering.cpp) (libSceAudioOut) - flag values unknown; only 0 is accepted. Whether the console reports an error for a repeated `sceAudioOut2MasteringInit` or for `sceAudioOut2MasteringTerm` without a matching initialization is unknown; both return 0. PPSA01289 creates the audio device of its next world before it destroys the previous one, so it calls Init twice before Term twice, and only logs a non-zero result - [sceAudioOutSetMixLevelPadSpk](../../core/libs/prx/libSceAudioOut/src/AudioOut.cpp) (libSceAudioOut) - effect of a negative mix level unknown; only 0 to 32768 (0 dB) is accepted - [sceAudioOutSetMixLevelPadSpk](../../core/libs/prx/libSceAudioOut/src/AudioOut.cpp) (libSceAudioOut) - no source applies the level (shadPS4 only stores it); it is assumed to be a linear gain on the port's own output, and the -9 dB default (11626) is applied to every pad speaker port, including those of titles that never call the function - [sceAudioOutGetLastOutputTime](../../core/libs/prx/libSceAudioOut/src/AudioOut.cpp) (libSceAudioOut) - signature and behaviour taken from the [shadPS4](https://github.com/shadps4-emu/shadPS4) and [fpPS4](https://github.com/red-prig/fpPS4) reimplementations of the PS4 library, not confirmed on a PS5 title: the time is the process time at which `sceAudioOutOutput` or `sceAudioOutOutputs` accepted the port's last block, not the time the block reached the device -- [sceAudioOut2PortCreate](../../core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp) (libSceAudioOut) - `data_format` is read as the channel count (bits 8-11: only 1, 2, 6, 8 or 12) and sample type (bits 0-6: 0 float, 1 int16); bit 7 and the port `flags` are not decoded, and bits 12-31 throw +- [sceAudioOut2PortCreate](../../core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp) (libSceAudioOut) - MAIN (type 0) format validation follows PS5 firmware 13.60 measurements: bits 8-11 accept 1, 2, 8 or 12, bits 0-6 accept 0 or 1, bit 7 requires 8 channels, and bits 12-31 do not affect validation. The sample types are still read as float and int16; bit 7's layout, the PCM effect of bits 12-31 and the port `flags` are unverified. Other port types retain unverified format validation, including 6 channels and rejection of bits 12-31 - [sceAudioOut2PortCreate](../../core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp) (libSceAudioOut) - 12 channels are assumed to be L R C LFE Ls Rs Lb Rb Ltf Rtf Ltb Rtb as in KytyPS5's 12-to-8 fold, and 6 and 8 channels to be its first 6 and 8; no title or SDK header confirms the order - [sceAudioOut2PortCreate](../../core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp) (libSceAudioOut) - the stereo fold is assumed, not the console's downmix: C at -3 dB into both sides, Ls/Rs, Lb/Rb and Ltf/Rtf at -3 dB and Ltb/Rtb at -6 dB into their own side, LFE dropped - [scePsmlMfsrGetContextBufferRequirement1100, scePsmlMfsrCreateContext1100, scePsmlMfsrGetDispatchMfsrPacket1100](../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml_debug) - argument layouts unknown; only 0x8A810001 (not initialized) is returned -- [BnMAMrsfVWo](../../core/libs/prx/libc/src/HeapDiagnostics.cpp) (libc) - unknown name, signature - [scePngEncEncode](../../core/libs/prx/libScePngEnc/Export.cpp) (libScePngEnc) - whether rows may stay unfiltered when `filter_type` names a subset of the filters is unknown; as in shadPS4, they may only for 0 and for the all-filters mask - [sceJpegEncEncode](../../core/libs/prx/libSceJpegEnc/Export.cpp) (libSceJpegEnc) - `restart_interval` follows SharpProspero's field doc (0 none, -1 every row of blocks, positive a block count); a positive value is taken as the JPEG DRI interval in MCUs and -1 as one restart per MCU row, which no reference confirms. Other negative values throw -- [sceVoiceQoSInit](../../core/libs/prx/libSceVoiceQoS/Export.cpp) (libSceVoiceQoS) - the error codes are unknown: a null or empty memory block, an app type other than 0x20000000 (PPSA14632) or 0x10000000 (PPSA26344) and a second initialization throw. The minimum memory size is unknown; the app type and the block are not used, as no endpoint is implemented +- [sceVoiceQoSInit](../../core/libs/prx/libSceVoiceQoS/Export.cpp) (libSceVoiceQoS) - `0x804E0905` for a second initialization comes from [prosper](https://github.com/mattias800/prosper)'s reading of the module and was not measured: on a PS5 a valid call blocks inside a homebrew payload, so it never returns there. The app type and the block are not used, as no endpoint is implemented - AudioIn and NpSessionSignaling exports added without an implementation have assumed signatures +- [sceNpSessionSignalingGetConnectionFromPeerAddress](../../core/libs/prx/libSceNpSessionSignaling/Export.cpp) (libSceNpSessionSignaling) - imported by PPSA01289 and added without an implementation; unknown signature +- [libSceCdlgG2P](../../core/libs/prx/libSceCdlgG2P/Export.cpp) - `sceG2PDialogInitialize`, `Terminate`, `Open`, `GetStatus`, `UpdateStatus` and `GetResult`, imported by PPSA01289, are added without an implementation and declared without parameters - [vieBRwlh1Lw](../../core/libs/prx/libSceAgc/Unimplemented.cpp) (libSceAgc) - unknown name, signature -- [fCWdlnmB1Ks](../../core/libs/prx/libScePad/Export.cpp) (libScePad) - unknown name, signature +- [scePadIsRemoteController](../../core/libs/prx/libScePad/Export.cpp) (libScePad) - the name hashes to NID `fCWdlnmB1Ks`; the signature `(int handle, bool* remote)` comes from PPSA16064, which zeroes a byte, passes its address after `scePadGetControllerInformation` and ignores the result. The error codes follow the other pad calls. Controllers are local devices, so the answer is always false +- [scePadSetAngularVelocityBiasCorrectionState](../../core/libs/prx/libScePad/Export.cpp) (libScePad) - imported by PPSA16064 and not called during its start-up; unknown signature - [sceKernelGetOperationMode](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - the signature `(int* mode, int* submode)` comes from PPSA12544, which logs both values after the call; the values are unknown and 0 is reported for both, as in the [prosper](https://github.com/mattias800/prosper/pull/4153) reimplementation. Null outputs throw - Font, Http2, Net, Ssl, SystemService and libkernel exports imported by PPSA12544 and added without an implementation have unknown signatures +- [sceHttp2GetMemoryPoolStats, sceSslGetMemoryPoolStats, sceNetGetMemoryPoolStats](../../core/libs/prx/libSceNet/Export.cpp) (libSceHttp2, libSceSsl, libSceNet) - signatures `(int id, stats*)` and the statistics layouts (pool size, peak and current use, and for Http2 and Ssl a 32-bit reserved field) follow the PS4 SDK's `SceHttp2MemoryPoolStats`, `SceSslMemoryPoolStats` and `SceNetMemoryPoolStats`, not confirmed for the PS5. The pool size is the one given at init or pool creation; nothing is allocated from these pools, so the peak and current use are 0. Net returns `EBADF` for an unknown pool and `EINVAL` for a null output, as its other pool calls do; for Http2 and Ssl the error codes are unknown, so both cases throw +- [sceHttp2SetResolveRetry](../../core/libs/prx/libSceHttp2/Export.cpp) (libSceHttp2) - signature `(int id, int32_t retry)` from libSceHttp's `sceHttpSetResolveRetry`; the value is accepted and ignored like the other Http2 settings, since no request reaches the network +- [sceHttp2WebSocketCreateRequest, sceHttp2WebSocketCloseAsync, sceHttp2WebSocketSendTextMessageAsync, sceHttp2WebSocketSendDataMessageAsync](../../core/libs/prx/libSceHttp2/Export.cpp) (libSceHttp2) - imported by PPSA16064 and not called during its start-up; unknown signatures - [libSceVrSetupDialog](../../core/libs/prx/libSceVrSetupDialog/Export.cpp), and the Share, NpTrophy2, NpEntitlementAccess and WebBrowserDialog exports imported by PPSA12544's Unity plugins and added without an implementation, have unknown signatures -- [Z4HDpe1ZEa4, oXShHcqMcX0](../../core/libs/prx/libSceAjm.native/Export.cpp) (libSceAjmWVorbis, implemented in libSceAjm.native) - unknown name, signature +- [sceAjmDecWVorbisCreateHeaderPacket, sceAjmDecWVorbisCreateSetupPacket](../../core/libs/prx/libSceAjm.native/Export.cpp) (libSceAjmWVorbis, implemented in libSceAjm.native) - unknown signatures - Acm, GameLiveStreaming, Net, NpEntitlementAccess, NpManager, NpWebApi2, VoiceQoS, WebBrowserDialog and libkernel exports imported by PPSA14632 and added without an implementation are declared without parameters - [libSceAudioPropagation](../../core/libs/prx/libSceAudioPropagation/Export.cpp) (libSceAudioPropagation) - signatures and struct descriptors recovered from PPSA21567's calls (#481), not from documentation. The system options layout is unknown (only checked for non-null); the fourth argument of `sceAudioPropagationSourceCalculateAudioPaths` is unknown; `sceAudioPropagationSourceGetAudioPath` is assumed to output an 8-byte path handle. The meaning of the `RenderInfo` word at +0x28 is unknown (PPSA21567 and PPSA21564 pass 2); only 2 is accepted. Both titles pass one `RenderInfo` per call: a count of 0 throws, and larger counts are handled element by element without proof. No error code is known, so invalid input throws instead of returning one - [NGS2 mastering voice gain](../../core/libs/prx/libSceNgs2.native/src/Voice.cpp) (libSceNgs2.native) - the param id 0x30000004 is taken from PPSA22520 and the layout from [shadPS4](https://github.com/shadps4-emu/shadPS4)'s `OrbisNgs2MasteringVoiceGainParam`; no reference gives the units or range. The levels are assumed to be linear factors applied when the mastering voice is mixed into the render buffer, so the voice's own samples stay unscaled; the LFE level applies to channel 3 of 6- and 8-channel voices, assuming the L R C LFE order of the AudioOut2 entries above, and the full-band level to the rest. Negative levels are accepted, and a new setup resets both levels to 1 - [NGS2 stereo mastering voice into a surround buffer](../../core/libs/prx/libSceNgs2.native/src/Render.cpp) (libSceNgs2.native) - PPSA21402 renders a 2-channel mastering voice into a 6- or 8-channel render buffer, which the mixing stage rejected. The console's conversion is unknown: the left and right channels are assumed to go to the first two buffer channels (front left and right) and the remaining channels stay silent, with no centre, LFE or surround content. Any other channel count mismatch still throws +- [NGS2 streaming PCM waveforms](../../core/libs/prx/libSceNgs2.native/src/Voice.cpp) (libSceNgs2.native) - PPSA21402 streams its music by adding one block whose `num_samples` is the whole track and whose `data_size` is a 16 KiB first piece, with `CONTINUE`, and then adds each following piece with flag `0x2` (called `APPEND` here) and the same `num_samples`. No reference describes flag `0x2`. A 16-bit PCM block without skipped samples or repeats whose `num_samples` exceed its data is assumed to start such a stream; flag `0x2` is assumed to append the block's data to the last streaming block, ignoring its `num_samples`; the voice is assumed to output silence without advancing while it has played all appended data and `CONTINUE` is set; and the waveform is assumed to end at `num_samples` or, once `CONTINUE` is cleared, at the end of the appended data. `sceNgs2VoiceGetState` reports the bytes played as `decoded_data_size`, which the title uses to release its queued data, and the read position as `waveform_data`. Appending to a block that is not a stream, or other formats, skipped samples, repeats and partial frames, throw - [sceAvPlayerStartEx](../../core/libs/prx/libSceAvPlayer/Export.cpp) (libSceAvPlayer) - start info layout unknown; it is ignored and playback starts as with `sceAvPlayerStart` - [sceAvPlayerInit / sceAvPlayerInitEx](../../core/libs/prx/libSceAvPlayer/src/Player.cpp) (libSceAvPlayer) - behaviour without a memory replacement unknown; frame and sample buffers then come from the guest heap - [SceAvPlayerVideoEx](../../core/libs/SceTypes.hpp) (libSceAvPlayer) - frame rate field and encoding unknown; it is left zero in frame and stream info @@ -162,6 +192,8 @@ Throughout the project, every function at every stage either **does exactly what - [sceHttpUriUnescape](../../core/libs/prx/libSceHttp/src/Uri.cpp) (libSceHttp) - signature follows [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Net/Http.cs); percent decoding, invalid sequences and error codes follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.cpp), not confirmed on a PS5 title or hardware. Size includes the final NUL; malformed escapes and `+` remain unchanged, and decoding runs once. - [sceHttpUriSweepPath](../../core/libs/prx/libSceHttp/src/Uri.cpp) (libSceHttp) - behaviour follows the [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.cpp) reimplementation of the PS4 library and its tests (including the PS4 SDK example), not confirmed on a PS5 title or hardware. It is not RFC 3986 5.2.4: a path that doesn't start with `/` is copied unchanged and a trailing `.` or `..` without a slash is kept. Unlike shadPS4, nothing at or past `srcSize - 1` is read, and `srcSize == 1` writes only the terminator - [sceHttpSetResponseHeaderMaxSize, sceHttpRedirectCacheFlush, sceHttpsUnloadCert](../../core/libs/prx/libSceHttp/Export.cpp) (libSceHttp) - signatures follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.cpp) (PS4); no PS5 reference has them +- [sceHttpSetRedirectCallback, sceHttpSetCookieRecvCallback, sceHttpsSetSslVersion, sceHttpsGetSslError](../../core/libs/prx/libSceHttp/Export.cpp) (libSceHttp) - signatures follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.cpp) (PS4), which only logs and returns 0; a null `errNum` or `detail` in `sceHttpsGetSslError` is assumed to return `SCE_HTTP_ERROR_INVALID_VALUE` like `sceHttpGetLastErrno` +- [sceHttpSetAuthInfoCallback](../../core/libs/prx/libSceHttp/Export.cpp) (libSceHttp) - signature and callback type follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.h) (PS4); no PS5 reference has them. The `id` is not checked, as in the other libSceHttp setters - [sceHttp2GetMemoryPoolStats](../../core/libs/prx/libSceHttp2/Export.cpp) (libSceHttp2) - unknown signature - [libSceHttp2 cookie boxes](../../core/libs/prx/libSceHttp2/Export.cpp) (libSceHttp2) - `sceHttp2CreateCookieBox` is taken to take the library context id first, as in the [prosper](https://github.com/mattias800/prosper) reimplementation; further arguments are unknown and ignored. `sceHttp2SetRequestNoContentLength` is assumed to take only the request id. Cookies are never stored, so `sceHttp2CookieFlush` has nothing to discard - [sceNetGetMemoryPoolStats](../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - unknown signature @@ -170,8 +202,6 @@ Throughout the project, every function at every stage either **does exactly what - [sceNpSessionSignalingGetConnectionStatistics](../../core/libs/prx/libSceNpSessionSignaling/Export.cpp) (libSceNpSessionSignaling) - unknown signature - [sceUserServiceGetAgeLevel](../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - the meaning of the value is unknown; 0 is reported, as for the unset game presets. PPSA12544 (Unity) only stores it for its user profile - [sceGameUpdateGetAddcontLatestVersion](../../core/libs/prx/libSceGameUpdate/Export.cpp) (libSceGameUpdate) - a null `entitlement_label` throws: KytyPS5 accepts it, prosper returns `0x80412803`. Bytes after `found` are zeroed as in KytyPS5; prosper leaves them untouched -- [sceSystemServiceDisableMediaPlay](../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - unknown signature -- [sceSystemServiceReenableMediaPlay](../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - unknown signature - [sceSystemServiceParamGetString](../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - the system name (parameter 6) is the fixed `PS5`, not the console's name; other parameters and buffers of 1 to 64 bytes throw - [sceNgs2SystemSetSampleRate, sceNgs2SystemSetUserData, sceNgs2SystemGetUserData, sceNgs2SystemLock, sceNgs2SystemUnlock](../../core/libs/prx/libSceNgs2.native/src/System.cpp) (libSceNgs2.native) - shadPS4 only checks the system handle (`INVALID_SYSTEM_HANDLE`) and SharpProspero names what each call does; neither gives the other limits. A sample rate of 0 and a null user data pointer throw, any other rate is accepted, and changing the rate while a sampler filter is enabled throws because the filter coefficients were computed for the old rate. Lock and unlock take the same lock as `sceNgs2RackLock` - [sceNgs2RackGetInfo](../../core/libs/prx/libSceNgs2.native/src/Rack.cpp) (libSceNgs2.native) - field order from [shadPS4](https://github.com/shadps4-emu/shadPS4)'s `OrbisNgs2RackInfo` with the 64-byte name of the other info structures (216 bytes); `type` is assumed to be the rack id divided by 0x1000, `min_grain_samples` 64 (the system minimum) and `state_flags` 1; `uid` comes from a per-process rack counter, `render_count` is the owning system's render count and `active_voice_count` counts voices that are not empty (paused and stopped voices included). The last process ratio and tick and `active_channel_work_count` are left zero @@ -198,7 +228,6 @@ Throughout the project, every function at every stage either **does exactly what - [sceRtcFormatRFC2822, sceRtcFormatRFC2822LocalTime, sceRtcFormatRFC3339LocalTime](../../core/libs/prx/libSceRtc/Export.cpp) (libSceRtc) - only shadPS4 (PS4) has bodies. A null tick returns `INVALID_POINTER` like the existing `sceRtcFormatRFC3339`, where shadPS4 formats the current time instead; offsets outside ±1439 minutes return `INVALID_VALUE` like `sceRtcFormatRFC3339`; the local-time variants use the host's offset at the given instant (shadPS4 uses `sceKernelGettimezone`, the current offset) - [snwprintf_s](../../core/libs/prx/libc/src/FormattingWide.cpp) (libc) - follows C11 K.3.9.1.3 with `RSIZE_MAX` assumed to be `SIZE_MAX >> 1`. A runtime-constraint violation only returns a negative value: `set_constraint_handler_s` is not exported and the console's default handler is unknown. An invalid multibyte `%s` argument is copied byte by byte instead of being reported as an encoding error, as in `vswprintf` - [libSceUlt queues](../../core/libs/prx/libSceUlt/Export.cpp) - the added `sceUltQueuePop`, `sceUltQueueTryPush`, `sceUltQueueDestroy` and `sceUltQueueDataResourcePoolDestroy` signatures are inferred from the existing push/pop and object APIs. Queue destruction is assumed to reject blocked callers with `ULT_ERROR_BUSY` and otherwise discard queued data; pool exhaustion returns `ULT_ERROR_AGAIN`. These rules, alignment checks and shared slot limits have not been confirmed on PS5 hardware or a title. Non-null queue and data-pool option parameters throw because their layouts are unknown. -- [sceKernelReadv / sceKernelWritev / sceKernelPreadv / sceKernelPwritev](../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) are implemented on Linux only; on Windows they throw - [LinuxProtFromSce](../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - protection bits 0x100 and 0x200 are accepted and assumed not to affect host CPU access; host protection is derived only from the read, write and execute bits - [_sceUlobjmgrRegisterObject](../../core/libs/prx/ulobjmgr/Export.cpp) (ulobjmgr) - parameters taken from shadPS4 (PS4): a nonzero 64-bit object, a nonzero 32-bit kind and a 32-bit id output; the meaning of the first two and the PS5 signature are unverified. The id bound of `_sceUlobjmgrUnregisterObject` (below 0x4000) and the raw `EINVAL` (22) return also come from shadPS4 - [sceShareGetCurrentStatus](../../core/libs/prx/libSceShare/Export.cpp) (libSceShare) - validation (feature flag non-zero, status non-null) and the all-zero 16-byte status taken from the [KytyPS5](https://github.com/KytyPS5/KytyPS5) reimplementation, not confirmed on a PS5 title; the meaning of the recording status values is unknown @@ -207,40 +236,59 @@ Throughout the project, every function at every stage either **does exactly what - [Resource registration](../../core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp) (libSceAgcDriver) - registration always fails with `RESOURCE_REGISTRATION_UNAVAILABLE`, so the owner and resource queries, `sceAgcDriverSetResourceUserData`, `sceAgcDriverFindResourcesPublic` and `sceAgcDriverUnregisterAllResourcesForOwner` return the same error without reading their arguments. Their parameter lists follow the libSceGnmDriver counterparts and are unverified. `sceAgcDriverGetResourceRegistrationMaxNameLength` writes 0xfc and returns 0, as [prosper](https://github.com/mattias800/prosper) does from a title's registration configuration; the `uint32_t*` parameter matches the SDK wrapper `sce::Agc::ResourceRegistration::getMaxNameLength(unsigned int*)`. sharpemu writes 256 and KytyPS5 returns `RESOURCE_REGISTRATION_UNAVAILABLE`, so the value is unverified; a null output throws - [Shader registration](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a shader header is accepted at any address: PPSA26344 passes headers 4 and 1 bytes past an 8-byte boundary; whether the console refuses an alignment is unknown - [sceNgs2VoiceControl sampler filter](../../core/libs/prx/libSceNgs2.native/src/Voice.cpp) (libSceNgs2.native) - param `0x1000000a`, its layout and meaning follow KytyPS5 only: location 1 type 1 is an RBJ low-pass biquad at the system sample rate after resampling, scaled by `level`, a set `channel_mask` bit bypasses that channel, type 0 turns the filter off. Other locations and types throw. The voice keeps playing a filter tail after its last block until the history is zero, with outputs below 1e-20 flushed to zero (a KytyPS5 choice); the console's tail length is unknown -- [sceRtcGetDosTime, sceRtcSetDosTime](../../core/libs/prx/libSceRtc/Export.cpp) (libSceRtc) - bit layout from shadPS4 (PS4, the standard FAT date/time). Years outside 1980-2107 throw in `GetDosTime`, since no reference says what the library returns for them. `SetDosTime` decodes the year as the unsigned 7-bit FAT field (1980-2107), where shadPS4 sign-extends the 16-bit date to years before 1980. `SetDosTime` does not validate the fields, as in shadPS4 +- [sceRtcGetDosTime, sceRtcSetDosTime](../../core/libs/prx/libSceRtc/Export.cpp) (libSceRtc) - bit layout from shadPS4 (PS4, the standard FAT date/time). A year outside 1980-2107 makes `GetDosTime` return `INVALID_YEAR` after writing 0 for an earlier one and `0xff9fbf7d` (2107-12-31 23:59:58, the largest encodable time) for a later one, and `SetDosTime` decodes the year as the unsigned 7-bit FAT field (1980-2107) without validating the fields, as the disassembly-derived [fpPS4](https://github.com/red-prig/fpPS4) reimplementation of the PS4 library does; not confirmed on a PS5 title, and shadPS4 instead masks out-of-range years into the packed value and sign-extends the 16-bit date - [sce::Json::InitParameter2](../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - layout inferred, not known: assumed to start with the allocator, its user data and the file buffer size, in that order, and the whole struct is assumed to fit in 40 bytes; `Initializer::initialize(InitParameter2)` ignores them as `initialize(InitParameter)` does - [sceAjmBatchJobSetResampleParameters, sceAjmBatchJobGetResampleInfo](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm) - the ratio is taken as decoded samples per output sample and num_samples as the decoded samples the resampler holds, both inferred from FMOD (Hades II); the console's interpolation is unknown, a 4-point Catmull-Rom is used. The flags argument is ignored, as in shadPS4 and Kyty - [scePlayerReviewDialogOpen, scePlayerReviewDialogGetResult](../../core/libs/prx/libSceCdlgPlayerReview/Export.cpp) (libSceCdlgPlayerReview) - the PS5-only dialog has no public reference: the parameter is not read and the result is assumed to start with the 32-bit common dialog result, as `sceWebBrowserDialogGetResult` assumes +- [sceHttpWaitRequest](../../core/libs/prx/libSceHttp/Export.cpp) (libSceHttp) - the timeout is assumed to be in microseconds and a negative timeout to wait without limit; the invalid-value error for a null handle or event array or a non-positive event count is assumed, not measured. No request is ever pending, so the call waits for the timeout and returns 0 events, and a wait without limit throws ### Functional +- [libSceHttp2 asynchronous requests](../../core/libs/prx/libSceHttp2/Export.cpp) (libSceHttp2) never reach the network: `sceHttp2SendRequestAsync` accepts the request, queues a completion whose result is the library's network error and triggers the user event named by the kqueue option (equeue at offset 0, event id at 8, user data at 16, layout inferred from PPSA17221); `sceHttp2WaitAsync` returns that completion. The completion's `event_type` value is unknown and left 0. PPSA17221 completes its request object only after `sceHttp2SendRequestAsync` returns, and its handler for a synchronous failure calls `shared_from_this` before that, so a send must not fail synchronously. A kqueue option or user event that is missing, an extra option, and a wait with no completed operation throw +- [sceKernelWriteThrottlingStatus](../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) reports no write throttling: the host SSD has no write budget, so the first qword of the 32-byte status is set to `UINT32_MAX` and the rest is zeroed. PPSA17221 reads only that qword: it multiplies it by 64 KiB and compares the product with byte thresholds up to 0x4AF00000 to pick one of five tiers. When the call fails the title assumes 0x4B000001 bytes, which selects the same top tier as `UINT32_MAX`. What the value measures on hardware (a remaining budget or a rate) and the meaning of the other three qwords are unknown +- [Wide-character conversions](../../core/libs/prx/libc/src/Strings.cpp) (libc) - `wcsrtombs` and [`swprintf`](../../core/libs/prx/libc/src/Formatting.cpp) still pass guest strings to the host's `wchar_t` functions. The guest's `wchar_t` is 16 bits, so on Linux, where the host's is 32 bits, they misread and miswrite guest strings. The wide `wcsto*` functions parse the ASCII prefix of the guest string, as the C locale accepts only ASCII white space and digits +- [SPIRV-Tools optimization](../../core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp) skips CCP and Simplification for modules with storage-only 8- or 16-bit types because the pinned folder produces illegal narrow constants. This also disables unrelated constant folding; the restricted full pipeline mirrors the pinned performance recipe. +- Windows host-pointer imports of the guest mapping itself are compared instead of write-watched: a driver can mark imported pages dirty after the synchronous import window. Shared memory imported through a separate writable alias retains protection-based tracking on the guest views; driver access to the import alias does not disarm those views. [GuestMemory](../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes every overlapping 64 KiB tracker block while its backing is retained. A full block replaced by fresh private backing regains tracking and gets a new generation; protection changes, partial replacements and shared remappings do not restore it. This preserves CPU edits through byte comparisons but disables snapshot reuse and unit shadows for excluded blocks; restoring tracking on import retirement without replacing the backing and measuring the comparison cost remain unimplemented. - [sceNetSendmsg and sceNetRecvmsg](../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) copy the scatter/gather list through one host buffer and accept only the flags `sceNetSend`/`sceNetRecv` accept. Sending control data throws; a receive never returns control data (`msg_controllen` 0), since no socket option that queues it is supported - [Additional content](../../core/libs/prx/libSceAppContent/Export.cpp) (libSceAppContent) is not installed or mounted: `sceAppContentAddcontMount` answers `NOT_FOUND` for every entitlement label and `sceAppContentAddcontUnmount` for every mount point, as no add-content mount is ever handed out; `sceAppContentGetAddcontInfo` answers `DRM_NO_ENTITLEMENT` for every label and `sceAppContentGetAddcontInfoList` reports 0 entries +- [sceKernelPread, sceKernelPwrite, sceKernelPreadv, sceKernelPwritev](../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) on Windows transfer at the given offset with an overlapped `ReadFile`/`WriteFile` and restore the descriptor's file pointer afterwards, so a concurrent `read`/`write` on the same descriptor can see the moved pointer in between, unlike on the console. Handles that are not disk files return `ESPIPE`. `readv` on a pipe or console handle stops after the first iovec read, a short read the console is assumed to allow; each iovec is transferred separately, so a failure after the first transfer returns the bytes already done +- [sceKernelSync](../../core/libs/prx/libc/src/General.cpp) (libkernel) on Windows flushes only the host files and directories recorded through `RecordWrittenPath_nid_no_patch` (the libc, libkernel, save data and app content file functions) and their changed parent directories. A host write in a library that does not record its path is not flushed by `sceKernelSync` +- [sceKernelSync](../../core/libs/prx/libc/src/General.cpp) (libkernel) on Windows skips a recorded file it cannot open for writing (read-only, or held open by a handle that denies write sharing), since `FlushFileBuffers` needs a writable handle. Such a file's data may not reach the disk before a host crash; the guest reads the same data either way - [sceKernelInstallExceptionHandler](../../core/libs/prx/libkernel/System/src/Exception.cpp) records the handler, but a host fault still ends the process: only signals raised with `sceKernelRaiseException` are delivered. Unity's PS5Util module installs a SIGUSR1 handler that reads `uc_mcontext.mc_rsp` (offset 0xf8 of the context) and raises SIGUSR1 on other threads to stop them for garbage collection - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) delivers on Windows only and throws on Linux - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) builds the handler's `ucontext` with FreeBSD's amd64 layout (16-byte sigmask, `mcontext` at 0x40, `mc_rsp` at 0xf8); the console's layout is not verified beyond the fields PS5Util reads - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) does not consult the target thread's signal mask -- [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) saves and restores only `CONTEXT_FULL | CONTEXT_SEGMENTS` on Windows, so the upper halves of the YMM registers are not preserved if the handler or the entry uses AVX - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) reaches a thread that waits in an alertable `TimedWait` wait through an APC. A thread blocked anywhere else, including all waits under `APS5_COARSE_TIMED_WAITS` and host waits inside the emulator, is redirected and runs the handler only when that wait returns - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) on Windows runs the handler wherever the target thread was interrupted. It waits only while the thread is inside winpthreads or restoring a context through `NtContinue`, and throws if that lasts 1 s, so a handler that calls into other host code the thread was interrupted in, such as the C runtime's heap or a lock of the emulator, can deadlock on it or reenter it - [libSceUlt queues](../../core/libs/prx/libSceUlt/Export.cpp) use preallocated host storage instead of the supplied guest work area; its existing size formula and the 512-byte object layouts remain unverified. Waiting-queue pool thread limits and waiter priority ordering are not implemented. Finalization wakes blocked queue callers with `ULT_ERROR_STATE`. - [Thread cancellation](../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) is not modelled: `scePthreadCancel` throws, so no cancel request is ever pending and `scePthreadTestcancel` and `pthread_testcancel` return without acting; the cancel state and type set by `scePthreadSetcancelstate` and `scePthreadSetcanceltype` are stored and not used -- [MIMG `tfe` and `lwe`](../../core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp) never write the status VGPR after the data. On hardware it is written only when a texel fetch hits an unmapped page of a partially resident texture, or, for `lwe` on sample/gather, when the LOD is below the T# `MIN_LOD_WARN`; texture memory is always resident here, and `lwe` on sample, gather and `image_get_lod` throws. +- [MIMG `tfe` and `lwe`](../../core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp) never write the status VGPR after the data. On hardware it is written only when a texel fetch hits an unmapped page of a partially resident texture, or, for `lwe` on sample/gather, when the LOD is below the T# `MIN_LOD_WARN`; texture memory is always resident here, the LOD warning (texture-streaming feedback, like the T# `MIN_LOD_WARN` itself) is never reported, and `lwe` on `image_get_lod` throws. +- [Packed horizontal gathers](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (`image_gather4h_pck`, MIMG opcode 98): the four window texels are read raw and concatenated into a little-endian bitstream, with DMASK bit i writing stream word i to the next destination register; measured on RDNA2 (gfx1031) at every edge: the anchor is floor(x-0.5) with elements at -1, 0, +1 and +2 clamped horizontally into the width, the row is floor(y), negative rows clamp to row 0, rows at or past the height return zeros, sampler addressing modes do not affect the window, and formats wider than 32 bits per texel write zeros in every register. The emitter fetches each window texel at LOD 0 and packs its bits like `image_load_pck`, so the sampler's filter, addressing and LOD state are not used; formats whose bits are not recoverable from the view and D16 data throw. The anchor and row multiply the normalized coordinate by the image size without contraction. The stream rule was measured for R8_UNORM, RG8_UNORM and RGBA8; only RGBA8 has an execution test. `image_gather8h_pck` (opcode 99) never returns texel data on this hardware - across dmasks 0x1 to 0xFF including the upper nibble, RGBA8, RGBA16 and RGBA32 formats, 1D, 2D and 3D descriptors and normalized and unnormalized sampling it only zero-fills popcount(dmask) registers (sentinel-prefilled destinations) - so it decodes but throws at translation; its eight-texel semantics remain unmeasured. +- [MIMG BY2/BY4 loads](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (`image_load[_mip]_by2/by4`, opcodes 0x42, 0x43, 0x4a and 0x4b) were measured on RDNA2 (gfx1035) with the [hardware oracle](HW_ORACLE.md) for 2D images with identity swizzle. They read N texels (BY2: 2, BY4: 4) from `x & ~(N - 1)` on row y, texel by texel and component by component within each texel, with each texel converted like `image_load`. Every register is zero when the mip level is past the last level, `x + N - 1` is outside the level's width (unaligned x, unsigned), or y is outside its height. Data only comes back when the N texels fit in one dword: R8 and R16 (DMASK 0x3) or RG8 (0xf) for BY2, and R8 (0xf) for BY4. With 32-bit texels (R32, RG32, RG16, RGBA8) the hardware returns zeros. Other formats, masks, dimensions, swizzles, D16/A16 and indirect descriptors throw. The stores (0x52, 0x53, 0x5a and 0x5b) decode but throw at translation. Measured on hardware, they write from the unaligned x up to the end of its dword, which is not implemented. +- [MIMG PCK2/PCK4 loads](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (`image_load[_mip]_pck2/pck4`, opcodes 0x70, 0x71, 0x73 and 0x74) were measured on RDNA2 (gfx1035) with the [hardware oracle](HW_ORACLE.md) for 2D images with identity swizzle. With data mask 0x1, they write one dword: the raw bits of the N texels (PCK2: 2, PCK4: 4) from `x & ~(N - 1)` on row y, concatenated (element k at bit `k * texel bits`) and zero-extended. The dword is zero when the mip level is past the last level, `x + N - 1` is outside the level's width (unaligned x, unsigned), or y is outside its height. Data only comes back when N texels fit in 32 bits: R8, R16 or RG8 for PCK2, R8 for PCK4. Wider texels (R32, RG16, RGBA8) and masks other than 0x1 write zeros on hardware. Those masks throw at decode, and formats outside UNORM/UINT/SINT R8, R16 and RG8 throw. #1314's earlier measurement saw only zeros because it used masks 0x3 and 0xf. The stores (0x76, 0x77, 0x79 and 0x7a) decode but throw at translation. They are not measured, and opcode 0x76 on 32_32_32_32_FLOAT hung the GPU (#1314). - [Unnormalized S# coordinates](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) - `FORCE_UNNORMALIZED` uses a Vulkan `unnormalizedCoordinates` sampler, so the S# LOD range, LOD bias and mip filter are not applied, which is exact only on a one-level view. - [Unnormalized S#s](../../core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp) are accepted only when direct and used only by explicit-LOD `image_sample` without offset, comparison, derivatives or `_a`, on a single-level, single-layer 1D or 2D view of mip 0, even when the image is also sampled through a normalized S#. - An [unnormalized S#](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) throws with unequal minification and magnification filters, anisotropy, `TRUNC_COORD`, `MC_COORD_TRUNC` or X/Y clamping other than last-texel or border. - [Image atomics](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) on 32_SINT and 32_FLOAT surfaces act on the raw dword through an R32_UINT storage view; their float results are assumed to match those measured on 32_UINT surfaces. Other formats throw. +- [`s_memtime`](../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) on RADV with a GFX10.3 to GFX11.7 GPU (RDNA 2, RDNA 3, RDNA 3.5) reads the Vulkan device clock instead of the subgroup clock, which [ACO](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp) reads from the 20-bit `SHADER_CYCLES` register there and which wraps every 2^20 cycles. The device clock is a monotonic 64-bit counter, but it counts the 100 MHz reference clock (the same one as `s_memrealtime`), not shader cycles, so `s_memtime` deltas are about 20 times smaller than on hardware. The families are matched by the name RADV reports ([`NarrowSubgroupClock`](../../core/libs/prx/libSceAgcDriver/Execution/src/SubgroupClock.cpp)); a family added to Mesa later reads the subgroup clock until it is listed - [`ds_wrxchg2_rtn_b32`, `ds_wrxchg2st64_rtn_b32`, `ds_wrxchg2_rtn_b64` and `ds_wrxchg2st64_rtn_b64`](../../core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp) are translated as two separate 32-bit or 64-bit atomic swaps per invocation. RDNA2 hardware (measured on gfx1035) performs every lane's first exchange before any lane's second one, so when lanes of the same wave or of different waves overlap addresses, the result can differ from hardware. `offset0 == offset1` throws, although on hardware it is two exchanges in order. +- [`ds_condxchg32_rtn_b64`](../../core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp) uses two 32-bit atomic conditional exchanges. RDNA2 (gfx1035) measurements confirmed aligned return values, conditional bit clearing and 16-bit offsets; out-of-range accesses return zero without modifying LDS. Misaligned accesses fault the wave on gfx1035; the recompiler does not model this fault. Radeon 780M (gfx1103, RDNA3) measurements confirmed aligned return values, conditional bit clearing, EXEC masking and operand overlap in wave32 and wave64; contended cases passed per-dword conservation checks. Contended lane ordering and gfx1013 remain unmeasured; pair-wide indivisibility is not assumed, and concurrent 64-bit LDS atomics use a separate lock. GDS throws. VGPR pairs extending beyond v255 throw; allocated-VGPR EXEC nullification is not emulated. - [64-bit image atomics](../../core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp) (`image_atomic_*` with a two-dword data mask) need `VK_EXT_shader_image_atomic_int64` and act on 32_32 UINT, SINT and FLOAT surfaces through an R64_UINT storage view. 64-bit `image_atomic_inc`, `image_atomic_dec` and the float atomics throw, as does any other access in the same shader to an image it updates with 64-bit atomics. - [8_SRGB and 8_8_SRGB loads](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) on a device that cannot sample R8_SRGB or R8G8_SRGB return the sRGB EOTF rounded to nearest even at bf16 precision, as adriadam10 measured on RDNA2 (gfx1035) in #866; the PS5's gfx1013 is not measured. - [8_SRGB and 8_8_SRGB textures](../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp) (libSceAgcDriver) that the device samples as R8_SRGB or R8G8_SRGB use its own decode, which need not match gfx1035: NVIDIA's R8_SRGB is one bf16 step off in 29 of 256 codes. - [Samples and gathers](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8_SRGB and 8_8_SRGB textures throw on a device that cannot sample R8_SRGB or R8G8_SRGB. - [Bindless image table](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) entries of 8_SRGB and 8_8_SRGB textures read zeros, like converted formats, on a device that cannot sample R8_SRGB or R8G8_SRGB. - The shader recompiler [ignores `s_setreg_b32`](../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp): writes to MODE (float rounding and denormal controls) and the other hardware registers have no effect, and `s_getreg_b32` reads of the MODE round fields report round to nearest even even after `s_setreg_b32` wrote another mode. The initial round mode from the shader's program registers is not read either. +- A [flat SRT read](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) through a null or unmapped pointer loaded from memory raises a [BDA fault](../../core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp) only if the program executes it, then stops or reads zero like a BDA read. +- Repeated `s_load`s of one [flat SRT](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) dword through a null or unmapped pointer report one of their pcs, and an `s_load` whose inaccessible dwords are all unused, or only address repeated loads, reports no fault of its own. +- A value read through a null or unmapped pointer still throws at record time if a [descriptor, branch condition, uniform fill or table base](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) needs it, or if the pointer comes from user data, even if the console skips the read. - On a host subgroup narrower than 32 lanes, [`threadBit`](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) reads an EXEC written as a value (`s_mov_b32`/`s_mov_b64`) at bit (subgroup invocation id) of EXEC_LO, so lanes at or above the subgroup size read a lower lane's bit. +- [EXEC read as a value](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) (`s_mov_b64 sN, exec`, the `s_*_saveexec` copy, ALU sources) holds only lanes with a host invocation; the console's also holds the wave's other lanes, e.g. 32-63 of a wave64 on 32 host lanes. - [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) and fetch-inactive `v_permlane16_b32`/`v_permlanex16_b32` read undefined data from a lane that the host subgroup does not hold or whose host invocation is inactive; the console reads the lane's VGPR. - Without fetch-inactive, [`v_permlane16_b32` and `v_permlanex16_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) take a source lane that the host subgroup does not hold as inactive and keep the old value (0 with bound_ctrl); the console reads that lane's VGPR. +- On a host subgroup narrower than the wave, [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) of a constant lane outside it builds only as lane 63 of a half-wave scan (DPP row_shr 1/2/4/8, `v_permlanex16_b32`, EXEC -1); other such reads throw. - [Shader recompilation](../../core/shader/recompiler/Recompiler.cpp) currently occurs right before it was transferred to Vulkan with caching, but should be moved to the [relinker](../../core/relinker/main.cpp) stage. For this purpose, [shader/recompiler](../../core/shader/recompiler) was written completely independently from [libs/prx](../../core/libs/prx). +- [v_interp_p1ll_f16, v_interp_p1lv_f16 and v_interp_p2_f16](../../core/shader/recompiler/Translation/src/AttributeInstructions.cpp) read the 16-bit halves of each vertex's parameter through `VK_KHR_fragment_shader_barycentric` and throw without it, as fixed-function interpolation cannot split a 32-bit attribute. P10 and P20 are the vertex differences rounded to f16 to nearest even, which is not confirmed on hardware. v_interp_p1ll_f16 and v_interp_p1lv_f16 round the exact `P10 * I + P0` once to f32 and v_interp_p2_f16 rounds the exact `P20 * J + S2` once to f16, as measured on RDNA2 (#697); the exact sums are computed in f64, so these instructions need the Float64 capability. They follow the shader's float mode, measured on RDNA2 (gfx1035) in all eight combinations of `IEEE_MODE` and kept/flushed f32 and f16 denormals over 768 special and 1,024 random rows. With IEEE=1 a NaN operand is returned quieted; with IEEE=0 it is returned unquieted, except that v_interp_p2_f16's f16 result is always quiet. Flushed f32 denormals flush I/J, v_interp_p2_f16's f32 S2 and the f32 result; flushed f16 denormals flush P0, P10/P20, v_interp_p1lv_f16's f16 S2 and the f16 result. Denormal modes 1 and 2 (flush only inputs or only outputs) and rounding other than to nearest even are not measured and throw. An unknown mode follows IEEE=0 with denormals kept. Lavapipe has no barycentrics, so the draw test is skipped in CI; `agc_driver_interpolation_modes` checks the arithmetic in every mode against the hardware rows on any device with Float64. - [v_fma_f64](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp) is fused exactly only for normal inputs: with a subnormal input it is a separate multiply and add, which rounds twice. - [SINT storage images](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8- and 16-bit components saturate each stored 32-bit value to the component range, as adriadam10 measured for image stores on RDNA2 (gfx1035) in #944 (448 rows match); the PS5's gfx1013 is not measured. - [`StoreTexel`](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) writes channel 1 of a two-channel format from the data slot whose DST_SEL is Y, for SINT and UINT alike. With DST_SEL X,Y,Z,W, gfx1035 writes it from the W slot, as adriadam10 measured in #944. @@ -248,39 +296,73 @@ Throughout the project, every function at every stage either **does exactly what - A [flip inside a COND_EXEC range](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/CommandValidation.cpp) throws: the flip is reserved with the video output when the buffer is submitted, and how the console withdraws a skipped one is unknown. - A [COND_EXEC range](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp) must hold whole packets and end inside its command buffer. A chained INDIRECT_BUFFER or a REWIND inside a range throws, as what the CP does with the skipped dwords then is unknown. - A [COND_INDIRECT_BUFFER](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp) (`sceAgcCbBranch`) runs only with the always-pass compare function; a comparison or a predicated packet throws. Modes 1 and 2 run the first buffer, assumed to be the if-then and if-then-else modes of AMD's PM4 packet. [sceAgcCbBranch](../../core/libs/prx/libSceAgc/Cb/src/Branch.cpp) does not check the compare address of an always-pass branch or the address of an empty buffer, as PPSA14632 passes an uninitialised compare address and a null else buffer of 0 dwords. -- [v_mullit_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the hardware measured with the MODE register at IEEE=1 and denormals kept (a signaling NaN src0 is quieted, a positive denormal src2 counts as positive): the recompiler does not read MODE, so other IEEE/denormal settings are not modelled. +- [SET_*_REG_INDIRECT](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp) (libSceAgcDriver) reads its register list when the command buffer is submitted, with the packets. The console's graphics PFP reads it when it reaches the packet, so a list written after the submission (by the CPU, or by GPU work ahead of the packet behind PFP_SYNC_ME) is read here before that write. +- The [shader float mode](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (`FLOAT_MODE`, `DX10_CLAMP`, `IEEE_MODE` and `FP16_OVFL` of `COMPUTE_PGM_RSRC1` and `SPI_SHADER_PGM_RSRC1_PS/GS/HS`, bit positions from Mesa's `gfx10.json` and `gfx103.json`) is decoded into `GuestContext::floatMode` and is part of the shader cache keys and the serialized request (#981). `IEEE_MODE` is read by `v_min_f32`, `v_max_f32`, `v_min3_f32`, `v_max3_f32` and `v_med3_f32`, `IEEE_MODE` and the `FLOAT_MODE` output denormal bits by the output modifier, `DX10_CLAMP` by the clamp modifier, and the whole mode by the 16-bit interpolation instructions (see their entries). With IEEE=1 a signaling NaN operand returns that NaN quieted (the first operand checked first, min3/max3 as two nested pairs, med3 as min3 when any operand is NaN), measured on RDNA2 (gfx1035) over every combination of ±0, ±1, ±inf, denormals, quiet and signaling NaNs with f32 denormals kept and flushed. IEEE=0 and unknown modes keep the previous lowering, which matches IEEE=0. The f16 and f64 min/max/med3, NaN quieting in f16 arithmetic (the unary instructions and the f32 arithmetic follow it, see below), `FP16_OVFL` and `FLOAT_MODE` (f32 denormals flushed before the min/max compare in Astro's Playroom's 0xc0) still follow a single mode, as the entries below record. The mode comes from the registered shader header's `sh_registers`, the same source at registration and at draw/dispatch so the prepared artifact keys match. A title that rewrites RSRC1 at run time, or a raw compute program with no header, compiles with that mode or none. +- [v_mullit_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the hardware measured with the MODE register at IEEE=1 and denormals kept (a signaling NaN src0 is quieted only with IEEE=1, as measured at IEEE=0 too; a positive denormal src2 counts as positive): other denormal settings are not modelled. - [v_fma_legacy_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (VOP3 0x140) is assumed to be fused, which has not been measured on hardware. LLVM decodes this encoding as `v_mad_legacy_f32` for gfx1013, and `v_mac_legacy_f32` (VOP2 0x06) was measured to round its product before the add, so 0x140 may do the same. +- [v_fma_f16, v_pk_fma_f16 and v_pk_fmac_f16](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) are an f32 FMA rounded to f16, which rounds twice: 0x62e9 * 0xd159 + 0x23ec gives 0xf89e, the hardware 0xf89f. +- [Signaling NaN inputs](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) of the f32 and f16 unary instructions (`v_sqrt/rcp/rsq/log/exp/sin/cos/fract/floor/ceil/trunc/rndne/rcp_iflag/frexp_mant`), `v_ldexp_f32/f16`, `v_frexp_mant_f16`, `v_div_fixup_f32/f16` and `v_mullit_f32` are returned quieted only with IEEE=1 and unchanged with IEEE=0, measured on RDNA2 (gfx1035) in both modes; shaders without a known float mode follow IEEE=0. `v_add/sub/subrev/mul/fma/fmac/fmamk/fmaak/mul_legacy/fma_legacy_f32` and `v_fma_mix_f32` with f32 sources choose their NaN result on the bits, measured over 1000 operand triples in both modes: the first NaN source in operand order after its modifiers (the subtrahend negated, as the hardware adds its negation), quieted only with IEEE=1, or the default NaN 0xffc00000 for an invalid operation; in the fma forms an invalid product (inf * 0) gives the default NaN even when src2 is a NaN, and in the legacy forms a zero factor makes the product 0 even when the other factor is a NaN (a NaN src2 of `v_fma_legacy_f32` still wins). The f16 arithmetic, `v_fma_mix_f32` with f16 sources, `v_cubema`, `v_mac_legacy_f32` and the separately rounded mad/mac forms (not measurable on RDNA2) follow the same rule on hardware but their NaN bits are left to the driver. `v_cvt_f16_f32` and `v_dot2_f32_f16` always quiet and `v_div_scale_f32` never does. +- [Packed f16 NaN results](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (`v_pk_add/mul/fma/min/max_f16`) follow the hardware measured on RDNA2 (gfx1035) at IEEE=0 and IEEE=1 with f16 round-to-nearest-even and f16 denormals kept: with IEEE=1 a signaling NaN input is quieted and min/max return the other operand only for a quiet NaN; with IEEE=0 NaN inputs pass through unquieted and min/max ignore signaling. In `v_pk_fma_f16` an invalid product (inf * 0) gives the default NaN even when src2 is a NaN. A shader without a known float mode is translated as IEEE=0, the mode titles use; a mode with another f16 rounding or that flushes f16 denormals throws. +- [f32 denormals](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follow the shader's `FLOAT_MODE` f32 denormal field (bits 5:4) when it flushes them (0, and shaders without a known float mode, as in the 0xc0 most titles use), measured on RDNA2 (gfx1035) over 1444 rows per instruction at IEEE=0 and IEEE=1. `v_add/sub/subrev/mul/mul_legacy/fma/fmac/fmamk/fmaak/min/max/min3/max3/med3/fract/floor/ceil/frexp_mant/sin/cos_f32` flush their f32 inputs and their result to a zero of the same sign; the f32 compares, `v_cvt_i32/u32/rpi_i32/flr_i32_f32`, `v_frexp_exp_i32_f32`, `v_cvt_f64_f32`, `v_cvt_f16_f32`, `v_cvt_pkrtz_f16_f32`, `v_cvt_pknorm_i16/u16_f32` and `v_cvt_pk_u8_f32` flush their f32 inputs. Tininess is detected before rounding: a product whose exact value is below 2^-126 gives zero even when it rounds to 2^-126, computed exactly from the significands for `v_mul_f32`, `v_mul_legacy_f32`, and for `v_fma_f32`, `v_fmac_f32`, `v_fmamk_f32`, `v_fmaak_f32` and `v_fma_mix_f32` when the addend is zero; with a nonzero addend the fused result is kept, so an exact result just below 2^-126 that rounds to it is not flushed. `v_fma_mix_f32` was measured with f32 sources only; its f16 sources are not flushed. Denormal modes 1 and 2 throw. Not covered, and not measurable on RDNA2 (no such opcodes) or not measured: `v_mac_f32`, `v_mad_f32`, `v_madak_f32` and `v_madmk_f32` (rounded separately), `v_mac_legacy_f32`, `v_dot2_f32_f16`, the division steps (on hardware `v_div_fixup_f32` flushes only src1 and src2, `v_div_fmas_f32` only its result), f32 interpolation and LDS float atomics keep the previous lowering. With f32 denormals kept (3) the results depend on the driver's default denormal behaviour, as no `DenormPreserve` execution mode is requested, and `v_ldexp_f32` flushes in every mode. - The executable file that [relinker](../../core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp) generates opens the console when launched, which is inconvenient for playability. - [PA_CL_CLIP_CNTL](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - `DX_LINEAR_ATTR_CLIP_ENA` (bit 24) clear is drawn like bit 24 set, with Vulkan's clipping of `NoPerspective` outputs; what a clear bit changes on the console is undocumented. - [Per-stage descriptor count](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - a pixel shader's colour attachments are not counted against `maxPerStageResources`, so its buffers and images can fit the limit while the pipeline exceeds it - [v_rcp, v_rsq, v_sqrt, v_log, v_exp, v_sin and v_cos](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (f32 and f16) use the driver's approximation for finite results, not the console's: the low bits can differ. Only NaN, invalid, zero and infinite results and the exact sin/cos zeros are chosen on the bits. v_sin_f32/v_cos_f32 with inputs beyond ±256 cycles were not compared with the console. -- `--to-intel` does not lower RDPRU/MCOMMIT; the [matcher](../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) fails the relink instead. SHA-1 and SHA-256 instructions with a RIP-relative operand fail the relink. -- The length-changing path of the [instruction rewriter](../../core/relinker/codegen/src/x86/X64InstructionRewriter.cpp) is not used by the [converter](../../core/relinker/codegen/src/Amd64OnlyConverter.cpp): it does not adjust VEX/0F38/0F3A RIP-relative operands, data-to-code references (relocations, FDEs, jump tables) or segment sizes, so every substitution keeps the instruction length. +- `--to-intel` does not lower RDPRU/MCOMMIT; the [matcher](../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) fails the relink instead. +- The length-changing path of the [instruction rewriter](../../core/relinker/codegen/src/x86/X64InstructionRewriter.cpp) is not used by the [converter](../../core/relinker/codegen/src/Amd64OnlyConverter.cpp): it does not adjust data-to-code references (relocations, FDEs, jump tables) or segment sizes, so every substitution keeps the instruction length. - [DB_SHADER_CONTROL.CONSERVATIVE_Z_EXPORT](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp) (libSceAgcDriver) becomes `DepthLess`/`DepthGreater`. For a pixel shader that breaks the promise, the host driver decides whether its interpolated Z is depth-tested before shading; what the console does then was not measured. +- [DB_SHADER_CONTROL.PRIMITIVE_ORDERED_PIXEL_SHADER](../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - the host pixel interlock spans the whole shader, not only up to `s_sendmsg` ORDERED_PS_DONE. +- [POPS collision SGPRs](../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - the collision wave ID and intrawave collision mask read 0, so a title skips its POPS wait and runs its per-collision loop once. +- [Image loads](../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) (shader recompiler) - an image_load reads a sampled binding, also of an image the shader stores to, so in a primitive-ordered pixel shader it misses an earlier overlapping primitive's image_store in the same draw. +- [PA_SC_CONSERVATIVE_RASTERIZATION_CNTL](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - 0x6001 uses Vulkan overestimation on devices that add at most 1/256 pixel and keep degenerate triangles; the console's uncertainty region and degenerate handling are unmeasured. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. - The libc [SSE4a trap emulation](../../core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp) on Windows is superseded by `--to-intel` and remains only until the relinked title has been verified without it. -- [DCC display buffers](../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) are presented only from uniform uncompressed or fast-clear keys; other keys, and register-clear keys over a pending image, throw. Keys are read as one byte per 256 bytes (the driver's model), not in the console's layout. +- [Descriptor pool chain](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - each pool holds two storage buffer descriptors more than a set can take, as a workaround: RADV (Mesa 26.0 and later) lets a `FREE_DESCRIPTOR_SET` pool place a set up to 32 bytes past the pool's buffer ([`util_vma_heap_init`](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/vulkan/radv_descriptor_pool.c) is given `bo_size + RADV_POOL_HEAP_OFFSET` as a size), and with a page-multiple pool size those bytes are an unmapped page +- [CMASK fast clears](../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) - only a CMASK whose nibbles are all fast-cleared (0) or all expanded (0xF) is modeled, checked over the target's whole CMASK, 4 KiB per 1024x512-pixel metablock (the size PS5 titles clear, for every color format seen); the addressing within a metablock is not decoded. The first draw or fast-clear eliminate over a fast-cleared CMASK fills the whole target with `CB_COLOR_CLEAR_WORD` and stores those bytes as expanded, so a draw that covers only part of the target reads the clear value in the rest as the eliminate would leave it. On a DCC target the clear is in the DCC keys, so its CMASK must be all expanded. A mixed CMASK throws, as does a DCC target's CMASK that is not all expanded and a fast clear of a mipmapped, 3D or array color target +- [Host imports](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver) on Linux skip ranges that are not mapped read-write, which are copied instead. The NVIDIA driver (595.91.07) refuses `VK_EXT_external_memory_host` imports of read-only or executable mappings (and shared mappings of regular files) with `VK_ERROR_OUT_OF_DEVICE_MEMORY`, after which every `vkAllocateMemory` on the device fails the same way. Read-only ranges are never imported, even on drivers that accept them. Shared mappings of regular files are not checked for. +- [Cube texture descriptors](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) with equal `BASE_ARRAY` and `DEPTH` expose six faces starting at `BASE_ARRAY`. Measured with `image_load`, `image_sample` and `image_get_resinfo` on RX 6600 (gfx1032), using 64x64 R32_UINT faces and base slices 0, 5 and 6. Other ranges containing fewer than six faces or incomplete cube arrays still throw: hardware load bounds and filtered-sample addressing differ. Seamless cube filtering and out-of-range cube-array sampling remain unimplemented. PS5 equivalence has not been measured on console hardware. +- [DCC display buffers](../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) are presented only when all keys of the console's DCC extent agree on uncompressed or a fast-clear code, padding keys included; other keys, and register-clear keys over a pending image, throw. +- [DCC key stores](../../core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp) cover the console's DCC extent only for pipe-aligned DCC of single-level, single-layer 4-byte SW_64KB_R_X surfaces; other surfaces, and every texture and color target key read, cover one byte per 256 bytes. +- [8_SRGB color targets](../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) on a device that cannot attach and blend R8_SRGB render into an RGBA8_SRGB copy of the target, so the sRGB encode, decode and blend rounding are the device's; the console's are not measured. +- [8_SRGB color targets](../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) on such a device throw unless they are single-mip 2D surfaces with a resident image; linear targets have none. +- [COPY_DATA from the GPU clock](../../core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp) (libSceAgcDriver, `SRC_SEL` 9) writes the host steady clock in 10 ns ticks when the packet runs, the same counter as the `RELEASE_MEM` timestamp, not the console's GPU clock. The `COPY_DATA` cache policy fields are accepted and have no effect. - `--to-intel` guest module trampolines are covered only by a synthetic relinker test; no game title has been verified with them on Linux or Windows. -- `--to-intel` replaces the approximate VEX.128 register forms of VRSQRTPS and VRCPPS with [correctly rounded](../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). The 256-bit, scalar, legacy and memory forms keep the native instruction, as does a 4-byte site with no movable instruction after it for the jump (5 sites in PPSA21564). +- `--to-intel` replaces the approximate VEX.128 register forms of VRSQRTPS and VRCPPS with [correctly rounded](../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). The 256-bit, scalar, legacy and memory forms keep the native instruction, as does a 4-byte site with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - [sceKeyboardGetKey2Char](../../core/libs/prx/libSceKeyboard/src/keyboard_impl.cpp) (libSceKeyboard) translates the 101-key (US) arrangement and the alphanumeric layer of the 106-key (Japanese) one, which follows the JIS layout of the HID usages: the Yen (0x89) and Ro (0x87) keys return `\` (0x5C) unshifted, as JIS X 0201 places the yen sign there; whether the console returns U+00A5 instead is unverified. With the Kana LED (bit 4, the HID position) set the 106-key arrangement throws, as the kana the console returns are unknown. Ctrl and Alt do not change the character. -- [sceAudioOut2Set3DLatency](../../core/libs/prx/libSceAudioOut/src/AudioOut2System.cpp) and [sceAudioOut2MasteringInit and sceAudioOut2MasteringSetParam](../../core/libs/prx/libSceAudioOut/src/AudioOut2Mastering.cpp) (libSceAudioOut) have no effect: there is no 3D rendering or mastering stage. +- [sceAudioOut2Set3DLatency](../../core/libs/prx/libSceAudioOut/src/AudioOut2System.cpp) and [sceAudioOut2MasteringInit, sceAudioOut2MasteringTerm and sceAudioOut2MasteringSetParam](../../core/libs/prx/libSceAudioOut/src/AudioOut2Mastering.cpp) (libSceAudioOut) have no effect: there is no 3D rendering or mastering stage. - [sceImeKeyboardGetInfo](../../core/libs/prx/libSceIme/Export.cpp) (libSceIme) returns `0x80bc0023` (no resource id) for every id while a keyboard is open, since the IME keyboard never reports a connected device. [sceImeKeyboardSetMode](../../core/libs/prx/libSceIme/Export.cpp) validates the mode bits but the mode has no effect, as no IME keyboard events are delivered. - [AudioOut2 contexts](../../core/libs/prx/libSceAudioOut/src/AudioOut2Context.cpp) (libSceAudioOut) with an SDL device read a grain's port buffers at the next push, at least 100 us after the grain was pushed, and play the mix behind a 40 ms cushion that the queue level leaves out. When the hardware reads the buffers is not known. The timing follows a title whose mixer fills a buffer after pushing it, reuses it two pushes later, and may still be writing it when the next push follows within microseconds. The last grain pushed before a title stops pushing is never read or played. - [libSceAudiodec](../../core/libs/prx/libSceAudiodec/Export.cpp) throws for the 24-bit PCM word size (`iBwPcm` 0): its sample layout is unknown. ATRAC9 decoding is covered only by configuration and error tests, as no ATRAC9 encoder is available for a fixture. - [sceAjmDecMp3ParseFrame](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm.native) throws when the original file length is requested (`parse_ofl` non-zero) for a layer other than III, for a CRC-protected frame, or for a VBRI header outside MPEG-1 stereo: shadPS4 looks for the tag after the CRC and after the side information, while LAME and FFmpeg place it without the CRC and VBRI always at byte 36. -- The [output modifier](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) (`mul:2`, `mul:4`, `div:2`) follows the hardware with the IEEE mode off and f32 denormals flushed, the LLVM graphics default; the shader's MODE register is not read, and the hardware ignores the modifier in the other modes. On f16 results it is ignored, as on the hardware while f16 denormals are enabled; with them flushed the hardware applies it. +- The [output modifier](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) (`mul:2`, `mul:4`, `div:2`) follows the shader's float mode as measured on RDNA2 (gfx1035): the hardware applies it only with IEEE mode off and output denormals of the result's precision flushed (bit 1 of the `FLOAT_MODE` denormal field clear), and ignores it otherwise. On f32 results it is applied as the hardware does with f32 denormals flushed; a shader without a known float mode is translated with IEEE mode off and f32 denormals flushed, the mode titles use. On f16 and f64 results it is ignored, and it throws in the modes where the hardware applies it. +- The [clamp modifier](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follows `DX10_CLAMP`, measured on RDNA2 (gfx1035) for f32, f16, packed f16 and f64 results at IEEE=0 and IEEE=1: with `DX10_CLAMP`=1 (and in shaders without a known float mode) a NaN result clamps to +0; with `DX10_CLAMP`=0 a NaN result is written unchanged and every other value clamps to [0, 1] as before. f64 results follow the same rule on hardware, but a clamped f64 result with `DX10_CLAMP`=0 throws until it has an execution test. With `DX10_CLAMP`=0 the NaN's bits are those the translation already produces for the instruction without clamp, so where those are left to the driver (f32 and scalar f16 arithmetic) the payload is the driver's, not the hardware's input propagation. `v_dot2_f32_f16` and the integer clamps are unchanged. - [Comparison sampling of a color texture](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (`image_sample_c`/`image_sample_c_lz` on a texture that is not R32 float or R16 unorm) is emulated in the shader, since Vulkan compares only depth formats: the red channel is fetched, compared and, for bilinear filtering, blended (compare, then filter). Only 2D and 2D array views, float/unorm/snorm formats, wrap, clamp-to-edge and clamp-to-border addressing (border red 0 for the black borders, 1 for opaque white) and point or bilinear filtering at one level are implemented, with texel offsets added to the texel indices; LOD biases and clamps are accepted, as they cannot select another level of a single-level sample. Half-border addressing, the border color table, gradients, gathers, other dimensions, anisotropy and mip filtering throw. Under bilinear filtering, taps past a clamp-to-border edge compare the border value and blend with the texel taps, so results near the edge depend on the filter weights. AMD documentation does not say how a color format's reference is handled: it is clamped to [0, 1] for unorm and [-1, 1] for snorm, as Vulkan does for unorm depth, and not clamped for float. Bilinear weights are full float, not the hardware's fixed-point subtexel weights - [libScePlayerInvitationDialog](../../core/libs/prx/libScePlayerInvitationDialog/libScePlayerInvitationDialog.cpp) simulates dialog completion without displaying UI or sending invitations, so `scePlayerInvitationDialogGetResult` reports `USER_CANCELED`; its parameter ABI remains unverified. The 40-byte result layout (error code, result, 32 reserved bytes) is taken from [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Dialog/PlayerInvitationDialog.cs) and the error codes from the other common dialogs, not confirmed on a title. - [Converted unorm images](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (10_11_11_UNORM, read and written through an R32_UINT view): samples, gathers, LOD queries, 16-bit data and a DST_SEL of the missing W channel throw. - [Converted unorm image loads](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) divide by 2^n - 1 with the buffer format path's `OpFDiv`, which Vulkan allows to be up to 2.5 ULP off, so some drivers are not bit-exact. - [64-bit LDS atomics](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_*_u64`, `ds_*_b64`, `ds_*_f64`) read and write the two dwords under a workgroup lock stored after the guest LDS, as LDS is a 32-bit array without 64-bit atomics. They are atomic with respect to each other only: 32-bit accesses to the same dwords issued concurrently are not ordered by the lock. 64-bit GDS atomics throw. `ds_min/max(_rtn)_f64` and `ds_cmpst(_rtn)_f64` follow the NaN and signed-zero rules measured for the f32 forms; the f64 forms were not measured on hardware. +- [V# bases off a DWORD boundary](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - raw accesses use the exact bytes and `s_buffer_load` reads `(base & ~3) + (offset & ~3)`, as measured on gfx1035 (Linux, alignment mode UNALIGNED; the console's mode is unknown). Formatted accesses off their element's alignment and atomics throw. +- [Vector buffer offsets that are not a multiple of 4](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - dword accesses count whole dwords from the V# base, where the hardware reads the exact bytes at base + offset. - [Min/max S# reduction](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) - `FILTER_MODE` 1 and 2 use Vulkan's min and max reduction (the same `SQ_IMG_FILTER_MODE` values RADV programs). Measured on RDNA2 (gfx1035), not on the console: only texels of non-zero weight are reduced, and the weight is taken after rounding to the 8-bit subtexel grid (a sample within 1/512 texel of a centre reduces that texel alone); a driver with other `subTexelPrecisionBits` reduces a different set there. Gathers ignore the mode. - [Min/max S#](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver, shader recompiler) - throws with anisotropy, a linear mip filter, depth or color-texture compare, or bilinear filtering of an image the recompiler point-samples (sint, converted, depth-bits). - [Raw `buffer_load_dwordx2`](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) is one 64-bit access on the hardware (with or without glc), so it never sees half of a `buffer_atomic_swap_x2`. Only a glc load that is 8-byte aligned and in bounds, in a program that has the 64-bit storage view (it uses 64-bit buffer atomics), is read with one 64-bit atomic load; every other dwordx2 load is two dword loads and can tear against a concurrent 64-bit write: loads without glc, and loads in a program without 64-bit buffer atomics while another shader writes the pair with `swap_x2`. +- [DS src2 operations](../../core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp) (`ds_*_src2_*`, GFX10.1 only) read `DS[B]`, with `B = A + 4 * sext(offset1[6:0]:offset0)`, and then update `DS[A]` with the matching 32-bit atomic, so the read of `B` is not atomic with the update of `A`. The form that takes the offset from the address (`offset1[7]` set), the 64-bit forms and GDS throw: whether the LDS address keeps `A[31:17]` in that form, and whether the 64-bit forms scale the offset by 4 or 8, is not documented. `ds_min_src2_f32`, `ds_max_src2_f32` and `ds_add_src2_f32` throw: on RDNA2 (gfx1035, the oracle's hardware, which has no src2 opcodes) `ds_min_f32`, `ds_max_f32` and `ds_add_f32` ignore `IEEE_MODE` but depend on whether f32 denormals are flushed (28, 28 and 12 of 196 special rows differ), and that is not modelled for LDS float atomics yet (#981). - [wcsrtombs_s](../../core/libs/prx/libc/src/Multibyte.cpp) (libc) follows the console's `libc.prx`, but a runtime-constraint violation only returns its error: the default handler there writes the message and a newline to `stderr` before returning, which is not done here. Conversion uses the C locale of `wcrtomb` (`MB_CUR_MAX` 1), so the path where a multibyte character does not fit the remaining space is not reached +- [The narrow printf family](../../core/libs/prx/libc/src/Formatting.cpp) (libc) passes `%ls` and `%S` to glibc on Linux, which reads them as strings of 32-bit `wchar_t`; the guest's `wchar_t` is 16 bits (UTF-16), so these conversions fail with -1. The Windows path converts them to UTF-8, and `guest_formatting` checks wide conversions on Windows only - [collate::_Getcat](../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) builds the facet for the C locale only and throws for any other locale name. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size, and for results of 4095 characters or more, whose string storage uses the guest's over-aligned large allocation. +- [socketpair](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) - the pair is the host's `AF_UNIX` pair on Linux and, on Windows, a loopback TCP connection for `SOCK_STREAM` or two connected UDP sockets for `SOCK_DGRAM`; `SOCK_SEQPACKET` throws on Windows. A datagram sent to a full receiver gets `ENOBUFS` on the console, waits or gets `EAGAIN` on Linux and is dropped on Windows, and on Windows a datagram `recv` after the peer closed reports `ECONNRESET` instead of the console's `ENOTCONN`, and a stream `send` after the peer closed reports `ECONNABORTED` instead of the console's `EPIPE`. `read` and `write` on a pair end go to the host like on every guest socket. `getpeername` on a pair end succeeds after the peer closed, where the console returns `ENOTCONN`, and `connect`, `bind` and `sendto` with an address on a pair end report an address-family error instead of the console's `EISCONN`. On Windows `poll` on a stream end reports `POLLHUP` without `POLLIN` after the peer's `shutdown(SHUT_WR)` or `close`, where the console reports `POLLIN`, with `POLLHUP` only after `close` +- [poll](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) waits only on guest sockets, through `WSAPoll` on Windows and `poll` on Linux; the `pollfd` layout and flag values are FreeBSD's. A descriptor below the socket range (files, pipes, stdio) throws, as do event bits other than `POLLIN`, `POLLPRI`, `POLLOUT`, `POLLRDNORM`, `POLLRDBAND` and `POLLWRBAND` (the `POLLERR`, `POLLHUP` and `POLLNVAL` bits are ignored in `events`), `POLLPRI` on Windows, which `WSAPoll` rejects, and an infinite timeout with no socket to wait on. A closed socket descriptor reports `POLLNVAL`; the console's limit on the descriptor count is unknown and not checked - [Thread exit destructors](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) run when a libkernel thread finishes; on the main or a host thread only `_sceLibcInternalThreadDtors` runs them, as `exit` does, else they are dropped - [sceFontOpenFontSet](../../core/libs/prx/libSceFont/src/SystemFont.cpp) (libSceFont) opens only the primary file of a system font set. Characters the console takes from the set's other files (Hangul, Thai and Arabic in Japanese and Chinese sets, Arabic in European sets) are reported as unsupported glyphs. The console file names are those of the PS4 firmware; the names of the PS5-only JG2 sets (`0x1A......`) are unknown, so they use the SSTJpPro files. When a console file is missing, an openly licensed substitute (Noto Sans, Noto Sans Mono, Noto Sans Thai, Noto Sans CJK) is loaded instead. It is not the system font: its metrics differ, so text width, line height and wrapping differ from the console. System font faces are loaded by FreeType in host memory, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for. -- [Font writing lines](../../core/libs/prx/libSceFont/src/WritingLine.cpp) (libSceFont) - `sceFontCreateWritingLine`, `sceFontWritingLineWritesOrder`, `sceFontWritingLineRefersRenderStep`, `sceFontWritingLineGetRenderMetrics` and `sceFontWritingLineClear` follow [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/main/src/libs/libFont.cpp): runs are placed left to right, each at the line's advance so far, with no spacing or adjustment. Only horizontal and left-to-right lines, writing attribute 0 and non-null run metrics are modelled; other values, the metrics of an empty line and unknown or null handles throw, since no reference gives their result or error code. `sceFontWritingLineGetOrderingSpace` still throws (KytyPS5's values are placeholders). Lines are allocated on the host, not in the font memory +- [Font writing lines](../../core/libs/prx/libSceFont/src/WritingLine.cpp) (libSceFont) - `sceFontCreateWritingLine`, `sceFontWritingLineWritesOrder`, `sceFontWritingLineRefersRenderStep`, `sceFontWritingLineGetRenderMetrics` and `sceFontWritingLineClear` follow [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/main/src/libs/libFont.cpp): runs are placed left to right, each at the line's advance so far, with no spacing or adjustment. Only horizontal and left-to-right lines, writing attribute 0 and non-null run metrics are modelled; other values, the metrics of an empty line and unknown or null handles throw, since no reference gives their result or error code. `sceFontWritingLineGetOrderingSpace` still throws (KytyPS5's values are placeholders). Lines are allocated on the host, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for. +- [s_swappc_b64](../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) writes the link pair with the same model as `s_getpc_b64`: the guest return address `pc + 4` as a byte offset over `GetShaderBase`. Any other use of the link pair (arithmetic, compare, hash, store, or a second `s_swappc_b64` sharing it) throws; constant-offset folding over the link is not modeled. The special encoding `s_swappc_b64` with SDST 125 still decodes as `s_setpc_b64`, keeping the existing decode quirk. +- The [s_swappc_b64 fetch-shader call at vertex/local/tessellation/mesh entry](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) assumes the called fetch-shader blob ends in `s_setpc_b64` to the link pair (the GNM fetch-shader shape shadPS4's `EmitFetch` copies); the blob is not part of the shader program, so its return closes the pairing by assumption and any in-program reader of the link pair throws. Attribute fetches are inlined from the request's parsed fetch-shader plan through the same `GetAttribute` path as the embedded fetch analyzer, honoring the buffer resource `dst_sel`; an attribute writing more than four registers, or an entry call without a parsed plan, throws. The vertex/instance offset accumulator inside the fetch blob is not replayed for external blobs. - [sceImeKeyboardGetResourceId](../../core/libs/prx/libSceIme/Export.cpp) (libSceIme) reports that no USB keyboard is connected (`0x80bc0004`, as [shadPS4](https://github.com/shadps4-emu/shadPS4) does) because IME keyboard events are not delivered to the handler, although [libSceKeyboard](../../core/libs/prx/libSceKeyboard/src/keyboard_impl.cpp) exposes the host keyboard - [Player review dialog](../../core/libs/prx/libSceCdlgPlayerReview/Export.cpp) (libSceCdlgPlayerReview) shows nothing: `scePlayerReviewDialogOpen` finishes at once and the result reports that the user canceled, as `sceWebBrowserDialogOpen` does +- [NP reachability callbacks](../../core/libs/prx/libSceNpManager/Export.cpp) retain one callback and its user data; duplicate registration and unregistering an absent callback return the errors used by [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/945dbc3cc3eee80ac3e053b438502ed936fa6bb2/src/core/libraries/np/np_manager.cpp). No callback is delivered because the existing NP model remains signed out with reachability unavailable. +- [Running application ID](../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - `sceSystemServiceGetAppIdOfRunningBigApp` reports 1 as the running title's app ID. The value is AnyPS5's own: the console's shell assigns app IDs and their values are not known here, so other calls that take an app ID must use the same one. `sceSystemServiceKillApp` with that ID and the arguments -1, 0 and 0, which [SwanStationPS5](https://github.com/darkxex/SwanStationPS5/blob/4cdc981d1a64ffb7f2af73f7a27a9e91a3cb5eb4/src/main.c#L1050-L1058) passes to close itself and [ps5-payload-dev's hbldr](https://github.com/ps5-payload-dev/websrv/blob/46374edee6627885abac717de24da7a1d73af6f1/src/ps5/hbldr.c#L425-L435) passes to close the running game, ends the title as its own `exit(0)` would: the guest's shutdown and `atexit` handlers run and the process exits with status 0. On the console the call returns and the shell then ends the title, and whether those handlers run there is unknown; here there is no shell, so the call does not return. Another app ID or other arguments throw. +- [Temporary app-content storage](../../core/libs/prx/libSceAppContent/Export.cpp) tracks one `/temp0` mount, preserves its files on unmount, and rejects new path resolutions until remount. Mount, format, space-query and unmount errors follow [fpPS4](https://github.com/red-prig/fpPS4/blob/04cefd43e6fddd1ab033e7980cd356d14c964905/sys/sys_path.pas); the pointer ABI is also declared by [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/AppContent/AppContent.cs). Existing open host descriptors remain usable after unmount; console behaviour with open files is unverified. Host filesystem failures still throw. +- [Guest path resolution](../../core/libs/prx/libc/src/General.cpp) falls back to ASCII case-insensitive component lookup, following [shadPS4's host filesystem](https://github.com/shadps4-emu/shadPS4/blob/945dbc3cc3eee80ac3e053b438502ed936fa6bb2/src/core/file_sys/backends/host_fs.cpp). PPSA03747 requests `Skyrim.INI`, `Data/Video` and lowercase archive/plugin names that differ from the dump's filenames. Exact host names take precedence; multiple fallback matches throw instead of selecting an arbitrary file. Existing parent names are resolved when creating a new file, and guest mount aliases use the same ASCII comparison. Results are not cached, so renames and newly created files remain visible. Unicode case folding and PS5 behaviour for case-colliding host entries are unverified; host roots are used as supplied. +- [Static scalar calls](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) (`s_call_b64` and `s_swappc_b64`) require one call per ordinary even-aligned SGPR link pair (s0:s1 through s104:s105) and one later `s_setpc_b64` return reading that exact pair. Regions must nest or be disjoint; recursion, shared links, escaping link values (including overlapping tuples and resource descriptors), invalid targets, interleaved regions, cross-region branches or fall-through, relative SGPR reads and dynamic `s_setpc_b64` in a program with calls throw. The analysis checks all decoded instructions, including unreachable code. Arbitrary indirect calls and general call frames are not implemented. +- [s_call_b64](../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) saves `PC + 4` over `GetShaderBase` and branches to `PC + 4 + signext(SIMM16) * 4`, following [AMD RDNA2 ISA](https://gpuopen.com/download/rdna2-shader-instruction-set-architecture.pdf) sections 3.6.3, 12.2 and 12.3. Forward callees after `s_endpgm` are decoded. Native RDNA2/PS5 execution and title compatibility for these bounded scalar calls remain unverified; `s_rfe_b64` still throws in translation. +- [sceVideodec2CreateDecoder](../../core/libs/prx/libSceVideodec2/Export.cpp) (libSceVideodec2) decodes HEVC (codec type 974921) and VP9 (2382845) through FFmpeg, besides H.264. The tags come from SharpProspero's `Videodec2CodecType` and EVO-PLAYER-PS5's `sce_videodec2.h`, which records them from hardware runs. HEVC input is converted from length-prefixed NAL units to Annex B as for H.264, and VP9 frames are passed through. Output is the same 8-bit NV12 for every codec; a decoder for HEVC Main10 or VP9 Profile 2 is created, but a frame deeper than 8 bits throws, since the console's 10-bit output layout is unknown, and `QueryDecoderMemoryInfo` reports the 8-bit frame size for it. `sceVideodec2GetPictureInfo` and `sceVideodec2GetAvcPictureInfo` throw for HEVC and VP9 pictures, which use other layouts (`SceVideodec2HevcPictureInfo`, `SceVideodec2Vp9PictureInfo`), and `sceVideodec2GetHevcPictureInfo` and `sceVideodec2GetVp9PictureInfo` are not exported diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index 785c57379..8405c9cd6 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -70,6 +70,8 @@ app0/ Use `sce_modules/` or `prx/` instead of `sce_module/` if that is the input directory name. Relinker preserves each module's directory under `app0/` and prints its exact path. Place app resources in `app0/` separately. Copy the built system libraries from `build/core/libs/libs/*.prx` into `libs/`; use libraries built for the target OS. A custom `--rpath` changes the system library location. +Use the generated files printed as `Guest module:` for bundled title modules. `libs/` is for AnyPS5 system libraries, not the original PS5 `.prx` files. Placing an original PS5 module in `libs/` on Windows makes Windows try to load it as a DLL and can fail with error 193 (not a valid Win32 application). + On Windows, direct memory (`sceKernelAllocateDirectMemory`, up to 13824 MiB per title) is committed in full when the title allocates it, not when its pages are first used. The system commit limit (installed memory plus page file size, the second value of Committed in Task Manager) must cover it together with all other committed memory. Otherwise the allocation throws `create direct memory backing of 0x bytes ( MiB)` with the Windows error; enlarge the page file or close other applications. Linux: @@ -96,3 +98,35 @@ The game runs on the first Vulkan 1.1 device with graphics and compute queues an ## Exit codes `0`: conversion succeeded. `1`: invalid arguments. `2`: conversion failed; the error is printed to stderr. With `--autorun`, successful conversion returns the launched application's exit code. + +## Import audit + +[`tools/import_audit.py`](../../tools/import_audit.py) lists the system functions a converted game imports and whether the built libraries provide them, before the game is launched. It needs Python 3 and nothing else. + +```sh +relinker --registry source/input.elf app.elf +python3 tools/import_audit.py app.registry.json --libs build/core/libs/libs --modules source/sce_module +``` + +`--registry` writes `app.registry.json` beside the output. `--libs` is the directory the `libs` target fills ([build instructions](../dev/BUILD.md)); the exports are read from the built `.prx` files, so the result matches what the loader finds. Imports are counted once per NID and library, and every reference lands in exactly one class: + +| Class | Meaning | +|-------|---------| +| `implemented` | A built library exports the NID and its function is not a throwing stub. | +| `stub` | Exported, but the whole function calls `NotImplemented_nid_no_patch`: the game loads and throws when it calls it. | +| `absent` | No built library exports the NID: the loader fails. | +| `module` | The import names a file found in `--modules` (the title's own library); its exports are not checked. | + +The report also lists needed libraries that have no file in `--libs` or `--modules`, and imports exported only by a library other than the one they name; those resolve on Linux and can fail on Windows. + +| Option | Effect | +|--------|--------| +| `--libs ` | Built `.prx` directory. Required; repeat for several. | +| `--modules ` | Directory of the title's own modules; repeat for several. | +| `--source ` | `core/libs/prx` tree that is searched for throwing stubs. Defaults to this repository's. | +| `--names ` | NID and name per line, as in `aerolib.csv` from [`tools/nid_names.py`](../../tools/nid_names.py); names the absent imports. Never downloaded; a name that does not hash to its NID is marked. | +| `--json ` | Full result, stamped with the repository's commit and whether its tree was modified. | + +Exit codes: `0` nothing blocks loading, `1` there are `absent` imports or needed libraries without a file, `2` an input could not be read; the message names the file and the value. + +The registry lists the imports of the executable, not of its bundled modules. diff --git a/tools/check_conventions.py b/tools/check_conventions.py index 933dec172..78a136d13 100644 --- a/tools/check_conventions.py +++ b/tools/check_conventions.py @@ -1,8 +1,10 @@ import argparse import json import os +import posixpath import re import subprocess +import sys from collections import defaultdict from pathlib import PurePosixPath @@ -114,7 +116,7 @@ class Check: if not exports: return pattern = r"\bAPS5_VABI\s+(" + "|".join(exports) + r")\s*\(" - result = subprocess.run(["git", "grep", "-nE", pattern, self.head, "--", "core/libs/prx"], capture_output=True, text=True) + result = subprocess.run(["git", "grep", "-nE", pattern, self.head, "--", "core/libs/prx"], capture_output=True, encoding="utf-8", errors="replace") for line in result.stdout.splitlines(): _, path, _, text = line.split(":", 3) name = EXPORT.search(text).group(1) @@ -186,7 +188,7 @@ def links(path, text): if re.match(r"[a-z]+:|#", target): continue target = target.split("#", 1)[0].split("?", 1)[0] - result.append((target, os.path.normpath(target.lstrip("/") if target.startswith("/") else str(PurePosixPath(path).parent / target)))) + result.append((target, posixpath.normpath(target.lstrip("/") if target.startswith("/") else str(PurePosixPath(path).parent / target)))) return result @@ -244,7 +246,7 @@ def output(check, repo): rows.append(f"| {level} | [{rule}](https://github.com/{repo}/blob/main/{doc}) | {where} | {cell} |") summary = os.environ.get("GITHUB_STEP_SUMMARY") if summary: - with open(summary, "a") as out: + with open(summary, "a", encoding="utf-8") as out: if rows: out.write("| Level | Rule | Where | What |\n|---|---|---|---|\n" + "\n".join(rows) + "\n") else: @@ -255,6 +257,10 @@ def output(check, repo): if __name__ == "__main__": + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + if hasattr(sys.stderr, "reconfigure"): + sys.stderr.reconfigure(encoding="utf-8", errors="replace") parser = argparse.ArgumentParser(description="check a branch against the contributing rules; errors fail the check") parser.add_argument("--base", default="origin/main", help="branch the pull request targets (default: origin/main)") parser.add_argument("--head", default="HEAD") @@ -268,9 +274,9 @@ if __name__ == "__main__": check.repository() check.docs() check.commits() - pr = json.load(open(args.event)).get("pull_request") if args.event else None + pr = json.load(open(args.event, encoding="utf-8")).get("pull_request") if args.event else None if pr: check.pull_request(pr["title"], pr["body"]) elif args.title is not None or args.body: - check.pull_request(args.title or "", open(args.body).read() if args.body else "") + check.pull_request(args.title or "", open(args.body, encoding="utf-8").read() if args.body else "") raise SystemExit(output(check, os.environ.get("GITHUB_REPOSITORY", "boykopovar/AnyPS5"))) diff --git a/tools/check_test_registration.py b/tools/check_test_registration.py new file mode 100644 index 000000000..a0115b2cd --- /dev/null +++ b/tools/check_test_registration.py @@ -0,0 +1,77 @@ +import argparse +import re +import subprocess +from pathlib import Path + +TEST_OUTPUT = r"tests[\\/]" + +KNOWN_UNRUN = { + "agc_driver_recorder_tests", + "video_out_flip_tests", +} + + +def built(build): + listed = subprocess.run(["ninja", "-C", str(build), "-t", "targets", "all"], + capture_output=True, text=True, check=True).stdout + names = set() + for line in listed.splitlines(): + path, _, rule = line.partition(": ") + if "EXECUTABLE" not in rule: + continue + found = re.match(rf"^{TEST_OUTPUT}([A-Za-z0-9_.]+)$", path) + if not found: + continue + name = found.group(1).rstrip(".") + if name.endswith(".exe"): + name = name[:-4] + if name: + names.add(name) + return names + + +def registered(build): + names = set() + for file in Path(build).rglob("CTestTestfile.cmake"): + for line in file.read_text(errors="replace").splitlines(): + found = re.match(r"\s*add_test\s*\((.*)\)\s*$", line) + if not found: + continue + for token in re.findall(r'"([^"]*)"|(\S+)', found.group(1)): + argument = token[0] or token[1] + base = re.split(r"[\\/]", argument)[-1] + if base.endswith(".exe"): + base = base[:-4] + if base: + names.add(base) + return names + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="fail when a test executable is built that no ctest test runs") + parser.add_argument("build", type=Path, nargs="?", default=Path("build"), help="configured build directory") + parser.add_argument("--list", action="store_true", help="print every test executable and whether ctest runs it") + args = parser.parse_args() + every = sorted(built(args.build)) + run = registered(args.build) + if args.list: + for name in every: + print(f"{'run' if name in run else 'NOT RUN'} {name}") + raise SystemExit + unrun = sorted(set(every) - run) + if not every: + print(f"no test executables found under {args.build}; is it configured with -DBUILD_TESTING=ON?") + raise SystemExit(1) + for name in sorted(set(unrun) & KNOWN_UNRUN): + print(f"note: {name} is built and not run, as recorded") + stale = sorted(KNOWN_UNRUN & run) + for name in stale: + print(f"error: {name} is run by ctest now; remove it from KNOWN_UNRUN") + fresh = sorted(set(unrun) - KNOWN_UNRUN) + for name in fresh: + print(f"error: {name} is built but no ctest test runs it") + if fresh: + print(f"{len(fresh)} test executable(s) would never run: ctest reports a full pass over what it knows, so a test nobody registered is invisible in CI") + if stale or fresh: + raise SystemExit(1) + print(f"{len(every)} test executable(s), {len(unrun)} built and not run as recorded, none new") diff --git a/tools/hw-oracle/hw_oracle.py b/tools/hw-oracle/hw_oracle.py index eee171f50..b3d6e4ed7 100644 --- a/tools/hw-oracle/hw_oracle.py +++ b/tools/hw-oracle/hw_oracle.py @@ -36,12 +36,15 @@ def oracle(): if binary.exists() and binary.stat().st_mtime >= source.stat().st_mtime: return binary CACHE.mkdir(parents=True, exist_ok=True) - command = [os.environ.get("CC", "cc"), "-O1", str(source), "-o", str(binary)] - root = rocm_root() - if root: - lib = root / "lib" - command += [f"-I{root / 'include'}", f"-L{lib}", f"-Wl,-rpath,{lib}"] - subprocess.run(command + ["-lhsa-runtime64"], check=True) + with tempfile.TemporaryDirectory(dir=CACHE) as tmp: + output = Path(tmp) / "oracle" + command = [os.environ.get("CC", "cc"), "-O1", str(source), "-o", str(output)] + root = rocm_root() + if root: + lib = root / "lib" + command += [f"-I{root / 'include'}", f"-L{lib}", f"-Wl,-rpath,{lib}"] + subprocess.run(command + ["-lhsa-runtime64"], check=True) + output.replace(binary) return binary @@ -50,18 +53,26 @@ def target(): return os.environ.get("HW_ORACLE_TARGET") or subprocess.run([oracle(), "--target"], capture_output=True, text=True, check=True).stdout.strip() -def assemble(body, work, wave64, ieee, denorm32, denorm16): +def assemble(body, work, wave64, *, ieee, denorm32, denorm16, dx10_clamp, round32, round16, fp16_overflow): + modes = (("ieee", ieee, 2), ("denorm32", denorm32, 4), ("denorm16", denorm16, 4), + ("dx10_clamp", dx10_clamp, 2), ("round32", round32, 4), ("round16", round16, 4), + ("fp16_overflow", fp16_overflow, 2)) + for name, value, limit in modes: + if not isinstance(value, int) or value not in range(limit): + raise ValueError(f"{name} must be an integer from 0 to {limit - 1}") text = (HERE / "template.s").read_text() for key, value in (("@TARGET@", target()), ("@WAVE32@", "0" if wave64 else "1"), ("@WAVESIZE@", "64" if wave64 else "32"), - ("@DENORM@", str(denorm32)), ("@DENORM16@", str(denorm16)), ("@IEEE@", str(ieee)), ("@BODY@", body)): + ("@DENORM@", str(int(denorm32))), ("@DENORM16@", str(int(denorm16))), ("@IEEE@", str(int(ieee))), + ("@DX10_CLAMP@", str(int(dx10_clamp))), ("@ROUND32@", str(int(round32))), ("@ROUND16@", str(int(round16))), + ("@FP16_OVERFLOW@", str(int(fp16_overflow))), ("@BODY@", body)): text = text.replace(key, value) (work / "k.s").write_text(text) - subprocess.run([tool("clang"), "-x", "assembler", "-target", "amdgcn-amd-amdhsa", f"-mcpu={target()}", "-c", str(work / "k.s"), "-o", str(work / "k.o")], check=True) + subprocess.run([tool("clang"), "-x", "assembler", "-target", "amdgcn-amd-amdhsa", f"-mcpu={target()}", *(["-mwavefrontsize64"] if wave64 else []), "-c", str(work / "k.s"), "-o", str(work / "k.o")], check=True) subprocess.run([tool("ld.lld"), "-shared", str(work / "k.o"), "-o", str(work / "k.co")], check=True) return work / "k.co" -def run(body, rows, extra=b"", wave64=False, ieee=1, denorm32=3, denorm16=3, coarse=False): +def run(body, rows, extra=b"", wave64=False, coarse=False, *, ieee, denorm32, denorm16, dx10_clamp, round32, round16, fp16_overflow): rows = [tuple(r) for r in rows] wave = 64 if wave64 else 32 padded = rows + [(0, 0, 0, 0)] * (-len(rows) % wave) @@ -72,7 +83,8 @@ def run(body, rows, extra=b"", wave64=False, ieee=1, denorm32=3, denorm16=3, coa out = [] with tempfile.TemporaryDirectory() as tmp: work = Path(tmp) - code = assemble(body, work, wave64, ieee, denorm32, denorm16) + code = assemble(body, work, wave64, ieee=ieee, denorm32=denorm32, denorm16=denorm16, + dx10_clamp=dx10_clamp, round32=round32, round16=round16, fp16_overflow=fp16_overflow) for start in range(0, len(padded), ROWS_PER_DISPATCH): chunk = padded[start:start + ROWS_PER_DISPATCH] (work / "in.bin").write_bytes(b"".join(struct.pack("<4I", *r) for r in chunk) + extra) @@ -87,9 +99,13 @@ def main(): parser.add_argument("body", type=Path, help="assembly inserted into template.s") parser.add_argument("rows", type=Path, help="one row per line: 4 u32 values (decimal or 0x hex)") parser.add_argument("--wave64", action="store_true") - parser.add_argument("--ieee", type=int, choices=(0, 1), default=1) - parser.add_argument("--denorm32", type=int, choices=range(4), default=3) - parser.add_argument("--denorm16", type=int, choices=range(4), default=3) + parser.add_argument("--ieee", type=int, choices=(0, 1), required=True) + parser.add_argument("--denorm32", type=int, choices=range(4), required=True) + parser.add_argument("--denorm16", type=int, choices=range(4), required=True) + parser.add_argument("--dx10-clamp", type=int, choices=(0, 1), required=True) + parser.add_argument("--round32", type=int, choices=range(4), required=True) + parser.add_argument("--round16", type=int, choices=range(4), required=True) + parser.add_argument("--fp16-overflow", type=int, choices=(0, 1), required=True) parser.add_argument("--coarse", action="store_true", help="input and output in coarse-grained GPU memory") parser.add_argument("--extra", type=Path, help="bytes appended after the rows") parser.add_argument("--outs", type=int, choices=range(1, 17), default=16, metavar="N", help="print v10..v(10+N-1)") @@ -98,7 +114,10 @@ def main(): if any(len(r) != 4 for r in rows): sys.exit("every row needs 4 values") extra = args.extra.read_bytes() if args.extra else b"" - for result in run(args.body.read_text(), rows, extra, args.wave64, args.ieee, args.denorm32, args.denorm16, args.coarse): + for result in run(args.body.read_text(), rows, extra, args.wave64, args.coarse, + ieee=args.ieee, denorm32=args.denorm32, denorm16=args.denorm16, + dx10_clamp=args.dx10_clamp, round32=args.round32, round16=args.round16, + fp16_overflow=args.fp16_overflow): print(" ".join(f"{v:08x}" for v in result[:args.outs])) diff --git a/tools/hw-oracle/template.s b/tools/hw-oracle/template.s index ad5426247..915409b61 100644 --- a/tools/hw-oracle/template.s +++ b/tools/hw-oracle/template.s @@ -47,6 +47,10 @@ probe: .amdhsa_float_denorm_mode_32 @DENORM@ .amdhsa_float_denorm_mode_16_64 @DENORM16@ .amdhsa_ieee_mode @IEEE@ + .amdhsa_dx10_clamp @DX10_CLAMP@ + .amdhsa_float_round_mode_32 @ROUND32@ + .amdhsa_float_round_mode_16_64 @ROUND16@ + .amdhsa_fp16_overflow @FP16_OVERFLOW@ .end_amdhsa_kernel .amdgpu_metadata diff --git a/tools/hw-oracle/test_hw_oracle.py b/tools/hw-oracle/test_hw_oracle.py new file mode 100644 index 000000000..15a33fd05 --- /dev/null +++ b/tools/hw-oracle/test_hw_oracle.py @@ -0,0 +1,147 @@ +import contextlib +import io +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import hw_oracle + + +MODES = dict(ieee=0, denorm32=0, denorm16=3, dx10_clamp=1, round32=0, round16=0, fp16_overflow=0) + + +class OracleCacheTests(unittest.TestCase): + def test_failed_build_is_not_cached_and_can_be_retried(self): + for previous in (None, b"old executable"): + with self.subTest(previous=previous), tempfile.TemporaryDirectory() as tmp: + cache = Path(tmp) / "cache" + cache.mkdir() + binary = cache / "oracle" + if previous is not None: + binary.write_bytes(previous) + os.utime(binary, (0, 0)) + + def fail(command, *, check): + self.assertTrue(check) + Path(command[command.index("-o") + 1]).write_bytes(b"partial executable") + raise subprocess.CalledProcessError(1, command) + + def succeed(command, *, check): + self.assertTrue(check) + Path(command[command.index("-o") + 1]).write_bytes(b"complete executable") + + with patch.object(hw_oracle, "CACHE", cache), patch.object(hw_oracle, "rocm_root", return_value=None), \ + patch.object(hw_oracle.subprocess, "run", side_effect=fail) as compiler: + with self.assertRaises(subprocess.CalledProcessError): + hw_oracle.oracle() + self.assertEqual(binary.read_bytes() if binary.exists() else None, previous) + self.assertEqual(list(cache.iterdir()), [binary] if previous is not None else []) + compiler.side_effect = succeed + self.assertEqual(hw_oracle.oracle(), binary) + self.assertEqual(binary.read_bytes(), b"complete executable") + self.assertEqual(hw_oracle.oracle(), binary) + self.assertEqual(compiler.call_count, 2) + self.assertEqual(list(cache.iterdir()), [binary]) + + def test_overlapping_builds_do_not_expose_partial_output(self): + with tempfile.TemporaryDirectory() as tmp: + cache = Path(tmp) / "cache" + binary = cache / "oracle" + outputs = [] + + def compile(command, *, check): + self.assertTrue(check) + output = Path(command[command.index("-o") + 1]) + outputs.append(output) + output.write_bytes(b"partial executable") + if len(outputs) == 1: + self.assertFalse(binary.exists()) + self.assertEqual(hw_oracle.oracle().read_bytes(), b"complete executable") + output.write_bytes(b"complete executable") + + with patch.object(hw_oracle, "CACHE", cache), patch.object(hw_oracle, "rocm_root", return_value=None), \ + patch.object(hw_oracle.subprocess, "run", side_effect=compile) as compiler: + self.assertEqual(hw_oracle.oracle(), binary) + self.assertEqual(binary.read_bytes(), b"complete executable") + self.assertEqual(compiler.call_count, 2) + self.assertNotEqual(outputs[0], outputs[1]) + self.assertEqual(list(cache.iterdir()), [binary]) + + +class FloatModeTests(unittest.TestCase): + def test_python_requires_each_mode(self): + for name in MODES: + with self.subTest(name=name): + modes = {key: value for key, value in MODES.items() if key != name} + with self.assertRaises(TypeError), patch.object(hw_oracle, "assemble") as assemble: + hw_oracle.run("s_nop 0", [], **modes) + assemble.assert_not_called() + + def test_invalid_modes_fail_before_tool_lookup(self): + with tempfile.TemporaryDirectory() as tmp, patch.object(hw_oracle, "target") as target: + for name in MODES: + limit = 4 if name in ("denorm32", "denorm16", "round32", "round16") else 2 + for value in (-1, limit, 0.5, "0"): + with self.subTest(name=name, value=value): + with self.assertRaisesRegex(ValueError, name): + hw_oracle.assemble("s_nop 0", Path(tmp), False, **(MODES | {name: value})) + target.assert_not_called() + + def test_cli_requires_each_mode(self): + flags = {name: ["--" + name.replace("_", "-"), str(value)] for name, value in MODES.items()} + for name in MODES: + argv = ["hw_oracle.py", "missing.s", "missing.txt"] + argv += [item for key, flag in flags.items() if key != name for item in flag] + with self.subTest(name=name), patch("sys.argv", argv), contextlib.redirect_stderr(io.StringIO()) as errors: + with self.assertRaises(SystemExit) as error: + hw_oracle.main() + self.assertEqual(error.exception.code, 2) + self.assertIn(flags[name][0], errors.getvalue()) + + def test_cli_forwards_explicit_modes(self): + with tempfile.TemporaryDirectory() as tmp: + body = Path(tmp) / "body.s" + rows = Path(tmp) / "rows.txt" + body.write_text("s_nop 0") + rows.write_text("1 2 3 4\n") + modes = dict(ieee=1, denorm32=2, denorm16=1, dx10_clamp=0, round32=3, round16=2, fp16_overflow=1) + argv = ["hw_oracle.py", str(body), str(rows), "--wave64", "--coarse"] + argv += [item for name, value in modes.items() for item in ["--" + name.replace("_", "-"), str(value)]] + with patch("sys.argv", argv), patch.object(hw_oracle, "run", return_value=[]) as run: + hw_oracle.main() + run.assert_called_once_with("s_nop 0", [(1, 2, 3, 4)], b"", True, True, **modes) + + def test_run_forwards_modes_to_assembler(self): + with patch.object(hw_oracle, "assemble") as assemble: + hw_oracle.run("s_nop 0", [], wave64=True, **MODES) + self.assertEqual(assemble.call_args.args[0], "s_nop 0") + self.assertTrue(assemble.call_args.args[2]) + self.assertEqual(assemble.call_args.kwargs, MODES) + + +class AssemblyTests(unittest.TestCase): + def test_condition_operands_match_wave_size(self): + for name in ("clang", "ld.lld"): + try: + hw_oracle.tool(name) + except SystemExit as error: + self.skipTest(str(error)) + targets = subprocess.run([hw_oracle.tool("clang"), "--print-targets"], capture_output=True, text=True).stdout + if "amdgcn" not in targets: + self.skipTest("clang has no AMDGPU target") + for wave64, vcc, sgpr in ((False, "vcc_lo", "s8"), (True, "vcc", "s[8:9]")): + body = (f" v_cmp_lt_f32 {vcc}, v4, v5\n" + f" v_cndmask_b32 v10, v6, v7, {vcc}\n" + f" v_cmp_lt_f32 {sgpr}, v4, v5\n" + f" v_cndmask_b32 v11, v6, v7, {sgpr}\n") + with self.subTest(wave64=wave64), tempfile.TemporaryDirectory() as tmp: + with patch.object(hw_oracle, "target", return_value="gfx1036"): + code = hw_oracle.assemble(body, Path(tmp), wave64, **MODES) + self.assertTrue(code.is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/import_audit.py b/tools/import_audit.py new file mode 100644 index 000000000..ea046eab5 --- /dev/null +++ b/tools/import_audit.py @@ -0,0 +1,322 @@ +import argparse +import json +import re +import struct +import subprocess +import sys +from collections import defaultdict +from pathlib import Path + +import nid_names +import progress + +ROOT = Path(__file__).resolve().parent.parent +PRX = ROOT / "core" / "libs" / "prx" +NID = re.compile(r"[A-Za-z0-9+\-]{11}") +EXPORT_ALIAS = re.compile(r'APS5_EXPORT\("([A-Za-z0-9+\-]{11})",\s*(\w+)\)') +NID_POSTFIX = "_nid_postfix" +CLASSES = ("implemented", "stub", "absent", "module") +PE_DIRECTORY_OFFSET = {0x20b: 112, 0x10b: 96} + + +class AuditError(Exception): + pass + + +def take(data, offset, size, path, what): + if offset < 0 or size < 0 or offset + size > len(data): + raise AuditError(f"{path}: {what} at 0x{offset:x} with size {size} lies outside the file ({len(data)} bytes)") + return data[offset:offset + size] + + +def unpack(layout, data, offset, path, what): + return struct.unpack(layout, take(data, offset, struct.calcsize(layout), path, what)) + + +def cstring(data, offset, limit, path, what): + end = data.find(b"\0", offset, limit) + if offset < 0 or offset >= limit or end < 0: + raise AuditError(f"{path}: {what} at 0x{offset:x} is not a NUL-terminated string inside 0x{limit:x} bytes") + try: + return data[offset:end].decode("utf-8") + except UnicodeDecodeError as error: + raise AuditError(f"{path}: {what} at 0x{offset:x} is not UTF-8: {error}") from error + + +def elf_exports(path, data): + header = unpack("<16sHHIQQQIHHHHHH", data, 0, path, "ELF header") + ident, section_offset, section_size, section_count = header[0], header[6], header[11], header[12] + if ident[4] != 2 or ident[5] != 1 or section_size != 64: + raise AuditError(f"{path}: only little-endian ELF64 with 64-byte section headers is supported (class {ident[4]}, data {ident[5]}, shentsize {section_size})") + sections = [unpack("= len(sections) or sections[link][1] != 3: + raise AuditError(f"{path}: invalid dynamic symbol table (entry size {entry}, size {size}, string table section {link})") + strings_offset, strings_size = sections[link][4], sections[link][5] + take(data, strings_offset, strings_size, path, "dynamic string table") + for position in range(0, size, 24): + name, info, other, index, _, _ = unpack("> 4, other & 3 + if index == 0 or binding not in (1, 2, 10) or visibility not in (0, 3): + continue + names.add(cstring(data, strings_offset + name, strings_offset + strings_size, path, "export name")) + return names + + +def pe_exports(path, data): + header = unpack("= raw_size or length > raw_size - relative: + continue + take(data, raw_offset + relative, length, path, f"data at RVA 0x{address:x}") + return raw_offset + relative + raise AuditError(f"{path}: RVA 0x{address:x} is not inside any section") + + table = unpack("6} imports {summary['references_by_class'][kind]:>6} references") + for title, kind in (("Absent: no built library exports these, so the loader fails", "absent"), ("Stub: exported but throws when called", "stub")): + selected = [record for record in records if record["class"] == kind] + if selected: + lines.append("") + lines.append(f"{title} ({len(selected)})") + current = None + for record in selected: + if record["library"] != current: + current = record["library"] + lines.append(f" [{current or 'library not named'}]") + lines.append(f" {label(record)}") + if missing: + lines.append("") + lines.append(f"Needed libraries with no file in --libs or --modules ({len(missing)})") + lines.extend(f" {library}: {count} imports" for library, count in missing.items()) + if summary["library_mismatch"]: + lines.append("") + lines.append(f"{summary['library_mismatch']} imports are exported only by a library other than the one they name: they resolve on Linux and can fail on Windows") + return "\n".join(lines) + + +def main(argv=None): + parser = argparse.ArgumentParser(description="classify the imports of a converted game against the built system libraries") + parser.add_argument("registries", nargs="+", type=Path, help="JSON written by 'relinker --registry'") + parser.add_argument("--libs", action="append", required=True, help="directory of built .prx files; repeat for several") + parser.add_argument("--modules", action="append", default=[], help="directory of the title's own modules; libraries named there are not checked") + parser.add_argument("--source", default=str(PRX), help="core/libs/prx tree used to find throwing stubs (default: this repository's)") + parser.add_argument("--names", help="aerolib.csv style file (NID, name per line) used to name absent imports") + parser.add_argument("--json", type=Path, help="write the full result to this file") + args = parser.parse_args(argv) + try: + imports = [entry for registry in args.registries for entry in read_registry(registry)] + exports, libraries = built_libraries(args.libs) + modules = module_files(args.modules) + records, missing = audit(imports, exports, source_stubs(args.source), libraries, modules) + if args.names: + attach_names(records, load_names(args.names)) + summary = summarize(records, len(imports), missing) + if args.json: + commit, dirty = git_stamp() + result = {"source_commit": commit, "source_dirty": dirty, "registries": [str(path) for path in args.registries], **summary, "imports": records} + args.json.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + except AuditError as error: + print(f"FAIL: {error}", file=sys.stderr) + return 2 + print(render(summary, records, missing)) + return 1 if summary["unique_by_class"]["absent"] or missing else 0 + + +if __name__ == "__main__": + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + raise SystemExit(main()) diff --git a/tools/pr_overlap.py b/tools/pr_overlap.py index ab94229c8..db2bbaa90 100644 --- a/tools/pr_overlap.py +++ b/tools/pr_overlap.py @@ -14,11 +14,11 @@ MARKER = "" def git(*args): - return subprocess.run(["git", *args], capture_output=True, text=True, check=True).stdout + return subprocess.run(["git", *args], capture_output=True, text=True, encoding="utf-8", check=True).stdout def gh(*args): - return subprocess.run(["gh", *args], capture_output=True, text=True, check=True).stdout + return subprocess.run(["gh", *args], capture_output=True, text=True, encoding="utf-8", check=True).stdout def upstream(): @@ -46,15 +46,22 @@ def fetch(base, prs): def merge(*args): result = subprocess.run(["git", "merge-tree", "--write-tree", "--name-only", "--no-messages", *args], - capture_output=True, text=True) + capture_output=True, text=True, encoding="utf-8") lines = result.stdout.splitlines() + if result.returncode not in (0, 1) or not lines: + raise RuntimeError(f"git merge-tree {' '.join(args)} failed: {result.stderr.strip()}") return lines[0], set(lines[1:]) if result.returncode else set() +def commit(tree): + return git("-c", "user.name=pr_overlap", "-c", "user.email=pr_overlap", "commit-tree", tree, + "-p", "refs/pr/base", "-m", "pr_overlap").strip() + + def scan(pr): tree, conflicted = merge("refs/pr/base", pr["ref"]) - pr["tree"] = None if conflicted else tree - old, new = (git("merge-base", "refs/pr/base", pr["ref"]).strip(), pr["ref"]) if conflicted else ("refs/pr/base", tree) + pr["tree"] = None if conflicted else commit(tree) + old, new = (git("merge-base", "refs/pr/base", pr["ref"]).strip(), pr["ref"]) if conflicted else ("refs/pr/base", pr["tree"]) pr["files"], pr["added"] = set(), set() for line in git("diff", "--name-status", "--no-renames", old, new).splitlines(): status, path = line.split("\t", 1) diff --git a/tools/progress.py b/tools/progress.py index fcb383408..dc566a69a 100644 --- a/tools/progress.py +++ b/tools/progress.py @@ -53,10 +53,6 @@ OPCODE_VARIANTS = { "VMacLegacyF32": ("V_FMAC_LEGACY_F32",), "VMadLegacyF32": ("V_FMA_LEGACY_F32",), "ImageSample": ("IMAGE_SAMPLE_L", "IMAGE_SAMPLE_B", "IMAGE_SAMPLE_C_LZ", "IMAGE_SAMPLE_L_O", "IMAGE_SAMPLE_D_CL_O"), - "ImageLoadPck": ("IMAGE_LOAD_PCK2", "IMAGE_LOAD_PCK4"), - "ImageLoadMipPck": ("IMAGE_LOAD_MIP_PCK2", "IMAGE_LOAD_MIP_PCK4"), - "ImageStorePck": ("IMAGE_STORE_PCK2", "IMAGE_STORE_PCK4"), - "ImageStoreMipPck": ("IMAGE_STORE_MIP_PCK2", "IMAGE_STORE_MIP_PCK4"), } REPORT_ROWS = 100 PANEL_WIDTH, GAP, MAP_HEIGHT, HEADER = 495, 10, 280, 30 @@ -86,6 +82,8 @@ def stub_calls(text): def scan_library(path): done, todo = set(), set() for source in path.rglob("*.cpp"): + if "tests" in source.relative_to(path).parts: + continue text = source.read_text(errors="ignore") calls = stub_calls(text) for match in DEFINITION.finditer(text): @@ -95,8 +93,14 @@ def scan_library(path): body = text[match.end() - 1:body_end(text, match.end() - 1)] (todo if any(call in body for call in calls) else done).add(name) todo -= done - return {"name": path.name, "label": path.name.removeprefix("libSce"), "done": len(done), "todo": len(todo), - "done_names": sorted(done), "todo_names": sorted(todo)} + group = {"name": path.name, "label": path.name.removeprefix("libSce"), "done": len(done), "todo": len(todo), + "done_names": sorted(done), "todo_names": sorted(todo)} + cmake = path / "CMakeLists.txt" + if not (done or todo) and cmake.is_file(): + shared = re.search(r'^\s*include\(\s*"?\$\{CMAKE_CURRENT_SOURCE_DIR\}/\.\./([\w.-]+)/[\w.-]+\.cmake"?\s*\)', cmake.read_text(), re.M) + if shared and (path.parent / shared[1]).is_dir() and shared[1] != path.name: + group["shared_sources"] = shared[1] + return group def summarize(groups): @@ -205,7 +209,9 @@ def treemap(title, data, left): parts = [text(left + 4, 21, f'{title}: {data["percent"]}% ({data["done"]}/{data["total"]})', 16)] for group, (x, y, w, h) in zip(groups, squarify([g["done"] + g["todo"] for g in groups], left, HEADER, PANEL_WIDTH, MAP_HEIGHT)): total = group["done"] + group["todo"] - parts.append(f'{escape(group["name"])}: {group["done"]}/{total} ({100 * group["done"] / total:.0f}%)') + shared = [g["name"] for g in data["groups"] if g.get("shared_sources") == group["name"]] + suffix = "; shared by " + ", ".join(shared) if shared else "" + parts.append(f'{escape(group["name"])}: {group["done"]}/{total} ({100 * group["done"] / total:.0f}%){escape(suffix)}') for i, cell in enumerate(cells(total, x + 1, y + 1, w - 2, h - 2)): parts.append(rect(*cell, DONE_COLOR if i < group["done"] else TODO_COLOR)) parts.append(f'') @@ -242,6 +248,11 @@ def table(heading, column, data): rows = [f"

{heading}

", "", f""] for group in sorted(data["groups"], key=lambda g: g["name"].lower()): + if "shared_sources" in group: + owner = escape(group["shared_sources"]) + rows.append(f'') + continue total = group["done"] + group["todo"] percent = f'{100 * group["done"] / total:.0f}%' if total else "-" rows.append(f'') @@ -266,7 +277,8 @@ def summary(libraries, shaders): table("System libraries", "Library", libraries), "

A function is implemented when it no longer calls NotImplemented_nid_no_patch. " f'The total only includes functions already declared in core/libs/prx, ' - "not every function exported by the PS5 firmware.

", + "not every function exported by the PS5 firmware. Shared-source libraries are counted only at their source library. " + f'Counting rules.

', table("GPU shader instructions", "Encoding", shaders), f'

The total is the AMD RDNA 1 + RDNA 2 instruction list (tools/rdna_isa.txt). ' f'An instruction is implemented when the decoder recognizes it. ' @@ -275,8 +287,9 @@ def summary(libraries, shaders): ]) + "\n" -def names(data, state): - return {(g["name"], n) for g in data["groups"] for n in g.get(f"{state}_names", [])} +def names(data, state, owners=None): + owners = owners or {} + return {(owners.get(g["name"], g["name"]), n) for g in data["groups"] for n in g.get(f"{state}_names", [])} def details(icon, title, column, items): @@ -290,21 +303,26 @@ def details(icon, title, column, items): def compare(title, column, unit, base, head): - base_done, head_done = names(base, "done"), names(head, "done") - base_all, head_all = base_done | names(base, "todo"), head_done | names(head, "todo") + owners = {g["name"]: g["shared_sources"] for g in head["groups"] if "shared_sources" in g} + previous = {g["name"]: g.get("shared_sources") for g in base["groups"]} + shared = {(name, owner) for name, owner in owners.items() if previous.get(name) != owner} + base_done, head_done = names(base, "done", owners), names(head, "done", owners) + base_all, head_all = base_done | names(base, "todo", owners), head_done | names(head, "todo", owners) implemented, declared = head_done - base_done, head_all - base_all - head_done regressed, removed = base_done & (head_all - head_done), base_all - head_all - if not (implemented or declared or regressed or removed): + if not (implemented or declared or regressed or removed or shared): return [] delta = round(head["percent"] - base["percent"], 2) icon = "📈" if delta > 0 else "📉" if delta < 0 else "➖" counts = [f"{n:+} {label}" for n, label in ((len(implemented), "implemented"), (len(declared), "declared"), (-len(regressed), "reverted"), (-len(removed), "removed")) if n] - lines = [f'{icon} **{title}**: {head["percent"]}% ({delta:+}%, {", ".join(counts)} {unit})', ""] + changes = f', {", ".join(counts)} {unit}' if counts else "" + lines = [f'{icon} **{title}**: {head["percent"]}% ({delta:+}%{changes})', ""] lines += details("✅", "implemented", column, implemented) lines += details("🆕", "declared as stubs", column, declared) lines += details("⚠️", "went back to stubs", column, regressed) lines += details("🗑️", "removed", column, removed) + lines += details("🔗", "now sharing sources", column, shared) return lines + [""] diff --git a/tools/tests/test_import_audit.py b/tools/tests/test_import_audit.py new file mode 100644 index 000000000..860a5b9e5 --- /dev/null +++ b/tools/tests/test_import_audit.py @@ -0,0 +1,278 @@ +import contextlib +import io +import json +import struct +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import import_audit +import nid_names + +GLOBAL_FUNCTION = 0x12 +LOCAL_FUNCTION = 0x02 +STUB_SOURCE = "int APS5_VABI {name}(int value) {{ NotImplemented_nid_no_patch(__func__); return 0; }}\n" +DONE_SOURCE = "int APS5_VABI {name}(int value) {{ return value; }}\n" +MESSAGE_NID = "wTpfglkmv34" +MESSAGE_NAME = "sceMsgDialogProgressBarSetValue" + + +def build_elf(symbols): + strings = b"\0" + entries = [struct.pack("Shared sources:', html) + self.assertIn("shared by " + alias.name, "\n".join(progress.treemap("Libraries", head, 0))) + report = "\n".join(progress.compare("Libraries", "Library", "functions", base, head)) + self.assertIn("sharing sources", report) + self.assertNotIn("removed", report) + self.assertNotIn("implemented", report) + self.assertEqual(progress.compare("Libraries", "Library", "functions", head, head), []) + (owner / "Export.cpp").write_text(source.replace("return 0;", "NotImplemented_nid_no_patch(__func__);")) + regressed = progress.collect_libraries() + report = "\n".join(progress.compare("Libraries", "Library", "functions", base, regressed)) + self.assertIn("-1 reverted", report) + self.assertIn("Ready", report) + (owner / "Export.cpp").write_text("int APS5_VABI Ready() { return 0; }\n") + removed = progress.collect_libraries() + report = "\n".join(progress.compare("Libraries", "Library", "functions", base, removed)) + self.assertIn("-1 removed", report) + self.assertIn("Pending", report) + + +if __name__ == "__main__": + unittest.main()

{column}ImplementedTotal%
{escape(group["name"])}Shared sources: ' + f'{owner}
{escape(group["name"])}{group["done"]}{total}{percent}